def wrapper(f): hyp_func = hy.seed(0)(hy.settings(max_examples=1)(hy.given( *given_args, **given_kwargs)(f))) fixed_seed_func = hy.seed(0)(hy.settings(max_examples=1)(hy.given( *given_args, **given_kwargs)(f))) def func(self, *args, **kwargs): self.should_serialize = True fixed_seed_func(self, *args, **kwargs) self.should_serialize = False hyp_func(self, *args, **kwargs) return func
def create_test(endpoint: Endpoint, test: Callable, settings: Optional[hypothesis.settings] = None, seed: Optional[int] = None) -> Callable: """Create a Hypothesis test.""" hook_dispatcher = getattr(test, "_schemathesis_hooks", None) feedback: Optional[Feedback] if endpoint.schema.stateful == Stateful.links: feedback = Feedback(endpoint.schema.stateful, endpoint) else: feedback = None strategy = endpoint.as_strategy(hooks=hook_dispatcher, feedback=feedback) wrapped_test = hypothesis.given(case=strategy)(test) if seed is not None: wrapped_test = hypothesis.seed(seed)(wrapped_test) if asyncio.iscoroutinefunction(test): wrapped_test.hypothesis.inner_test = make_async_test( test) # type: ignore setup_default_deadline(wrapped_test) if settings is not None: wrapped_test = settings(wrapped_test) wrapped_test._schemathesis_feedback = feedback # type: ignore return add_examples(wrapped_test, endpoint, hook_dispatcher=hook_dispatcher)
def create_test( *, operation: APIOperation, test: Callable, settings: Optional[hypothesis.settings] = None, seed: Optional[int] = None, data_generation_method: DataGenerationMethod = DataGenerationMethod. default(), _given_args: Tuple[GivenInput, ...] = (), _given_kwargs: Optional[Dict[str, GivenInput]] = None, ) -> Callable: """Create a Hypothesis test.""" hook_dispatcher = getattr(test, "_schemathesis_hooks", None) strategy = operation.as_strategy( hooks=hook_dispatcher, data_generation_method=data_generation_method) _given_kwargs = (_given_kwargs or {}).copy() _given_kwargs.setdefault("case", strategy) wrapped_test = hypothesis.given(*_given_args, **_given_kwargs)(test) if seed is not None: wrapped_test = hypothesis.seed(seed)(wrapped_test) if asyncio.iscoroutinefunction(test): wrapped_test.hypothesis.inner_test = make_async_test( test) # type: ignore setup_default_deadline(wrapped_test) if settings is not None: wrapped_test = settings(wrapped_test) existing_settings = getattr(wrapped_test, "_hypothesis_internal_use_settings", None) if existing_settings and Phase.explicit in existing_settings.phases: wrapped_test = add_examples(wrapped_test, operation, hook_dispatcher=hook_dispatcher) return wrapped_test
def create_test( *, endpoint: Endpoint, test: Callable, settings: Optional[hypothesis.settings] = None, seed: Optional[int] = None, _given_args: Tuple[GivenInput, ...] = (), _given_kwargs: Optional[Dict[str, GivenInput]] = None, ) -> Callable: """Create a Hypothesis test.""" hook_dispatcher = getattr(test, "_schemathesis_hooks", None) feedback: Optional[Feedback] if endpoint.schema.stateful == Stateful.links: feedback = Feedback(endpoint.schema.stateful, endpoint) else: feedback = None strategy = endpoint.as_strategy(hooks=hook_dispatcher, feedback=feedback) _given_kwargs = (_given_kwargs or {}).copy() _given_kwargs.setdefault("case", strategy) wrapped_test = hypothesis.given(*_given_args, **_given_kwargs)(test) if seed is not None: wrapped_test = hypothesis.seed(seed)(wrapped_test) if asyncio.iscoroutinefunction(test): wrapped_test.hypothesis.inner_test = make_async_test(test) # type: ignore setup_default_deadline(wrapped_test) if settings is not None: wrapped_test = settings(wrapped_test) wrapped_test._schemathesis_feedback = feedback # type: ignore return add_examples(wrapped_test, endpoint, hook_dispatcher=hook_dispatcher)
def decorator(test_method): repro_seed = hypothesis_reproduction_seed() if repro_seed is not None: # This implements the semantics of TFP_HYPOTHESIS_REPRODUCE via # the `hp.reproduce_failure` decorator. test_method = hp.reproduce_failure('3.56.5', repro_seed)(test_method) elif randomize_hypothesis(): # Hypothesis defaults to seeding its internal PRNG from the system time, # so since we actually want randomization (including across machines) we # have to force it. entropy = os.urandom(64) test_method = hp.seed(int.from_bytes(entropy, 'big'))(test_method) return hp.settings(**kwds)(test_method)
def create_test( endpoint: Endpoint, test: Callable, settings: Optional[hypothesis.settings] = None, seed: Optional[int] = None ) -> Callable: """Create a Hypothesis test.""" hook_dispatcher = getattr(test, "_schemathesis_hooks", None) strategy = endpoint.as_strategy(hooks=hook_dispatcher) wrapped_test = hypothesis.given(case=strategy)(test) if seed is not None: wrapped_test = hypothesis.seed(seed)(wrapped_test) if asyncio.iscoroutinefunction(test): wrapped_test.hypothesis.inner_test = make_async_test(test) # type: ignore if settings is not None: wrapped_test = settings(wrapped_test) return add_examples(wrapped_test, endpoint, hook_dispatcher=hook_dispatcher)
def create_test( endpoint: Endpoint, test: Callable, settings: Optional[hypothesis.settings] = None, seed: Optional[int] = None ) -> Callable: """Create a Hypothesis test.""" strategy = endpoint.as_strategy() wrapped_test = hypothesis.given(case=strategy)(test) if seed is not None: wrapped_test = hypothesis.seed(seed)(wrapped_test) original_test = get_original_test(test) if asyncio.iscoroutinefunction(original_test): wrapped_test.hypothesis.inner_test = make_async_test(original_test) # type: ignore if settings is not None: wrapped_test = settings(wrapped_test) return add_examples(wrapped_test, endpoint)
def accept(test): verbose = os.environ.get('VERBOSE') == 'true' shrink = os.environ.get('SHRINK', 'true') == 'true' provided_seed = int(os.environ['SEED']) if verbose: verbosity = Verbosity.debug else: verbosity = Verbosity.normal seen_failure = False evaluations = 0 original_test = test @proxies(test) def recording_test(*args, **kwargs): nonlocal seen_failure, evaluations if seen_failure: evaluations += 1 try: return original_test(*args, **kwargs) except HypothesisException: raise except Exception: if not seen_failure: seen_failure = True evaluations += 1 raise test = seed(provided_seed)(settings( database=None, max_examples=10**6, suppress_health_check=HealthCheck.all(), verbosity=verbosity, phases=[Phase.generate, Phase.shrink] if shrink else [Phase.generate])(given(strat)(recording_test))) try: test() except Exception: if verbose: traceback.print_exc() print("EVALUATIONS:", evaluations) print("TEST FAILED") return print("Expected test to fail with assertion error", file=sys.stderr) sys.exit(1)
def decorator(test_method): repro_seed = hypothesis_reproduction_seed() if repro_seed is not None: # This implements the semantics of TFP_HYPOTHESIS_REPRODUCE via # the `hp.reproduce_failure` decorator. # TODO(jburnim): Catch Hypothesis version mismatches. test_method = hp.reproduce_failure(hp.__version__, repro_seed)(test_method) elif randomize_hypothesis(): # Hypothesis defaults to seeding its internal PRNG from the system time, # so since we actually want randomization (including across machines) we # have to force it. entropy = os.urandom(64) test_method = hp.seed(int.from_bytes(entropy, 'big'))(test_method) if hp.__version_info__ >= (4,): # TODO(jburnim): Comment this. timeout = kwds.pop('timeout') ret = hp.settings(**kwds)(test_method) object.__setattr__(ret._hypothesis_internal_use_settings, 'timeout', timeout) else: ret = hp.settings(**kwds)(test_method) return ret
def create_test( *, operation: APIOperation, test: Callable, settings: Optional[hypothesis.settings] = None, seed: Optional[int] = None, data_generation_method: DataGenerationMethod = DataGenerationMethod.default(), _given_args: Tuple[GivenInput, ...] = (), _given_kwargs: Optional[Dict[str, GivenInput]] = None, ) -> Callable: """Create a Hypothesis test.""" hook_dispatcher = getattr(test, "_schemathesis_hooks", None) strategy = operation.as_strategy(hooks=hook_dispatcher, data_generation_method=data_generation_method) _given_kwargs = (_given_kwargs or {}).copy() _given_kwargs.setdefault("case", strategy) # Each generated test should be a unique function. It is especially important for the case when Schemathesis runs # tests in multiple threads because Hypothesis stores some internal attributes on function objects and re-writing # them from different threads may lead to unpredictable side-effects. @proxies(test) # type: ignore def test_function(*args: Any, **kwargs: Any) -> Any: return test(*args, **kwargs) wrapped_test = hypothesis.given(*_given_args, **_given_kwargs)(test_function) if seed is not None: wrapped_test = hypothesis.seed(seed)(wrapped_test) if asyncio.iscoroutinefunction(test): wrapped_test.hypothesis.inner_test = make_async_test(test) # type: ignore setup_default_deadline(wrapped_test) if settings is not None: wrapped_test = settings(wrapped_test) existing_settings = getattr(wrapped_test, "_hypothesis_internal_use_settings", None) if existing_settings and Phase.explicit in existing_settings.phases: wrapped_test = add_examples(wrapped_test, operation, hook_dispatcher=hook_dispatcher) return wrapped_test
# v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at http://mozilla.org/MPL/2.0/. # # END HEADER from __future__ import division, print_function, absolute_import import pytest from hypothesis import seed, given, settings from hypothesis import strategies as st from tests.common.utils import validate_deprecation from hypothesis.database import InMemoryExampleDatabase @pytest.mark.parametrize('dec', [ settings(database=InMemoryExampleDatabase(), derandomize=True), seed(1) ]) def test_deprecated_determinism_with_database(dec): @dec @given(st.booleans()) def test(i): raise ValueError() with pytest.raises(ValueError): test() with validate_deprecation(): with pytest.raises(ValueError): test()
import numpy as np import pandas as pd import pytest from hypothesis import given, assume, example, seed from hypothesis.extra.numpy import arrays from hypothesis.strategies import integers, floats, composite, lists, one_of, text from hypothesis.extra.pandas import data_frames, column, range_indexes, series from toy.hypothesis.random_funcs import fibonacci, mean_absolute_error, error seed(2018) @given(integers(min_value=0, max_value=100)) @example(1) @example(0) def test_fibonacci(x): fib = fibonacci(x) if x >= 2: fib_less_1 = fibonacci(x - 1) fib_less_2 = fibonacci(x - 2) assert fib == fib_less_1 + fib_less_2 assert fib >= fib_less_1 assert fib_less_1 >= fib_less_2 elif x == 1: assert fib == 1 else: assert fib == 0 assert isinstance(fib, int)
'rfc_822_datetime': _build_path('rfc_822_datetime.lark'), 'rfc_1738': _build_path('rfc_1738.lark'), 'rfc_2397': _build_path('rfc_2397.lark'), 'rfc_2396': _build_path('rfc_2396.lark'), 'rfc_6531': _build_path('rfc_6531.lark'), 'rfc_5321': _build_path('rfc_5321.lark'), 'rfc_5545': _build_path('rfc_5545.lark'), 'robotstxt': _build_path('robotstxt.lark'), 'subunit_v1': _build_path('subunit_v1.lark'), 'tap13': _build_path('tap13.lark'), 'toml': _build_path('toml.lark'), 'yaml': _build_path('yaml.lark') } if __name__ == '__main__': hypothesis.seed(time.time()) parser = argparse.ArgumentParser(description='Generate grammar samples.') parser.add_argument('--grammar', dest='grammar', help='file with grammar syntax') parser.add_argument('--start', dest=DEFAULT_START, help='start terminal') args = parser.parse_args() if (not args.grammar): sys.exit(1) if args.grammar == 'mime.lark': sys.setrecursionlimit(10000) with open(args.grammar, 'r') as grammar: sample = from_lark(Lark(grammar, start=args.start)).example() print('{}'.format(sample))
# obtain one at http://mozilla.org/MPL/2.0/. # # END HEADER from __future__ import division, print_function, absolute_import import pytest from hypothesis import seed, given, settings from hypothesis import strategies as st from tests.common.utils import validate_deprecation from hypothesis.database import InMemoryExampleDatabase @pytest.mark.parametrize( 'dec', [settings(database=InMemoryExampleDatabase(), derandomize=True), seed(1)]) def test_deprecated_determinism_with_database(dec): @dec @given(st.booleans()) def test(i): raise ValueError() with pytest.raises(ValueError): test() with validate_deprecation(): with pytest.raises(ValueError): test()
class cases: # noqa: N """Generate test cases for the given function. """ func: typing.Callable """the function to test. Should be type annotated.""" count: int """how many test cases to generate, defaults to 50.""" kwargs: typing.Dict[str, typing.Any] """keyword arguments to pass into the function.""" check_types: bool """check that the result matches return type of the function. Enabled by default.""" settings: hypothesis.settings """Hypothesis settings to use instead of default ones.""" seed: typing.Optional[int] """Random seed to use when generating test cases. Use it to make tests deterministic.""" def __init__( self, func: typing.Callable, *, count: int = 50, kwargs: typing.Dict[str, typing.Any] = None, check_types: bool = True, settings: typing.Optional[hypothesis.settings] = None, seed: typing.Optional[int] = None, ) -> None: """ Create test cases generator. ```pycon >>> import deal >>> @deal.pre(lambda a, b: b != 0) ... def div(a: int, b: int) -> float: ... return a / b ... >>> cases = deal.cases(div) >>> ``` """ self.func = func # type: ignore self.count = count self.kwargs = kwargs or {} self.check_types = check_types self.settings = settings or self._default_settings self.seed = seed def __iter__(self) -> typing.Iterator[TestCase]: """Emits test cases. It can be helpful when you want to see what test cases are generated. The recommend way is to use `deal.cases` as a decorator instead. ```pycon >>> import deal >>> @deal.pre(lambda a, b: b != 0) ... def div(a: int, b: int) -> float: ... return a / b ... >>> cases = iter(deal.cases(div)) >>> next(cases) TestCase(args=(), kwargs=..., func=<function div ...>, exceptions=(), check_types=True) >>> for case in cases: ... result = case() # execute the test case >>> ``` """ cases: typing.List[TestCase] = [] test = self(cases.append) test() yield from cases def __repr__(self) -> str: args = [ getattr(self.func, '__name__', repr(self.func)), 'count={}'.format(self.count), ] if self.seed is not None: args.append('seed={}'.format(self.seed)) if self.kwargs: args.append('kwargs={!r}'.format(self.kwargs)) return 'deal.cases({})'.format(', '.join(args)) def make_case(self, *args, **kwargs) -> TestCase: """Make test case with the given arguments. """ return TestCase( args=args, kwargs=kwargs, func=self.func, exceptions=self.exceptions, check_types=self.check_types, ) @cached_property def validators(self) -> typing.Tuple[typing.Callable, ...]: """Returns pre-condition validators. It is used in the process of generating hypothesis strategies To let hypothesis more effectively avoid wrong input values. """ return tuple(self._get_validators(self.func)) @staticmethod def _get_validators(func: typing.Any) -> typing.Iterator[typing.Callable]: while True: if getattr(func, '__closure__', None): for cell in func.__closure__: obj = cell.cell_contents if isinstance(obj, Pre): yield obj.validate if not hasattr(func, '__wrapped__'): return func = func.__wrapped__ @cached_property def exceptions(self) -> typing.Tuple[typing.Type[Exception], ...]: """ Returns exceptions that will be suppressed by individual test cases. The exceptions are extracted from `@deal.raises` of the tested function. """ return tuple(self._get_excs(self.func)) @staticmethod def _get_excs(func: typing.Any) -> typing.Iterator[typing.Type[Exception]]: while True: if getattr(func, '__closure__', None): for cell in func.__closure__: obj = cell.cell_contents if isinstance(obj, Raises): yield from obj.exceptions if not hasattr(func, '__wrapped__'): return func = func.__wrapped__ @cached_property def strategy(self) -> hypothesis.strategies.SearchStrategy: """Hypothesis strategy that is used to generate test cases. """ kwargs = self.kwargs.copy() for name, value in kwargs.items(): if isinstance(value, hypothesis.strategies.SearchStrategy): continue kwargs[name] = hypothesis.strategies.just(value) def pass_along_variables(*args, **kwargs) -> ArgsKwargsType: return args, kwargs pass_along_variables.__signature__ = signature(self.func) # type: ignore update_wrapper(wrapper=pass_along_variables, wrapped=self.func) return hypothesis.strategies.builds(pass_along_variables, **kwargs) @property def _default_settings(self) -> hypothesis.settings: return hypothesis.settings( database=None, max_examples=self.count, # avoid showing deal guts verbosity=hypothesis.Verbosity.quiet, # raise the original exception instead of a fake one report_multiple_bugs=False, # print how to reproduce the failure print_blob=True, # if too many cases rejected, it is deal to blame suppress_health_check=[hypothesis.HealthCheck.filter_too_much], ) @typing.overload def __call__(self, test_func: F) -> F: """Wrap a function to turn it into a proper Hypothesis test. This is the recommend way to use `deal.cases`. It is powerful and extendable. ```python >>> import deal >>> @deal.pre(lambda a, b: b != 0) ... def div(a: int, b: int) -> float: ... return a / b ... >>> @deal.cases(div) ... def test_div(case): ... ... # do something before ... case() # run the test case ... ... # do something after ... >>> test_div() # run all test cases for `div` >>> ``` """ @typing.overload def __call__(self) -> None: """Generate and run tests for a function. This is the fastest way to generate tests for a function. ```python >>> import deal >>> @deal.pre(lambda a, b: b != 0) ... def div(a: int, b: int) -> float: ... return a / b ... >>> test_div = deal.cases(div) >>> test_div() # run the test ``` """ @typing.overload def __call__(self, buffer: FuzzInputType) -> typing.Optional[bytes]: """Use a function as a fuzzing target. This is a way to provide a random buffer for Hypothesis. It can be helpful for heavy testing of something really critical. ```python >>> import deal >>> @deal.pre(lambda a, b: b != 0) ... def div(a: int, b: int) -> float: ... return a / b ... >>> import atheris >>> test_div = deal.cases(div) >>> atheris.Setup([], test_div) ... >>> atheris.Fuzz() ... ``` """ def __call__(self, target=None): """Allows deal.cases to be used as decorator, test function, or fuzzing target. """ __tracebackhide__ = True if target is None: self._run() return None if callable(target): return self._wrap(target) return self._run.hypothesis.fuzz_one_input(target) # a hack to make the test discoverable by pytest @property def __func__(self) -> F: return self._run @cached_property def _run(self) -> F: return self._wrap(lambda case: case()) def _wrap(self, test_func: F) -> F: def wrapper(case: ArgsKwargsType, *args, **kwargs) -> None: ex = case __tracebackhide__ = True for validator in self.validators: try: validator(*ex[0], **ex[1]) except Exception: hypothesis.reject() case = self.make_case(*ex[0], **ex[1]) test_func(case, *args, **kwargs) wrapper = self._impersonate(wrapper=wrapper, wrapped=test_func) wrapper = hypothesis.given(case=self.strategy)(wrapper) wrapper = self.settings(wrapper) if self.seed is not None: wrapper = hypothesis.seed(self.seed)(wrapper) return wrapper
# v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at http://mozilla.org/MPL/2.0/. # # END HEADER from __future__ import division, print_function, absolute_import import pytest from hypothesis import strategies as st from hypothesis import seed, given, settings from tests.common.utils import validate_deprecation from hypothesis.database import InMemoryExampleDatabase @pytest.mark.parametrize('dec', [ settings(database=InMemoryExampleDatabase(), derandomize=True), seed(1) ]) def test_deprecated_determinism_with_database(dec): @dec @given(st.booleans()) def test(i): raise ValueError() with pytest.raises(ValueError): test() with validate_deprecation(): with pytest.raises(ValueError): test()