def test_detection_of_stateful_tests(): class Stuff(RuleBasedStateMachine): @rule(x=integers()) def a_rule(self, x): pass assert is_hypothesis_test(Stuff.TestCase().runTest)
def test_detection_of_methods(): class Foo(object): @given(integers()) def test(self, i): pass assert is_hypothesis_test(Foo().test)
def test_detection_of_methods(): class Foo: @given(integers()) def test(self, i): pass assert is_hypothesis_test(Foo().test)
def run( self, report: Optional[unittest.result.TestResult] = None ) -> unittest.result.TestResult: """ Some slightly awful magic here to arrange for setUp and tearDown to be called at the appropriate times when hypothesis is enabled for a test case. This can be removed once a future version of hypothesis ships with support for this baked in. """ if is_hypothesis_test(getattr(self, self._testMethodName)): try: old_setUp = self.setUp old_tearDown = self.tearDown self.setUp = lambda: None self.tearDown = lambda: None self.setup_example = old_setUp self.teardown_example = lambda _: old_tearDown() return super(EdenTestCase, self).run(report) finally: self.setUp = old_setUp self.tearDown = old_tearDown del self.setup_example del self.teardown_example else: return super(EdenTestCase, self).run(report)
def pytest_runtest_call(item): if not (hasattr(item, 'obj') and is_hypothesis_test(item.obj)): yield else: store = StoringReporter(item.config) def note_statistics(stats): gathered_statistics[item.nodeid] = stats statistics = {"passing_examples": stats.passing_examples, "failing_examples": stats.failing_examples, "invalid_examples": stats.invalid_examples, "runtimes": stats.runtimes} update_error_store("statistics", statistics) with collector.with_value(note_statistics): with with_reporter(store): clean_error_store() yield # (name, statistics) = gathered_statistics.items()[-1] # print(gathered_statistics.items()) # print(name, statistics) # update_error_store['test_name'] = name # update_error_store['statistics'] = statistics # print(name) # print(statistics.passing_examples, statistics.failing_examples, # statistics.invalid_examples) # update_error_store() if server_option: write_error_store_to_file(output_file_name) # print(gathered_statistics) if store.results: item.hypothesis_report_information = list(store.results)
def pytest_runtest_call(item): if not hasattr(item, "obj"): yield elif not is_hypothesis_test(item.obj): # If @given was not applied, check whether other hypothesis # decorators were applied, and raise an error if they were. if getattr(item.obj, "_hypothesis_internal_settings_applied", False): raise InvalidArgument( "Using `@settings` on a test without `@given` is completely pointless." ) yield else: if item.get_closest_marker("parametrize") is not None: # Give every parametrized test invocation a unique database key key = item.nodeid.encode("utf-8") item.obj.hypothesis.inner_test._hypothesis_internal_add_digest = key store = StoringReporter(item.config) def note_statistics(stats): lines = [item.nodeid + ":", "" ] + stats.get_description() + [""] item.hypothesis_statistics = lines with collector.with_value(note_statistics): with with_reporter(store): yield if store.results: item.hypothesis_report_information = list(store.results)
def pytest_runtest_call(item): # This hookwrapper checks for PRNG state leaks from Hypothesis tests. # See: https://github.com/HypothesisWorks/hypothesis/issues/1919 if not (hasattr(item, "obj") and is_hypothesis_test(item.obj)): yield elif "pytest_randomly" in sys.modules: # See https://github.com/HypothesisWorks/hypothesis/issues/3041 - this # branch exists to make it easier on external contributors, but should # never run in our CI (because that would disable the check entirely). assert not is_in_ci() yield else: # We start by peturbing the state of the PRNG, because repeatedly # leaking PRNG state resets state_after to the (previously leaked) # state_before, and that just shows as "no use of random". random.seed(independent_random.randrange(2**32)) before = random.getstate() yield after = random.getstate() if before != after: if after in random_states_after_tests: raise Exception( f"{item.nodeid!r} and {random_states_after_tests[after]!r} " "both used the `random` module, and finished with the " "same global `random.getstate()`; this is probably a nasty bug!" ) random_states_after_tests[after] = item.nodeid
def test_detection_of_stateful_tests(): class Stuff(GenericStateMachine): def steps(self): return integers() def execute_step(self, step): pass assert is_hypothesis_test(Stuff.TestCase().runTest)
def pytest_collection_modifyitems(items): if "hypothesis" not in sys.modules: return from hypothesis.internal.detection import is_hypothesis_test for item in items: if isinstance(item, pytest.Function) and is_hypothesis_test(item.obj): item.add_marker("hypothesis")
def _ban_given_call(self, function): if "hypothesis" in sys.modules: from hypothesis.internal.detection import is_hypothesis_test if is_hypothesis_test(function): raise RuntimeError( f"Can't apply @pytest.fixture() to {function.__name__} because " "it is already decorated with @hypothesis.given()" ) return _orig_call(self, function)
def pytest_runtest_call(item): if not (hasattr(item, 'obj') and is_hypothesis_test(item.obj)): yield else: store = StoringReporter(item.config) def note_statistics(stats): gathered_statistics[item.nodeid] = stats with collector.with_value(note_statistics): with with_reporter(store): yield if store.results: item.hypothesis_report_information = list(store.results)
def pytest_runtest_call(item): if not (hasattr(item, "obj") and is_hypothesis_test(item.obj)): yield else: store = StoringReporter(item.config) def note_statistics(stats): lines = [item.nodeid + ":", ""] + stats.get_description() + [""] gathered_statistics[item.nodeid] = lines item.hypothesis_statistics = lines with collector.with_value(note_statistics): with with_reporter(store): yield if store.results: item.hypothesis_report_information = list(store.results)
def pytest_runtest_call(item): if not hasattr(item, "obj"): yield elif not is_hypothesis_test(item.obj): # If @given was not applied, check whether other hypothesis # decorators were applied, and raise an error if they were. message = "Using `@%s` on a test without `@given` is completely pointless." if getattr(item.obj, "_hypothesis_internal_settings_applied", False): raise InvalidArgument(message % ("settings", )) if getattr(item.obj, "is_hypothesis_strategy_function", False): note_deprecation( "%s is a function that returns a Hypothesis strategy, but pytest " "has collected it as a test function. This is useless as the " "function body will never be executed. To define a test " "function, use @given instead of @composite." % (item.nodeid, ), since="2018-11-02", ) for name, attribute in [ ("example", "hypothesis_explicit_examples"), ("seed", "_hypothesis_internal_use_seed"), ("reproduce_example", "_hypothesis_internal_use_reproduce_failure"), ]: if hasattr(item.obj, attribute): note_deprecation(message % (name, ), since="2019-12-07") yield else: if item.get_closest_marker("parametrize") is not None: # Give every parametrized test invocation a unique database key key = item.nodeid.encode("utf-8") item.obj.hypothesis.inner_test._hypothesis_internal_add_digest = key store = StoringReporter(item.config) def note_statistics(stats): lines = [item.nodeid + ":", "" ] + stats.get_description() + [""] item.hypothesis_statistics = lines with collector.with_value(note_statistics): with with_reporter(store): yield if store.results: item.hypothesis_report_information = list(store.results)
def pytest_collection_modifyitems(items): for item in items: if not isinstance(item, pytest.Function): continue if is_hypothesis_test(item.obj): item.add_marker("hypothesis") if getattr(item.obj, "is_hypothesis_strategy_function", False): def note_strategy_is_not_test(*args, **kwargs): note_deprecation( "%s is a function that returns a Hypothesis strategy, " "but pytest has collected it as a test function. This " "is useless as the function body will never be executed. " "To define a test function, use @given instead of " "@composite." % (item.nodeid, ), since="2018-11-02", ) item.obj = note_strategy_is_not_test
def pytest_collection_modifyitems(items): for item in items: if not isinstance(item, pytest.Function): continue if is_hypothesis_test(item.obj): item.add_marker("hypothesis") if getattr(item.obj, "is_hypothesis_strategy_function", False): def note_strategy_is_not_test(*args, **kwargs): note_deprecation( "%s is a function that returns a Hypothesis strategy, " "but pytest has collected it as a test function. This " "is useless as the function body will never be executed. " "To define a test function, use @given instead of " "@composite." % (item.nodeid,), since="2018-11-02", ) item.obj = note_strategy_is_not_test
def pytest_runtest_call(item): # This hookwrapper checks for PRNG state leaks from Hypothesis tests. # See: https://github.com/HypothesisWorks/hypothesis/issues/1919 if not (hasattr(item, "obj") and is_hypothesis_test(item.obj)): yield else: # We start by peturbing the state of the PRNG, because repeatedly # leaking PRNG state resets state_after to the (previously leaked) # state_before, and that just shows as "no use of random". random.seed(independent_random.randrange(2**32)) before = random.getstate() yield after = random.getstate() if before != after: if after in random_states_after_tests: raise Exception( "%r and %r both used the `random` module, and finished with the " "same global `random.getstate()`; this is probably a nasty bug!" % (item.nodeid, random_states_after_tests[after])) random_states_after_tests[after] = item.nodeid
def pytest_runtest_call(item): if not (hasattr(item, "obj") and is_hypothesis_test(item.obj)): yield else: if item.get_closest_marker("parametrize") is not None: # Give every parametrized test invocation a unique database key item.obj.hypothesis.inner_test._hypothesis_internal_add_digest = item.nodeid.encode( "utf-8" ) store = StoringReporter(item.config) def note_statistics(stats): lines = [item.nodeid + ":", ""] + stats.get_description() + [""] gathered_statistics[item.nodeid] = lines item.hypothesis_statistics = lines with collector.with_value(note_statistics): with with_reporter(store): yield if store.results: item.hypothesis_report_information = list(store.results)
def run(self, result: Optional[unittest.TestResult] = None) -> Any: """ Some slightly awful magic here to arrange for setUp and tearDown to be called at the appropriate times when hypothesis is enabled for a test case. This can be removed once a future version of hypothesis ships with support for this baked in. """ if is_hypothesis_test(getattr(self, self._testMethodName)): try: old_setUp = self.setUp old_tearDown = self.tearDown self.setUp = lambda: None # type: ignore # (mypy issue 2427) self.tearDown = lambda: None # type: ignore # (mypy issue 2427) self.setup_example = old_setUp self.teardown_example = lambda _: old_tearDown() return super(EdenTestCase, self).run(result) finally: self.setUp = old_setUp # type: ignore # (mypy issue 2427) self.tearDown = old_tearDown # type: ignore # (mypy issue 2427) # pyre-fixme[16]: `EdenTestCase` has no attribute `setup_example`. del self.setup_example # pyre-fixme[16]: `EdenTestCase` has no attribute `teardown_example`. del self.teardown_example else: return super(EdenTestCase, self).run(result)
def pytest_runtest_call(item): if not (hasattr(item, "obj") and "hypothesis" in sys.modules): yield return from hypothesis import core from hypothesis.internal.detection import is_hypothesis_test core.running_under_pytest = True if not is_hypothesis_test(item.obj): # If @given was not applied, check whether other hypothesis # decorators were applied, and raise an error if they were. if getattr(item.obj, "is_hypothesis_strategy_function", False): from hypothesis.errors import InvalidArgument raise InvalidArgument( f"{item.nodeid} is a function that returns a Hypothesis strategy, " "but pytest has collected it as a test function. This is useless " "as the function body will never be executed. To define a test " "function, use @given instead of @composite.") message = "Using `@%s` on a test without `@given` is completely pointless." for name, attribute in [ ("example", "hypothesis_explicit_examples"), ("seed", "_hypothesis_internal_use_seed"), ("settings", "_hypothesis_internal_settings_applied"), ("reproduce_example", "_hypothesis_internal_use_reproduce_failure"), ]: if hasattr(item.obj, attribute): from hypothesis.errors import InvalidArgument raise InvalidArgument(message % (name, )) yield else: from hypothesis import HealthCheck, settings from hypothesis.internal.escalation import current_pytest_item from hypothesis.internal.healthcheck import fail_health_check from hypothesis.reporting import with_reporter from hypothesis.statistics import collector, describe_statistics # Retrieve the settings for this test from the test object, which # is normally a Hypothesis wrapped_test wrapper. If this doesn't # work, the test object is probably something weird # (e.g a stateful test wrapper), so we skip the function-scoped # fixture check. settings = getattr(item.obj, "_hypothesis_internal_use_settings", None) # Check for suspicious use of function-scoped fixtures, but only # if the corresponding health check is not suppressed. if (settings is not None and HealthCheck.function_scoped_fixture not in settings.suppress_health_check): # Warn about function-scoped fixtures, excluding autouse fixtures because # the advice is probably not actionable and the status quo seems OK... # See https://github.com/HypothesisWorks/hypothesis/issues/377 for detail. argnames = None for fx_defs in item._request._fixturemanager.getfixtureinfo( node=item, func=item.function, cls=None).name2fixturedefs.values(): if argnames is None: argnames = frozenset( signature(item.function).parameters) for fx in fx_defs: if fx.argname in argnames: active_fx = item._request._get_active_fixturedef( fx.argname) if active_fx.scope == "function": fail_health_check( settings, _FIXTURE_MSG.format( fx.argname, item.nodeid), HealthCheck.function_scoped_fixture, ) if item.get_closest_marker("parametrize") is not None: # Give every parametrized test invocation a unique database key key = item.nodeid.encode() item.obj.hypothesis.inner_test._hypothesis_internal_add_digest = key store = StoringReporter(item.config) def note_statistics(stats): stats["nodeid"] = item.nodeid item.hypothesis_statistics = base64.b64encode( describe_statistics(stats).encode()).decode() with collector.with_value(note_statistics): with with_reporter(store): with current_pytest_item.with_value(item): yield if store.results: item.hypothesis_report_information = list(store.results)
def pytest_runtest_call(item): if not hasattr(item, "obj"): yield elif not is_hypothesis_test(item.obj): # If @given was not applied, check whether other hypothesis # decorators were applied, and raise an error if they were. if getattr(item.obj, "is_hypothesis_strategy_function", False): raise InvalidArgument( "%s is a function that returns a Hypothesis strategy, but pytest " "has collected it as a test function. This is useless as the " "function body will never be executed. To define a test " "function, use @given instead of @composite." % (item.nodeid, )) message = "Using `@%s` on a test without `@given` is completely pointless." for name, attribute in [ ("example", "hypothesis_explicit_examples"), ("seed", "_hypothesis_internal_use_seed"), ("settings", "_hypothesis_internal_settings_applied"), ("reproduce_example", "_hypothesis_internal_use_reproduce_failure"), ]: if hasattr(item.obj, attribute): raise InvalidArgument(message % (name, )) yield else: # Warn about function-scoped fixtures, excluding autouse fixtures because # the advice is probably not actionable and the status quo seems OK... # See https://github.com/HypothesisWorks/hypothesis/issues/377 for detail. argnames = None for fx_defs in item._request._fixturemanager.getfixtureinfo( node=item, func=item.function, cls=None).name2fixturedefs.values(): if argnames is None: argnames = frozenset(signature(item.function).parameters) for fx in fx_defs: if fx.scope == "function" and fx.argname in argnames: note_deprecation( "%s uses the %r fixture, but function-scoped fixtures " "should not be used with @given(...) tests, because " "fixtures are not reset between generated examples!" % (item.nodeid, fx.argname), since="2020-02-29", ) if item.get_closest_marker("parametrize") is not None: # Give every parametrized test invocation a unique database key key = item.nodeid.encode("utf-8") item.obj.hypothesis.inner_test._hypothesis_internal_add_digest = key store = StoringReporter(item.config) def note_statistics(stats): lines = [item.nodeid + ":", "" ] + stats.get_description() + [""] item.hypothesis_statistics = lines with collector.with_value(note_statistics): with with_reporter(store): yield if store.results: item.hypothesis_report_information = list(store.results)
def test_functions_default_to_not_tests(): def foo(): pass assert not is_hypothesis_test(foo)
def test_detection_of_functions(): @given(integers()) def test(i): pass assert is_hypothesis_test(test)
def test_methods_default_to_not_tests(): class Foo: def foo(): pass assert not is_hypothesis_test(Foo().foo)
def test_methods_default_to_not_tests(): class Foo(object): def foo(): pass assert not is_hypothesis_test(Foo().foo)
def pytest_runtest_call(item): if not hasattr(item, "obj"): yield elif not is_hypothesis_test(item.obj): # If @given was not applied, check whether other hypothesis # decorators were applied, and raise an error if they were. if getattr(item.obj, "is_hypothesis_strategy_function", False): raise InvalidArgument( "%s is a function that returns a Hypothesis strategy, but pytest " "has collected it as a test function. This is useless as the " "function body will never be executed. To define a test " "function, use @given instead of @composite." % (item.nodeid, )) message = "Using `@%s` on a test without `@given` is completely pointless." for name, attribute in [ ("example", "hypothesis_explicit_examples"), ("seed", "_hypothesis_internal_use_seed"), ("settings", "_hypothesis_internal_settings_applied"), ("reproduce_example", "_hypothesis_internal_use_reproduce_failure"), ]: if hasattr(item.obj, attribute): raise InvalidArgument(message % (name, )) yield else: # Retrieve the settings for this test from the test object, which # is normally a Hypothesis wrapped_test wrapper. If this doesn't # work, the test object is probably something weird # (e.g a stateful test wrapper), so we skip the function-scoped # fixture check. settings = getattr(item.obj, "_hypothesis_internal_use_settings", None) # Check for suspicious use of function-scoped fixtures, but only # if the corresponding health check is not suppressed. if (settings is not None and HealthCheck.function_scoped_fixture not in settings.suppress_health_check): # Warn about function-scoped fixtures, excluding autouse fixtures because # the advice is probably not actionable and the status quo seems OK... # See https://github.com/HypothesisWorks/hypothesis/issues/377 for detail. msg = ( "%s uses the %r fixture, which is reset between function calls but not " "between test cases generated by `@given(...)`. You can change it to " "a module- or session-scoped fixture if it is safe to reuse; if not " "we recommend using a context manager inside your test function. See " "https://docs.pytest.org/en/latest/fixture.html#sharing-test-data " "for details on fixture scope.") argnames = None for fx_defs in item._request._fixturemanager.getfixtureinfo( node=item, func=item.function, cls=None).name2fixturedefs.values(): if argnames is None: argnames = frozenset( signature(item.function).parameters) for fx in fx_defs: if fx.argname in argnames: active_fx = item._request._get_active_fixturedef( fx.argname) if active_fx.scope == "function": fail_health_check( settings, msg % (item.nodeid, fx.argname), HealthCheck.function_scoped_fixture, ) if item.get_closest_marker("parametrize") is not None: # Give every parametrized test invocation a unique database key key = item.nodeid.encode("utf-8") item.obj.hypothesis.inner_test._hypothesis_internal_add_digest = key store = StoringReporter(item.config) def note_statistics(stats): stats["nodeid"] = item.nodeid item.hypothesis_statistics = base64.b64encode( describe_statistics(stats).encode()).decode() with collector.with_value(note_statistics): with with_reporter(store): yield if store.results: item.hypothesis_report_information = list(store.results)
def pytest_collection_modifyitems(items): for item in items: if isinstance(item, pytest.Function) and is_hypothesis_test( item.obj): item.add_marker("hypothesis")