def test_detection_of_stateful_tests():
class Stuff(RuleBasedStateMachine):
@rule(x=integers())
def a_rule(self, x):
pass
assert is_hypothesis_test(Stuff.TestCase().runTest)
def test_detection_of_methods():
class Foo(object):
@given(integers())
def test(self, i):
pass
assert is_hypothesis_test(Foo().test)
def test_detection_of_methods():
class Foo:
@given(integers())
def test(self, i):
pass
assert is_hypothesis_test(Foo().test)
def run(
self,
report: Optional[unittest.result.TestResult] = None
) -> unittest.result.TestResult:
""" Some slightly awful magic here to arrange for setUp and
tearDown to be called at the appropriate times when hypothesis
is enabled for a test case.
This can be removed once a future version of hypothesis
ships with support for this baked in. """
if is_hypothesis_test(getattr(self, self._testMethodName)):
try:
old_setUp = self.setUp
old_tearDown = self.tearDown
self.setUp = lambda: None
self.tearDown = lambda: None
self.setup_example = old_setUp
self.teardown_example = lambda _: old_tearDown()
return super(EdenTestCase, self).run(report)
finally:
self.setUp = old_setUp
self.tearDown = old_tearDown
del self.setup_example
del self.teardown_example
else:
return super(EdenTestCase, self).run(report)
def pytest_runtest_call(item):
if not (hasattr(item, 'obj') and is_hypothesis_test(item.obj)):
yield
else:
store = StoringReporter(item.config)
def note_statistics(stats):
gathered_statistics[item.nodeid] = stats
statistics = {"passing_examples": stats.passing_examples,
"failing_examples": stats.failing_examples,
"invalid_examples": stats.invalid_examples,
"runtimes": stats.runtimes}
update_error_store("statistics", statistics)
with collector.with_value(note_statistics):
with with_reporter(store):
clean_error_store()
yield
# (name, statistics) = gathered_statistics.items()[-1]
# print(gathered_statistics.items())
# print(name, statistics)
# update_error_store['test_name'] = name
# update_error_store['statistics'] = statistics
# print(name)
# print(statistics.passing_examples, statistics.failing_examples,
# statistics.invalid_examples)
# update_error_store()
if server_option:
write_error_store_to_file(output_file_name)
# print(gathered_statistics)
if store.results:
item.hypothesis_report_information = list(store.results)
def pytest_runtest_call(item):
if not hasattr(item, "obj"):
yield
elif not is_hypothesis_test(item.obj):
# If @given was not applied, check whether other hypothesis
# decorators were applied, and raise an error if they were.
if getattr(item.obj, "_hypothesis_internal_settings_applied",
False):
raise InvalidArgument(
"Using `@settings` on a test without `@given` is completely pointless."
)
yield
else:
if item.get_closest_marker("parametrize") is not None:
# Give every parametrized test invocation a unique database key
key = item.nodeid.encode("utf-8")
item.obj.hypothesis.inner_test._hypothesis_internal_add_digest = key
store = StoringReporter(item.config)
def note_statistics(stats):
lines = [item.nodeid + ":", ""
] + stats.get_description() + [""]
item.hypothesis_statistics = lines
with collector.with_value(note_statistics):
with with_reporter(store):
yield
if store.results:
item.hypothesis_report_information = list(store.results)
def pytest_runtest_call(item):
# This hookwrapper checks for PRNG state leaks from Hypothesis tests.
# See: https://github.com/HypothesisWorks/hypothesis/issues/1919
if not (hasattr(item, "obj") and is_hypothesis_test(item.obj)):
yield
elif "pytest_randomly" in sys.modules:
# See https://github.com/HypothesisWorks/hypothesis/issues/3041 - this
# branch exists to make it easier on external contributors, but should
# never run in our CI (because that would disable the check entirely).
assert not is_in_ci()
yield
else:
# We start by peturbing the state of the PRNG, because repeatedly
# leaking PRNG state resets state_after to the (previously leaked)
# state_before, and that just shows as "no use of random".
random.seed(independent_random.randrange(2**32))
before = random.getstate()
yield
after = random.getstate()
if before != after:
if after in random_states_after_tests:
raise Exception(
f"{item.nodeid!r} and {random_states_after_tests[after]!r} "
"both used the `random` module, and finished with the "
"same global `random.getstate()`; this is probably a nasty bug!"
)
random_states_after_tests[after] = item.nodeid
def test_detection_of_stateful_tests():
class Stuff(GenericStateMachine):
def steps(self):
return integers()
def execute_step(self, step):
pass
assert is_hypothesis_test(Stuff.TestCase().runTest)
def pytest_collection_modifyitems(items):
if "hypothesis" not in sys.modules:
return
from hypothesis.internal.detection import is_hypothesis_test
for item in items:
if isinstance(item, pytest.Function) and is_hypothesis_test(item.obj):
item.add_marker("hypothesis")
def _ban_given_call(self, function):
if "hypothesis" in sys.modules:
from hypothesis.internal.detection import is_hypothesis_test
if is_hypothesis_test(function):
raise RuntimeError(
f"Can't apply @pytest.fixture() to {function.__name__} because "
"it is already decorated with @hypothesis.given()"
)
return _orig_call(self, function)
def pytest_runtest_call(item):
if not (hasattr(item, 'obj') and is_hypothesis_test(item.obj)):
yield
else:
store = StoringReporter(item.config)
def note_statistics(stats):
gathered_statistics[item.nodeid] = stats
with collector.with_value(note_statistics):
with with_reporter(store):
yield
if store.results:
item.hypothesis_report_information = list(store.results)
def pytest_runtest_call(item):
if not (hasattr(item, "obj") and is_hypothesis_test(item.obj)):
yield
else:
store = StoringReporter(item.config)
def note_statistics(stats):
lines = [item.nodeid + ":", ""] + stats.get_description() + [""]
gathered_statistics[item.nodeid] = lines
item.hypothesis_statistics = lines
with collector.with_value(note_statistics):
with with_reporter(store):
yield
if store.results:
item.hypothesis_report_information = list(store.results)
def pytest_runtest_call(item):
if not (hasattr(item, "obj") and is_hypothesis_test(item.obj)):
yield
else:
store = StoringReporter(item.config)
def note_statistics(stats):
lines = [item.nodeid + ":", ""] + stats.get_description() + [""]
gathered_statistics[item.nodeid] = lines
item.hypothesis_statistics = lines
with collector.with_value(note_statistics):
with with_reporter(store):
yield
if store.results:
item.hypothesis_report_information = list(store.results)
def pytest_runtest_call(item):
if not hasattr(item, "obj"):
yield
elif not is_hypothesis_test(item.obj):
# If @given was not applied, check whether other hypothesis
# decorators were applied, and raise an error if they were.
message = "Using `@%s` on a test without `@given` is completely pointless."
if getattr(item.obj, "_hypothesis_internal_settings_applied",
False):
raise InvalidArgument(message % ("settings", ))
if getattr(item.obj, "is_hypothesis_strategy_function", False):
note_deprecation(
"%s is a function that returns a Hypothesis strategy, but pytest "
"has collected it as a test function. This is useless as the "
"function body will never be executed. To define a test "
"function, use @given instead of @composite." %
(item.nodeid, ),
since="2018-11-02",
)
for name, attribute in [
("example", "hypothesis_explicit_examples"),
("seed", "_hypothesis_internal_use_seed"),
("reproduce_example",
"_hypothesis_internal_use_reproduce_failure"),
]:
if hasattr(item.obj, attribute):
note_deprecation(message % (name, ), since="2019-12-07")
yield
else:
if item.get_closest_marker("parametrize") is not None:
# Give every parametrized test invocation a unique database key
key = item.nodeid.encode("utf-8")
item.obj.hypothesis.inner_test._hypothesis_internal_add_digest = key
store = StoringReporter(item.config)
def note_statistics(stats):
lines = [item.nodeid + ":", ""
] + stats.get_description() + [""]
item.hypothesis_statistics = lines
with collector.with_value(note_statistics):
with with_reporter(store):
yield
if store.results:
item.hypothesis_report_information = list(store.results)
def pytest_collection_modifyitems(items):
for item in items:
if not isinstance(item, pytest.Function):
continue
if is_hypothesis_test(item.obj):
item.add_marker("hypothesis")
if getattr(item.obj, "is_hypothesis_strategy_function", False):
def note_strategy_is_not_test(*args, **kwargs):
note_deprecation(
"%s is a function that returns a Hypothesis strategy, "
"but pytest has collected it as a test function. This "
"is useless as the function body will never be executed. "
"To define a test function, use @given instead of "
"@composite." % (item.nodeid, ),
since="2018-11-02",
)
item.obj = note_strategy_is_not_test
def pytest_collection_modifyitems(items):
for item in items:
if not isinstance(item, pytest.Function):
continue
if is_hypothesis_test(item.obj):
item.add_marker("hypothesis")
if getattr(item.obj, "is_hypothesis_strategy_function", False):
def note_strategy_is_not_test(*args, **kwargs):
note_deprecation(
"%s is a function that returns a Hypothesis strategy, "
"but pytest has collected it as a test function. This "
"is useless as the function body will never be executed. "
"To define a test function, use @given instead of "
"@composite." % (item.nodeid,),
since="2018-11-02",
)
item.obj = note_strategy_is_not_test
def pytest_runtest_call(item):
# This hookwrapper checks for PRNG state leaks from Hypothesis tests.
# See: https://github.com/HypothesisWorks/hypothesis/issues/1919
if not (hasattr(item, "obj") and is_hypothesis_test(item.obj)):
yield
else:
# We start by peturbing the state of the PRNG, because repeatedly
# leaking PRNG state resets state_after to the (previously leaked)
# state_before, and that just shows as "no use of random".
random.seed(independent_random.randrange(2**32))
before = random.getstate()
yield
after = random.getstate()
if before != after:
if after in random_states_after_tests:
raise Exception(
"%r and %r both used the `random` module, and finished with the "
"same global `random.getstate()`; this is probably a nasty bug!"
% (item.nodeid, random_states_after_tests[after]))
random_states_after_tests[after] = item.nodeid
def pytest_runtest_call(item):
if not (hasattr(item, "obj") and is_hypothesis_test(item.obj)):
yield
else:
if item.get_closest_marker("parametrize") is not None:
# Give every parametrized test invocation a unique database key
item.obj.hypothesis.inner_test._hypothesis_internal_add_digest = item.nodeid.encode(
"utf-8"
)
store = StoringReporter(item.config)
def note_statistics(stats):
lines = [item.nodeid + ":", ""] + stats.get_description() + [""]
gathered_statistics[item.nodeid] = lines
item.hypothesis_statistics = lines
with collector.with_value(note_statistics):
with with_reporter(store):
yield
if store.results:
item.hypothesis_report_information = list(store.results)
def run(self, result: Optional[unittest.TestResult] = None) -> Any:
""" Some slightly awful magic here to arrange for setUp and
tearDown to be called at the appropriate times when hypothesis
is enabled for a test case.
This can be removed once a future version of hypothesis
ships with support for this baked in. """
if is_hypothesis_test(getattr(self, self._testMethodName)):
try:
old_setUp = self.setUp
old_tearDown = self.tearDown
self.setUp = lambda: None # type: ignore # (mypy issue 2427)
self.tearDown = lambda: None # type: ignore # (mypy issue 2427)
self.setup_example = old_setUp
self.teardown_example = lambda _: old_tearDown()
return super(EdenTestCase, self).run(result)
finally:
self.setUp = old_setUp # type: ignore # (mypy issue 2427)
self.tearDown = old_tearDown # type: ignore # (mypy issue 2427)
# pyre-fixme[16]: `EdenTestCase` has no attribute `setup_example`.
del self.setup_example
# pyre-fixme[16]: `EdenTestCase` has no attribute `teardown_example`.
del self.teardown_example
else:
return super(EdenTestCase, self).run(result)
def pytest_runtest_call(item):
if not (hasattr(item, "obj") and "hypothesis" in sys.modules):
yield
return
from hypothesis import core
from hypothesis.internal.detection import is_hypothesis_test
core.running_under_pytest = True
if not is_hypothesis_test(item.obj):
# If @given was not applied, check whether other hypothesis
# decorators were applied, and raise an error if they were.
if getattr(item.obj, "is_hypothesis_strategy_function", False):
from hypothesis.errors import InvalidArgument
raise InvalidArgument(
f"{item.nodeid} is a function that returns a Hypothesis strategy, "
"but pytest has collected it as a test function. This is useless "
"as the function body will never be executed. To define a test "
"function, use @given instead of @composite.")
message = "Using `@%s` on a test without `@given` is completely pointless."
for name, attribute in [
("example", "hypothesis_explicit_examples"),
("seed", "_hypothesis_internal_use_seed"),
("settings", "_hypothesis_internal_settings_applied"),
("reproduce_example",
"_hypothesis_internal_use_reproduce_failure"),
]:
if hasattr(item.obj, attribute):
from hypothesis.errors import InvalidArgument
raise InvalidArgument(message % (name, ))
yield
else:
from hypothesis import HealthCheck, settings
from hypothesis.internal.escalation import current_pytest_item
from hypothesis.internal.healthcheck import fail_health_check
from hypothesis.reporting import with_reporter
from hypothesis.statistics import collector, describe_statistics
# Retrieve the settings for this test from the test object, which
# is normally a Hypothesis wrapped_test wrapper. If this doesn't
# work, the test object is probably something weird
# (e.g a stateful test wrapper), so we skip the function-scoped
# fixture check.
settings = getattr(item.obj, "_hypothesis_internal_use_settings",
None)
# Check for suspicious use of function-scoped fixtures, but only
# if the corresponding health check is not suppressed.
if (settings is not None and HealthCheck.function_scoped_fixture
not in settings.suppress_health_check):
# Warn about function-scoped fixtures, excluding autouse fixtures because
# the advice is probably not actionable and the status quo seems OK...
# See https://github.com/HypothesisWorks/hypothesis/issues/377 for detail.
argnames = None
for fx_defs in item._request._fixturemanager.getfixtureinfo(
node=item, func=item.function,
cls=None).name2fixturedefs.values():
if argnames is None:
argnames = frozenset(
signature(item.function).parameters)
for fx in fx_defs:
if fx.argname in argnames:
active_fx = item._request._get_active_fixturedef(
fx.argname)
if active_fx.scope == "function":
fail_health_check(
settings,
_FIXTURE_MSG.format(
fx.argname, item.nodeid),
HealthCheck.function_scoped_fixture,
)
if item.get_closest_marker("parametrize") is not None:
# Give every parametrized test invocation a unique database key
key = item.nodeid.encode()
item.obj.hypothesis.inner_test._hypothesis_internal_add_digest = key
store = StoringReporter(item.config)
def note_statistics(stats):
stats["nodeid"] = item.nodeid
item.hypothesis_statistics = base64.b64encode(
describe_statistics(stats).encode()).decode()
with collector.with_value(note_statistics):
with with_reporter(store):
with current_pytest_item.with_value(item):
yield
if store.results:
item.hypothesis_report_information = list(store.results)
def pytest_runtest_call(item):
if not hasattr(item, "obj"):
yield
elif not is_hypothesis_test(item.obj):
# If @given was not applied, check whether other hypothesis
# decorators were applied, and raise an error if they were.
if getattr(item.obj, "is_hypothesis_strategy_function", False):
raise InvalidArgument(
"%s is a function that returns a Hypothesis strategy, but pytest "
"has collected it as a test function. This is useless as the "
"function body will never be executed. To define a test "
"function, use @given instead of @composite." %
(item.nodeid, ))
message = "Using `@%s` on a test without `@given` is completely pointless."
for name, attribute in [
("example", "hypothesis_explicit_examples"),
("seed", "_hypothesis_internal_use_seed"),
("settings", "_hypothesis_internal_settings_applied"),
("reproduce_example",
"_hypothesis_internal_use_reproduce_failure"),
]:
if hasattr(item.obj, attribute):
raise InvalidArgument(message % (name, ))
yield
else:
# Warn about function-scoped fixtures, excluding autouse fixtures because
# the advice is probably not actionable and the status quo seems OK...
# See https://github.com/HypothesisWorks/hypothesis/issues/377 for detail.
argnames = None
for fx_defs in item._request._fixturemanager.getfixtureinfo(
node=item, func=item.function,
cls=None).name2fixturedefs.values():
if argnames is None:
argnames = frozenset(signature(item.function).parameters)
for fx in fx_defs:
if fx.scope == "function" and fx.argname in argnames:
note_deprecation(
"%s uses the %r fixture, but function-scoped fixtures "
"should not be used with @given(...) tests, because "
"fixtures are not reset between generated examples!"
% (item.nodeid, fx.argname),
since="2020-02-29",
)
if item.get_closest_marker("parametrize") is not None:
# Give every parametrized test invocation a unique database key
key = item.nodeid.encode("utf-8")
item.obj.hypothesis.inner_test._hypothesis_internal_add_digest = key
store = StoringReporter(item.config)
def note_statistics(stats):
lines = [item.nodeid + ":", ""
] + stats.get_description() + [""]
item.hypothesis_statistics = lines
with collector.with_value(note_statistics):
with with_reporter(store):
yield
if store.results:
item.hypothesis_report_information = list(store.results)
def test_functions_default_to_not_tests():
def foo():
pass
assert not is_hypothesis_test(foo)
def test_detection_of_functions():
@given(integers())
def test(i):
pass
assert is_hypothesis_test(test)
def test_methods_default_to_not_tests():
class Foo:
def foo():
pass
assert not is_hypothesis_test(Foo().foo)
def test_functions_default_to_not_tests():
def foo():
pass
assert not is_hypothesis_test(foo)
def test_methods_default_to_not_tests():
class Foo(object):
def foo():
pass
assert not is_hypothesis_test(Foo().foo)
def test_detection_of_functions():
@given(integers())
def test(i):
pass
assert is_hypothesis_test(test)
def pytest_runtest_call(item):
if not hasattr(item, "obj"):
yield
elif not is_hypothesis_test(item.obj):
# If @given was not applied, check whether other hypothesis
# decorators were applied, and raise an error if they were.
if getattr(item.obj, "is_hypothesis_strategy_function", False):
raise InvalidArgument(
"%s is a function that returns a Hypothesis strategy, but pytest "
"has collected it as a test function. This is useless as the "
"function body will never be executed. To define a test "
"function, use @given instead of @composite." %
(item.nodeid, ))
message = "Using `@%s` on a test without `@given` is completely pointless."
for name, attribute in [
("example", "hypothesis_explicit_examples"),
("seed", "_hypothesis_internal_use_seed"),
("settings", "_hypothesis_internal_settings_applied"),
("reproduce_example",
"_hypothesis_internal_use_reproduce_failure"),
]:
if hasattr(item.obj, attribute):
raise InvalidArgument(message % (name, ))
yield
else:
# Retrieve the settings for this test from the test object, which
# is normally a Hypothesis wrapped_test wrapper. If this doesn't
# work, the test object is probably something weird
# (e.g a stateful test wrapper), so we skip the function-scoped
# fixture check.
settings = getattr(item.obj, "_hypothesis_internal_use_settings",
None)
# Check for suspicious use of function-scoped fixtures, but only
# if the corresponding health check is not suppressed.
if (settings is not None and HealthCheck.function_scoped_fixture
not in settings.suppress_health_check):
# Warn about function-scoped fixtures, excluding autouse fixtures because
# the advice is probably not actionable and the status quo seems OK...
# See https://github.com/HypothesisWorks/hypothesis/issues/377 for detail.
msg = (
"%s uses the %r fixture, which is reset between function calls but not "
"between test cases generated by `@given(...)`. You can change it to "
"a module- or session-scoped fixture if it is safe to reuse; if not "
"we recommend using a context manager inside your test function. See "
"https://docs.pytest.org/en/latest/fixture.html#sharing-test-data "
"for details on fixture scope.")
argnames = None
for fx_defs in item._request._fixturemanager.getfixtureinfo(
node=item, func=item.function,
cls=None).name2fixturedefs.values():
if argnames is None:
argnames = frozenset(
signature(item.function).parameters)
for fx in fx_defs:
if fx.argname in argnames:
active_fx = item._request._get_active_fixturedef(
fx.argname)
if active_fx.scope == "function":
fail_health_check(
settings,
msg % (item.nodeid, fx.argname),
HealthCheck.function_scoped_fixture,
)
if item.get_closest_marker("parametrize") is not None:
# Give every parametrized test invocation a unique database key
key = item.nodeid.encode("utf-8")
item.obj.hypothesis.inner_test._hypothesis_internal_add_digest = key
store = StoringReporter(item.config)
def note_statistics(stats):
stats["nodeid"] = item.nodeid
item.hypothesis_statistics = base64.b64encode(
describe_statistics(stats).encode()).decode()
with collector.with_value(note_statistics):
with with_reporter(store):
yield
if store.results:
item.hypothesis_report_information = list(store.results)
def pytest_collection_modifyitems(items):
for item in items:
if isinstance(item, pytest.Function) and is_hypothesis_test(
item.obj):
item.add_marker("hypothesis")