def evaluate_test_data(self, data):
try:
result = self.execute(data)
if result is not None:
fail_health_check(
self.settings,
('Tests run under @given should return None, but '
'%s returned %r instead.') % (self.test.__name__, result),
HealthCheck.return_value)
except UnsatisfiedAssumption:
data.mark_invalid()
except (
HypothesisDeprecationWarning,
FailedHealthCheck,
StopTest,
) + EXCEPTIONS_TO_RERAISE:
raise
except EXCEPTIONS_TO_FAIL as e:
escalate_hypothesis_internal_error()
tb = get_trimmed_traceback()
data.__expected_traceback = ''.join(
traceback.format_exception(type(e), e, tb))
data.__expected_exception = e
verbose_report(data.__expected_traceback)
origin = traceback.extract_tb(tb)[-1]
filename = origin[0]
lineno = origin[1]
data.mark_interesting((type(e), filename, lineno))
def evaluate_test_data(self, data):
if (time.time() - self.start_time >= HUNG_TEST_TIME_LIMIT):
fail_health_check(
self.settings,
('Your test has been running for at least five minutes. This '
'is probably not what you intended, so by default Hypothesis '
'turns it into an error.'), HealthCheck.hung_test)
try:
result = self.test_runner(
data, reify_and_execute(
self.search_strategy,
self.test,
))
if result is not None and self.settings.perform_health_check:
fail_health_check(
self.settings,
('Tests run under @given should return None, but '
'%s returned %r instead.') % (self.test.__name__, result),
HealthCheck.return_value)
self.at_least_one_success = True
return False
except UnsatisfiedAssumption:
data.mark_invalid()
except (
HypothesisDeprecationWarning,
FailedHealthCheck,
StopTest,
) + exceptions_to_reraise:
raise
except Exception:
escalate_hypothesis_internal_error()
self.last_exception = traceback.format_exc()
verbose_report(self.last_exception)
data.mark_interesting()
def evaluate_test_data(self, data):
try:
result = self.execute(data)
if result is not None:
fail_health_check(self.settings, (
'Tests run under @given should return None, but '
'%s returned %r instead.'
) % (self.test.__name__, result), HealthCheck.return_value)
return False
except UnsatisfiedAssumption:
data.mark_invalid()
except (
HypothesisDeprecationWarning, FailedHealthCheck,
StopTest,
) + EXCEPTIONS_TO_RERAISE:
raise
except Exception as e:
escalate_hypothesis_internal_error()
data.__expected_traceback = traceback.format_exc()
data.__expected_exception = e
verbose_report(data.__expected_traceback)
error_class, _, tb = sys.exc_info()
origin = traceback.extract_tb(tb)[-1]
filename = origin[0]
lineno = origin[1]
data.mark_interesting((error_class, filename, lineno))
def evaluate_test_data(self, data):
try:
result = self.test_runner(
data, reify_and_execute(
self.search_strategy,
self.test,
))
if result is not None and self.settings.perform_health_check:
fail_health_check(
self.settings,
('Tests run under @given should return None, but '
'%s returned %r instead.') % (self.test.__name__, result),
HealthCheck.return_value)
self.at_least_one_success = True
return False
except UnsatisfiedAssumption:
data.mark_invalid()
except (
HypothesisDeprecationWarning,
FailedHealthCheck,
StopTest,
):
raise
except Exception:
escalate_hypothesis_internal_error()
self.last_exception = traceback.format_exc()
verbose_report(self.last_exception)
data.mark_interesting()
def test_does_not_escalate_errors_in_hypothesis_file_if_disabled(monkeypatch):
monkeypatch.setattr(esc, 'is_hypothesis_file', lambda x: True)
monkeypatch.setattr(esc, 'PREVENT_ESCALATION', True)
try:
assert False
except AssertionError:
esc.escalate_hypothesis_internal_error()
def test_does_escalate_errors_in_hypothesis_file(monkeypatch):
monkeypatch.setattr(esc, 'is_hypothesis_file', lambda x: True)
with pytest.raises(AssertionError):
try:
assert False
except AssertionError:
esc.escalate_hypothesis_internal_error()
def test_does_escalate_errors_in_hypothesis_file(monkeypatch):
monkeypatch.setattr(esc, "is_hypothesis_file", lambda x: True)
with pytest.raises(AssertionError):
try:
assert False
except AssertionError:
esc.escalate_hypothesis_internal_error()
def test_does_not_escalate_errors_in_hypothesis_file_if_disabled(monkeypatch):
monkeypatch.setattr(esc, "is_hypothesis_file", lambda x: True)
monkeypatch.setattr(esc, "PREVENT_ESCALATION", True)
try:
assert False
except AssertionError:
esc.escalate_hypothesis_internal_error()
def _execute_once_for_engine(self, data):
"""Wrapper around ``execute_once`` that intercepts test failure
exceptions and single-test control exceptions, and turns them into
appropriate method calls to `data` instead.
This allows the engine to assume that any exception other than
``StopTest`` must be a fatal error, and should stop the entire engine.
"""
try:
result = self.execute_once(data)
if result is not None:
fail_health_check(
self.settings,
("Tests run under @given should return None, but "
"%s returned %r instead.") % (self.test.__name__, result),
HealthCheck.return_value,
)
except UnsatisfiedAssumption:
# An "assume" check failed, so instead we inform the engine that
# this test run was invalid.
data.mark_invalid()
except StopTest:
# The engine knows how to handle this control exception, so it's
# OK to re-raise it.
raise
except (
HypothesisDeprecationWarning,
FailedHealthCheck,
) + skip_exceptions_to_reraise():
# These are fatal errors or control exceptions that should stop the
# engine, so we re-raise them.
raise
except failure_exceptions_to_catch() as e:
# If the error was raised by Hypothesis-internal code, re-raise it
# as a fatal error instead of treating it as a test failure.
escalate_hypothesis_internal_error()
if data.frozen:
# This can happen if an error occurred in a finally
# block somewhere, suppressing our original StopTest.
# We raise a new one here to resume normal operation.
raise StopTest(data.testcounter)
else:
# The test failed by raising an exception, so we inform the
# engine that this test run was interesting. This is the normal
# path for test runs that fail.
tb = get_trimmed_traceback()
info = data.extra_information
info.__expected_traceback = "".join(
traceback.format_exception(type(e), e, tb))
info.__expected_exception = e
verbose_report(info.__expected_traceback)
origin = traceback.extract_tb(tb)[-1]
filename = origin[0]
lineno = origin[1]
data.mark_interesting((type(e), filename, lineno))
def evaluate_test_data(self, data):
try:
if self.collector is None:
result = self.execute(data)
else: # pragma: no cover
# This should always be a no-op, but the coverage tracer has
# a bad habit of resurrecting itself.
original = sys.gettrace()
sys.settrace(None)
try:
self.collector.data = {}
result = self.execute(data, collect=True)
finally:
sys.settrace(original)
covdata = CoverageData()
self.collector.save_data(covdata)
self.coverage_data.update(covdata)
for filename in covdata.measured_files():
if is_hypothesis_file(filename):
continue
data.tags.update(
arc(filename, source, target)
for source, target in covdata.arcs(filename))
if result is not None and self.settings.perform_health_check:
fail_health_check(
self.settings,
('Tests run under @given should return None, but '
'%s returned %r instead.') % (self.test.__name__, result),
HealthCheck.return_value)
self.at_least_one_success = True
return False
except UnsatisfiedAssumption:
data.mark_invalid()
except (
HypothesisDeprecationWarning,
FailedHealthCheck,
StopTest,
) + exceptions_to_reraise:
raise
except Exception as e:
escalate_hypothesis_internal_error()
data.__expected_traceback = traceback.format_exc()
data.__expected_exception = e
verbose_report(data.__expected_traceback)
error_class, _, tb = sys.exc_info()
origin = traceback.extract_tb(tb)[-1]
filename = origin[0]
lineno = origin[1]
data.mark_interesting((error_class, filename, lineno))
def evaluate_test_data(self, data):
try:
result = self.execute(data)
if result is not None:
fail_health_check(
self.settings,
(
"Tests run under @given should return None, but "
"%s returned %r instead."
)
% (self.test.__name__, result),
HealthCheck.return_value,
)
except UnsatisfiedAssumption:
data.mark_invalid()
except (
HypothesisDeprecationWarning,
FailedHealthCheck,
StopTest,
) + skip_exceptions_to_reraise():
raise
except failure_exceptions_to_catch() as e:
escalate_hypothesis_internal_error()
if data.frozen:
# This can happen if an error occurred in a finally
# block somewhere, suppressing our original StopTest.
# We raise a new one here to resume normal operation.
raise StopTest(data.testcounter)
else:
tb = get_trimmed_traceback()
info = data.extra_information
info.__expected_traceback = "".join(
traceback.format_exception(type(e), e, tb)
)
info.__expected_exception = e
verbose_report(info.__expected_traceback)
origin = traceback.extract_tb(tb)[-1]
filename = origin[0]
lineno = origin[1]
data.mark_interesting((type(e), filename, lineno))
def evaluate_test_data(self, data):
try:
result = self.execute(data)
if result is not None:
fail_health_check(
self.settings,
(
"Tests run under @given should return None, but "
"%s returned %r instead."
)
% (self.test.__name__, result),
HealthCheck.return_value,
)
except UnsatisfiedAssumption:
data.mark_invalid()
except (
HypothesisDeprecationWarning,
FailedHealthCheck,
StopTest,
) + skip_exceptions_to_reraise():
raise
except failure_exceptions_to_catch() as e:
escalate_hypothesis_internal_error()
if data.frozen:
# This can happen if an error occurred in a finally
# block somewhere, suppressing our original StopTest.
# We raise a new one here to resume normal operation.
raise StopTest(data.testcounter)
else:
tb = get_trimmed_traceback()
info = data.extra_information
info.__expected_traceback = "".join(
traceback.format_exception(type(e), e, tb)
)
info.__expected_exception = e
verbose_report(info.__expected_traceback)
origin = traceback.extract_tb(tb)[-1]
filename = origin[0]
lineno = origin[1]
data.mark_interesting((type(e), filename, lineno))
def evaluate_test_data(self, data):
if (time.time() - self.start_time >= HUNG_TEST_TIME_LIMIT):
fail_health_check(
self.settings,
('Your test has been running for at least five minutes. This '
'is probably not what you intended, so by default Hypothesis '
'turns it into an error.'), HealthCheck.hung_test)
try:
if self.collector is None:
result = self.test_runner(
data, reify_and_execute(
self.search_strategy,
self.test,
))
else: # pragma: no cover
# This should always be a no-op, but the coverage tracer has
# a bad habit of resurrecting itself.
original = sys.gettrace()
sys.settrace(None)
try:
try:
self.collector.data = {}
self.collector.start()
result = self.test_runner(
data,
reify_and_execute(
self.search_strategy,
self.test,
))
finally:
self.collector.stop()
finally:
sys.settrace(original)
covdata = CoverageData()
self.collector.save_data(covdata)
self.coverage_data.update(covdata)
for filename in covdata.measured_files():
if is_hypothesis_file(filename):
continue
for lineno in covdata.lines(filename):
data.add_tag(Line(filename, lineno))
for source, target in covdata.arcs(filename):
data.add_tag(Arc(filename, source, target))
if result is not None and self.settings.perform_health_check:
fail_health_check(
self.settings,
('Tests run under @given should return None, but '
'%s returned %r instead.') % (self.test.__name__, result),
HealthCheck.return_value)
self.at_least_one_success = True
return False
except UnsatisfiedAssumption:
data.mark_invalid()
except (
HypothesisDeprecationWarning,
FailedHealthCheck,
StopTest,
) + exceptions_to_reraise:
raise
except Exception:
escalate_hypothesis_internal_error()
data.__expected_exception = traceback.format_exc()
verbose_report(data.__expected_exception)
error_class, _, tb = sys.exc_info()
origin = traceback.extract_tb(tb)[-1]
filename = origin[0]
lineno = origin[1]
data.mark_interesting((error_class, filename, lineno))
def perform_health_checks(random, settings, test_runner, search_strategy):
# Tell pytest to omit the body of this function from tracebacks
__tracebackhide__ = True
if not settings.perform_health_check:
return
if not Settings.default.perform_health_check:
return
health_check_random = Random(random.getrandbits(128))
# We "pre warm" the health check with one draw to give it some
# time to calculate any cached data. This prevents the case
# where the first draw of the health check takes ages because
# of loading unicode data the first time.
data = ConjectureData(
max_length=settings.buffer_size,
draw_bytes=lambda data, n: uniform(health_check_random, n)
)
with Settings(settings, verbosity=Verbosity.quiet):
try:
test_runner(data, reify_and_execute(
search_strategy,
lambda *args, **kwargs: None,
))
except BaseException:
pass
count = 0
overruns = 0
filtered_draws = 0
start = time.time()
while (
count < 10 and time.time() < start + 1 and
filtered_draws < 50 and overruns < 20
):
try:
data = ConjectureData(
max_length=settings.buffer_size,
draw_bytes=lambda data, n: uniform(health_check_random, n)
)
with Settings(settings, verbosity=Verbosity.quiet):
test_runner(data, reify_and_execute(
search_strategy,
lambda *args, **kwargs: None,
))
count += 1
except UnsatisfiedAssumption:
filtered_draws += 1
except StopTest:
if data.status == Status.INVALID:
filtered_draws += 1
else:
assert data.status == Status.OVERRUN
overruns += 1
except InvalidArgument:
raise
except Exception:
escalate_hypothesis_internal_error()
if (
HealthCheck.exception_in_generation in
settings.suppress_health_check
):
raise
report(traceback.format_exc())
if test_runner is default_new_style_executor:
fail_health_check(
settings,
'An exception occurred during data '
'generation in initial health check. '
'This indicates a bug in the strategy. '
'This could either be a Hypothesis bug or '
"an error in a function you've passed to "
'it to construct your data.',
HealthCheck.exception_in_generation,
)
else:
fail_health_check(
settings,
'An exception occurred during data '
'generation in initial health check. '
'This indicates a bug in the strategy. '
'This could either be a Hypothesis bug or '
'an error in a function you\'ve passed to '
'it to construct your data. Additionally, '
'you have a custom executor, which means '
'that this could be your executor failing '
'to handle a function which returns None. ',
HealthCheck.exception_in_generation,
)
if overruns >= 20 or (
not count and overruns > 0
):
fail_health_check(settings, (
'Examples routinely exceeded the max allowable size. '
'(%d examples overran while generating %d valid ones)'
'. Generating examples this large will usually lead to'
' bad results. You should try setting average_size or '
'max_size parameters on your collections and turning '
'max_leaves down on recursive() calls.') % (
overruns, count
), HealthCheck.data_too_large)
if filtered_draws >= 50 or (
not count and filtered_draws > 0
):
fail_health_check(settings, (
'It looks like your strategy is filtering out a lot '
'of data. Health check found %d filtered examples but '
'only %d good ones. This will make your tests much '
'slower, and also will probably distort the data '
'generation quite a lot. You should adapt your '
'strategy to filter less. This can also be caused by '
'a low max_leaves parameter in recursive() calls') % (
filtered_draws, count
), HealthCheck.filter_too_much)
runtime = time.time() - start
if runtime > 1.0 or count < 10:
fail_health_check(settings, (
'Data generation is extremely slow: Only produced '
'%d valid examples in %.2f seconds (%d invalid ones '
'and %d exceeded maximum size). Try decreasing '
"size of the data you're generating (with e.g."
'average_size or max_leaves parameters).'
) % (count, runtime, filtered_draws, overruns),
HealthCheck.too_slow,
)
def test_does_not_escalate_errors_in_non_hypothesis_file():
try:
assert False
except AssertionError:
esc.escalate_hypothesis_internal_error()
def evaluate_test_data(self, data):
if (
time.time() - self.start_time >= HUNG_TEST_TIME_LIMIT
):
fail_health_check(self.settings, (
'Your test has been running for at least five minutes. This '
'is probably not what you intended, so by default Hypothesis '
'turns it into an error.'
), HealthCheck.hung_test)
try:
if self.collector is None:
result = self.test_runner(data, reify_and_execute(
self.search_strategy, self.test,
))
else: # pragma: no cover
# This should always be a no-op, but the coverage tracer has
# a bad habit of resurrecting itself.
original = sys.gettrace()
sys.settrace(None)
try:
self.collector.data = {}
result = self.test_runner(data, reify_and_execute(
self.search_strategy, self.test,
collector=self.collector
))
finally:
sys.settrace(original)
covdata = CoverageData()
self.collector.save_data(covdata)
self.coverage_data.update(covdata)
for filename in covdata.measured_files():
if is_hypothesis_file(filename):
continue
data.tags.update(
arc(filename, source, target)
for source, target in covdata.arcs(filename)
)
if result is not None and self.settings.perform_health_check:
fail_health_check(self.settings, (
'Tests run under @given should return None, but '
'%s returned %r instead.'
) % (self.test.__name__, result), HealthCheck.return_value)
self.at_least_one_success = True
return False
except UnsatisfiedAssumption:
data.mark_invalid()
except (
HypothesisDeprecationWarning, FailedHealthCheck,
StopTest,
) + exceptions_to_reraise:
raise
except Exception as e:
escalate_hypothesis_internal_error()
data.__expected_traceback = traceback.format_exc()
data.__expected_exception = e
verbose_report(data.__expected_traceback)
error_class, _, tb = sys.exc_info()
origin = traceback.extract_tb(tb)[-1]
filename = origin[0]
lineno = origin[1]
data.mark_interesting((error_class, filename, lineno))
def test_does_not_escalate_errors_in_non_hypothesis_file():
try:
assert False
except AssertionError:
esc.escalate_hypothesis_internal_error()
def perform_health_checks(random, settings, test_runner, search_strategy):
if not settings.perform_health_check:
return
if not Settings.default.perform_health_check:
return
health_check_random = Random(random.getrandbits(128))
# We "pre warm" the health check with one draw to give it some
# time to calculate any cached data. This prevents the case
# where the first draw of the health check takes ages because
# of loading unicode data the first time.
data = ConjectureData(max_length=settings.buffer_size,
draw_bytes=lambda data, n, distribution:
distribution(health_check_random, n))
with Settings(settings, verbosity=Verbosity.quiet):
try:
test_runner(
data,
reify_and_execute(
search_strategy,
lambda *args, **kwargs: None,
))
except BaseException:
pass
count = 0
overruns = 0
filtered_draws = 0
start = time.time()
while (count < 10 and time.time() < start + 1 and filtered_draws < 50
and overruns < 20):
try:
data = ConjectureData(max_length=settings.buffer_size,
draw_bytes=lambda data, n, distribution:
distribution(health_check_random, n))
with Settings(settings, verbosity=Verbosity.quiet):
test_runner(
data,
reify_and_execute(
search_strategy,
lambda *args, **kwargs: None,
))
count += 1
except UnsatisfiedAssumption:
filtered_draws += 1
except StopTest:
if data.status == Status.INVALID:
filtered_draws += 1
else:
assert data.status == Status.OVERRUN
overruns += 1
except InvalidArgument:
raise
except Exception:
escalate_hypothesis_internal_error()
if (HealthCheck.exception_in_generation
in settings.suppress_health_check):
raise
report(traceback.format_exc())
if test_runner is default_new_style_executor:
fail_health_check(
settings,
'An exception occurred during data '
'generation in initial health check. '
'This indicates a bug in the strategy. '
'This could either be a Hypothesis bug or '
"an error in a function you've passed to "
'it to construct your data.',
HealthCheck.exception_in_generation,
)
else:
fail_health_check(
settings,
'An exception occurred during data '
'generation in initial health check. '
'This indicates a bug in the strategy. '
'This could either be a Hypothesis bug or '
'an error in a function you\'ve passed to '
'it to construct your data. Additionally, '
'you have a custom executor, which means '
'that this could be your executor failing '
'to handle a function which returns None. ',
HealthCheck.exception_in_generation,
)
if overruns >= 20 or (not count and overruns > 0):
fail_health_check(
settings,
('Examples routinely exceeded the max allowable size. '
'(%d examples overran while generating %d valid ones)'
'. Generating examples this large will usually lead to'
' bad results. You should try setting average_size or '
'max_size parameters on your collections and turning '
'max_leaves down on recursive() calls.') % (overruns, count),
HealthCheck.data_too_large)
if filtered_draws >= 50 or (not count and filtered_draws > 0):
fail_health_check(
settings, ('It looks like your strategy is filtering out a lot '
'of data. Health check found %d filtered examples but '
'only %d good ones. This will make your tests much '
'slower, and also will probably distort the data '
'generation quite a lot. You should adapt your '
'strategy to filter less. This can also be caused by '
'a low max_leaves parameter in recursive() calls') %
(filtered_draws, count), HealthCheck.filter_too_much)
runtime = time.time() - start
if runtime > 1.0 or count < 10:
fail_health_check(
settings,
('Data generation is extremely slow: Only produced '
'%d valid examples in %.2f seconds (%d invalid ones '
'and %d exceeded maximum size). Try decreasing '
"size of the data you're generating (with e.g."
'average_size or max_leaves parameters).') %
(count, runtime, filtered_draws, overruns),
HealthCheck.too_slow,
)
