def compute_coverage(branch):
coverage_data = CoverageData()
try:
with project_path.join('.coverage').open() as fp:
coverage_data.read_file(fp)
except Exception:
print("No coverage data found", file=sys.stderr)
git_proc = subprocess.Popen(['git', 'diff', '-U0', branch],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
git_output = git_proc.stdout.read()
files = git_output.split("diff --git")
from collections import defaultdict
file_data = defaultdict(list)
for the_file in files:
filenames = re.findall('a/(.*?) b/(.*)', the_file)
if not filenames:
continue
filename = project_path.join(filenames[0][1])
if '.py' != filename.ext:
continue
the_file += "git_output_checker"
the_diffs = re.findall(
r'(@@.*?@@.*?(?=@@|git_output_checker))',
the_file,
re.M | re.S,
)
for diff in the_diffs:
diff_args = re.match(r'@@ -(\d+)(,(\d+))*\s+\+(\d+)(,(\d+))*',
diff).groups()
if diff_args[5]:
for extra_line in range(int(diff_args[5])):
file_data[filename].append(extra_line + int(diff_args[3]))
else:
file_data[filename].append(int(diff_args[3]))
line_count = 0
completed_lines = 0
for file_changed, lines in file_data.items():
for line in lines:
line_count += 1
used_lines = coverage_data.lines(file_changed)
if not used_lines:
continue
if isinstance(used_lines, int):
used_lines = set([used_lines])
else:
used_lines = set(used_lines)
if line in used_lines:
completed_lines += 1
return float(completed_lines) / line_count * 100
def compute_coverage(branch):
coverage_data = CoverageData()
try:
with project_path.join('.coverage').open() as fp:
coverage_data.read_file(fp)
except Exception:
print("No coverage data found", file=sys.stderr)
git_proc = subprocess.Popen(['git', 'diff', '-U0', branch],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
git_output = git_proc.stdout.read()
files = git_output.split("diff --git")
from collections import defaultdict
file_data = defaultdict(list)
for the_file in files:
filenames = re.findall('a/(.*?) b/(.*)', the_file)
if not filenames:
continue
filename = project_path.join(filenames[0][1])
if '.py' != filename.ext:
continue
the_file += "git_output_checker"
the_diffs = re.findall('(@@.*?@@.*?(?=@@|git_output_checker))', the_file, re.M | re.S, )
for diff in the_diffs:
diff_args = re.match('@@ -(\d+)(,(\d+))*\s+\+(\d+)(,(\d+))*', diff).groups()
if diff_args[5]:
for extra_line in range(int(diff_args[5])):
file_data[filename].append(extra_line + int(diff_args[3]))
else:
file_data[filename].append(int(diff_args[3]))
line_count = 0
completed_lines = 0
for file_changed, lines in file_data.items():
for line in lines:
line_count += 1
used_lines = coverage_data.lines(file_changed)
if not used_lines:
continue
if isinstance(used_lines, int):
used_lines = set([used_lines])
else:
used_lines = set(used_lines)
if line in used_lines:
completed_lines += 1
return float(completed_lines) / line_count * 100
def print_coverage(coverage_data: CoverageData, coverage: Coverage):
for f in coverage_data.measured_files():
lines = coverage_data.lines(f)
print('')
print('')
pprint(f)
print('lines:')
pprint(lines)
print('')
print('arcs:')
pprint(coverage_data.arcs(f))
print('')
print('')
zzz = coverage.analysis(f)
pprint(zzz)
print('')
print()
summary = coverage_data.line_counts()
print('line_counts')
pprint(summary)
class StateForActualGivenExecution(object):
def __init__(self, test_runner, search_strategy, test, settings, random,
had_seed):
self.test_runner = test_runner
self.search_strategy = search_strategy
self.settings = settings
self.last_exception = None
self.falsifying_examples = ()
self.__was_flaky = False
self.random = random
self.__warned_deadline = False
self.__existing_collector = None
self.__test_runtime = None
self.__had_seed = had_seed
self.test = test
self.coverage_data = CoverageData()
self.files_to_propagate = set()
self.failed_normally = False
self.used_examples_from_database = False
if settings.use_coverage and not IN_COVERAGE_TESTS: # pragma: no cover
if Collector._collectors:
parent = Collector._collectors[-1]
# We include any files the collector has already decided to
# trace whether or not on re-investigation we still think it
# wants to trace them. The reason for this is that in some
# cases coverage gets the wrong answer when we run it
# ourselves due to reasons that are our fault but are hard to
# fix (we lie about where certain functions come from).
# This causes us to not record the actual test bodies as
# covered. But if we intended to trace test bodies then the
# file must already have been traced when getting to this point
# and so will already be in the collector's data. Hence we can
# use that information to get the correct answer here.
# See issue 997 for more context.
self.files_to_propagate = set(parent.data)
self.hijack_collector(parent)
self.collector = Collector(
branch=True,
timid=FORCE_PURE_TRACER,
should_trace=self.should_trace,
check_include=hypothesis_check_include,
concurrency='thread',
warn=escalate_warning,
)
self.collector.reset()
# Hide the other collectors from this one so it doesn't attempt to
# pause them (we're doing trace function management ourselves so
# this will just cause problems).
self.collector._collectors = []
else:
self.collector = None
def execute(
self,
data,
print_example=False,
is_final=False,
expected_failure=None,
collect=False,
):
text_repr = [None]
if self.settings.deadline is None:
test = self.test
else:
@proxies(self.test)
def test(*args, **kwargs):
self.__test_runtime = None
initial_draws = len(data.draw_times)
start = benchmark_time()
result = self.test(*args, **kwargs)
finish = benchmark_time()
internal_draw_time = sum(data.draw_times[initial_draws:])
runtime = (finish - start - internal_draw_time) * 1000
self.__test_runtime = runtime
if self.settings.deadline is not_set:
if (not self.__warned_deadline and runtime >= 200):
self.__warned_deadline = True
note_deprecation(
('Test took %.2fms to run. In future the default '
'deadline setting will be 200ms, which will '
'make this an error. You can set deadline to '
'an explicit value of e.g. %d to turn tests '
'slower than this into an error, or you can set '
'it to None to disable this check entirely.') % (
runtime,
ceil(runtime / 100) * 100,
))
else:
current_deadline = self.settings.deadline
if not is_final:
current_deadline *= 1.25
if runtime >= current_deadline:
raise DeadlineExceeded(runtime, self.settings.deadline)
return result
def run(data):
if not hasattr(data, 'can_reproduce_example_from_repr'):
data.can_reproduce_example_from_repr = True
with self.settings:
with BuildContext(data, is_final=is_final):
with deterministic_PRNG():
args, kwargs = data.draw(self.search_strategy)
if expected_failure is not None:
text_repr[0] = arg_string(test, args, kwargs)
if print_example:
example = '%s(%s)' % (test.__name__,
arg_string(test, args, kwargs))
try:
ast.parse(example)
except SyntaxError:
data.can_reproduce_example_from_repr = False
report('Falsifying example: %s' % (example, ))
elif current_verbosity() >= Verbosity.verbose:
report(lambda: 'Trying example: %s(%s)' %
(test.__name__, arg_string(test, args, kwargs)))
if self.collector is None or not collect:
with deterministic_PRNG():
return test(*args, **kwargs)
else: # pragma: no cover
try:
self.collector.start()
with deterministic_PRNG():
return test(*args, **kwargs)
finally:
self.collector.stop()
result = self.test_runner(data, run)
if expected_failure is not None:
exception, traceback = expected_failure
if (isinstance(exception, DeadlineExceeded)
and self.__test_runtime is not None):
report(
('Unreliable test timings! On an initial run, this '
'test took %.2fms, which exceeded the deadline of '
'%.2fms, but on a subsequent run it took %.2f ms, '
'which did not. If you expect this sort of '
'variability in your test timings, consider turning '
'deadlines off for this test by setting deadline=None.') %
(exception.runtime, self.settings.deadline,
self.__test_runtime))
else:
report(
'Failed to reproduce exception. Expected: \n' +
traceback, )
self.__flaky(
('Hypothesis %s(%s) produces unreliable results: Falsified'
' on the first call but did not on a subsequent one') % (
test.__name__,
text_repr[0],
))
return result
def should_trace(self, original_filename, frame): # pragma: no cover
disp = FileDisposition()
assert original_filename is not None
disp.original_filename = original_filename
disp.canonical_filename = encoded_filepath(
canonical_filename(original_filename))
disp.source_filename = disp.canonical_filename
disp.reason = ''
disp.file_tracer = None
disp.has_dynamic_filename = False
disp.trace = hypothesis_check_include(disp.canonical_filename)
if not disp.trace:
disp.reason = 'hypothesis internal reasons'
elif self.__existing_collector is not None:
check = self.__existing_collector.should_trace(
original_filename, frame)
if check.trace:
self.files_to_propagate.add(check.canonical_filename)
return disp
def hijack_collector(self, collector): # pragma: no cover
self.__existing_collector = collector
original_save_data = collector.save_data
def save_data(covdata):
original_save_data(covdata)
if collector.branch:
covdata.add_arcs({
filename: {
arc: None
for arc in self.coverage_data.arcs(filename) or ()
}
for filename in self.files_to_propagate
})
else:
covdata.add_lines({
filename: {
line: None
for line in self.coverage_data.lines(filename) or ()
}
for filename in self.files_to_propagate
})
collector.save_data = original_save_data
collector.save_data = save_data
def evaluate_test_data(self, data):
try:
if self.collector is None:
result = self.execute(data)
else: # pragma: no cover
# This should always be a no-op, but the coverage tracer has
# a bad habit of resurrecting itself.
original = sys.gettrace()
sys.settrace(None)
try:
self.collector.data = {}
result = self.execute(data, collect=True)
finally:
sys.settrace(original)
covdata = CoverageData()
self.collector.save_data(covdata)
self.coverage_data.update(covdata)
for filename in covdata.measured_files():
if is_hypothesis_file(filename):
continue
data.tags.update(
arc(filename, source, target)
for source, target in covdata.arcs(filename))
if result is not None:
fail_health_check(
self.settings,
('Tests run under @given should return None, but '
'%s returned %r instead.') % (self.test.__name__, result),
HealthCheck.return_value)
return False
except UnsatisfiedAssumption:
data.mark_invalid()
except (
HypothesisDeprecationWarning,
FailedHealthCheck,
StopTest,
) + exceptions_to_reraise:
raise
except Exception as e:
escalate_hypothesis_internal_error()
data.__expected_traceback = traceback.format_exc()
data.__expected_exception = e
verbose_report(data.__expected_traceback)
error_class, _, tb = sys.exc_info()
origin = traceback.extract_tb(tb)[-1]
filename = origin[0]
lineno = origin[1]
data.mark_interesting((error_class, filename, lineno))
def run(self):
# Tell pytest to omit the body of this function from tracebacks
__tracebackhide__ = True
if global_force_seed is None:
database_key = str_to_bytes(fully_qualified_name(self.test))
else:
database_key = None
self.start_time = time.time()
global in_given
runner = ConjectureRunner(
self.evaluate_test_data,
settings=self.settings,
random=self.random,
database_key=database_key,
)
if in_given or self.collector is None:
runner.run()
else: # pragma: no cover
in_given = True
original_trace = sys.gettrace()
try:
sys.settrace(None)
runner.run()
finally:
in_given = False
sys.settrace(original_trace)
self.used_examples_from_database = \
runner.used_examples_from_database
note_engine_for_statistics(runner)
run_time = time.time() - self.start_time
self.used_examples_from_database = runner.used_examples_from_database
if runner.used_examples_from_database:
if self.settings.derandomize:
note_deprecation(
'In future derandomize will imply database=None, but your '
'test is currently using examples from the database. To '
'get the future behaviour, update your settings to '
'include database=None.')
if self.__had_seed:
note_deprecation(
'In future use of @seed will imply database=None in your '
'settings, but your test is currently using examples from '
'the database. To get the future behaviour, update your '
'settings for this test to include database=None.')
timed_out = runner.exit_reason == ExitReason.timeout
if runner.call_count == 0:
return
if runner.interesting_examples:
self.falsifying_examples = sorted(
[d for d in runner.interesting_examples.values()],
key=lambda d: sort_key(d.buffer),
reverse=True)
else:
if timed_out:
note_deprecation((
'Your tests are hitting the settings timeout (%.2fs). '
'This functionality will go away in a future release '
'and you should not rely on it. Instead, try setting '
'max_examples to be some value lower than %d (the number '
'of examples your test successfully ran here). Or, if you '
'would prefer your tests to run to completion, regardless '
'of how long they take, you can set the timeout value to '
'hypothesis.unlimited.') % (self.settings.timeout,
runner.valid_examples),
self.settings)
if runner.valid_examples == 0:
if timed_out:
raise Timeout(
('Ran out of time before finding a satisfying '
'example for %s. Only found %d examples in %.2fs.') %
(get_pretty_function_description(
self.test), runner.valid_examples, run_time))
else:
raise Unsatisfiable(
'Unable to satisfy assumptions of hypothesis %s.' %
(get_pretty_function_description(self.test), ))
if not self.falsifying_examples:
return
self.failed_normally = True
flaky = 0
for falsifying_example in self.falsifying_examples:
ran_example = ConjectureData.for_buffer(falsifying_example.buffer)
self.__was_flaky = False
assert falsifying_example.__expected_exception is not None
try:
self.execute(ran_example,
print_example=True,
is_final=True,
expected_failure=(
falsifying_example.__expected_exception,
falsifying_example.__expected_traceback,
))
except (UnsatisfiedAssumption, StopTest):
report(traceback.format_exc())
self.__flaky(
'Unreliable assumption: An example which satisfied '
'assumptions on the first run now fails it.')
except BaseException:
if len(self.falsifying_examples) <= 1:
raise
report(traceback.format_exc())
finally: # pragma: no cover
# This section is in fact entirely covered by the tests in
# test_reproduce_failure, but it seems to trigger a lovely set
# of coverage bugs: The branches show up as uncovered (despite
# definitely being covered - you can add an assert False else
# branch to verify this and see it fail - and additionally the
# second branch still complains about lack of coverage even if
# you add a pragma: no cover to it!
# See https://bitbucket.org/ned/coveragepy/issues/623/
if self.settings.print_blob is not PrintSettings.NEVER:
failure_blob = encode_failure(falsifying_example.buffer)
# Have to use the example we actually ran, not the original
# falsifying example! Otherwise we won't catch problems
# where the repr of the generated example doesn't parse.
can_use_repr = ran_example.can_reproduce_example_from_repr
if (self.settings.print_blob is PrintSettings.ALWAYS or
(self.settings.print_blob is PrintSettings.INFER
and not can_use_repr and len(failure_blob) < 200)):
report((
'\n'
'You can reproduce this example by temporarily '
'adding @reproduce_failure(%r, %r) as a decorator '
'on your test case') % (
__version__,
failure_blob,
))
if self.__was_flaky:
flaky += 1
# If we only have one example then we should have raised an error or
# flaky prior to this point.
assert len(self.falsifying_examples) > 1
if flaky > 0:
raise Flaky(
('Hypothesis found %d distinct failures, but %d of them '
'exhibited some sort of flaky behaviour.') %
(len(self.falsifying_examples), flaky))
else:
raise MultipleFailures(('Hypothesis found %d distinct failures.') %
(len(self.falsifying_examples, )))
def __flaky(self, message):
if len(self.falsifying_examples) <= 1:
raise Flaky(message)
else:
self.__was_flaky = True
report('Flaky example! ' + message)
def set_coverage(self, cov: coverage.CoverageData):
for file in cov.measured_files():
self.lines_covered[file] = frozenset(cov.lines(file))
self.branches_covered[file] = frozenset(cov.arcs(file))
def set_coverage(self, cov: coverage.CoverageData):
for file in cov.measured_files():
self.lines_covered[file] = frozenset(cov.lines(file))
self.branches_covered[file] = frozenset(cov.arcs(file))
class StateForActualGivenExecution(object):
def __init__(self, test_runner, search_strategy, test, settings, random):
self.test_runner = test_runner
self.search_strategy = search_strategy
self.settings = settings
self.at_least_one_success = False
self.last_exception = None
self.repr_for_last_exception = None
self.falsifying_examples = ()
self.__was_flaky = False
self.random = random
self.__warned_deadline = False
self.__existing_collector = None
self.__test_runtime = None
self.__in_final_replay = False
if self.settings.deadline is None:
self.test = test
else:
@proxies(test)
def timed_test(*args, **kwargs):
self.__test_runtime = None
start = time.time()
result = test(*args, **kwargs)
runtime = (time.time() - start) * 1000
self.__test_runtime = runtime
if self.settings.deadline is not_set:
if (not self.__warned_deadline and runtime >= 200):
self.__warned_deadline = True
note_deprecation(
('Test took %.2fms to run. In future the default '
'deadline setting will be 200ms, which will '
'make this an error. You can set deadline to '
'an explicit value of e.g. %d to turn tests '
'slower than this into an error, or you can set '
'it to None to disable this check entirely.') % (
runtime,
ceil(runtime / 100) * 100,
))
elif runtime >= self.current_deadline:
raise DeadlineExceeded(runtime, self.settings.deadline)
return result
self.test = timed_test
self.coverage_data = CoverageData()
self.files_to_propagate = set()
if settings.use_coverage and not IN_COVERAGE_TESTS: # pragma: no cover
if Collector._collectors:
self.hijack_collector(Collector._collectors[-1])
self.collector = Collector(
branch=True,
timid=FORCE_PURE_TRACER,
should_trace=self.should_trace,
check_include=hypothesis_check_include,
concurrency='thread',
warn=escalate_warning,
)
self.collector.reset()
# Hide the other collectors from this one so it doesn't attempt to
# pause them (we're doing trace function management ourselves so
# this will just cause problems).
self.collector._collectors = []
else:
self.collector = None
@property
def current_deadline(self):
base = self.settings.deadline
if self.__in_final_replay:
return base
else:
return base * 1.25
def should_trace(self, original_filename, frame): # pragma: no cover
disp = FileDisposition()
assert original_filename is not None
disp.original_filename = original_filename
disp.canonical_filename = encoded_filepath(
canonical_filename(original_filename))
disp.source_filename = disp.canonical_filename
disp.reason = ''
disp.file_tracer = None
disp.has_dynamic_filename = False
disp.trace = hypothesis_check_include(disp.canonical_filename)
if not disp.trace:
disp.reason = 'hypothesis internal reasons'
elif self.__existing_collector is not None:
check = self.__existing_collector.should_trace(
original_filename, frame)
if check.trace:
self.files_to_propagate.add(check.canonical_filename)
return disp
def hijack_collector(self, collector): # pragma: no cover
self.__existing_collector = collector
original_save_data = collector.save_data
def save_data(covdata):
original_save_data(covdata)
if collector.branch:
covdata.add_arcs({
filename:
{arc: None
for arc in self.coverage_data.arcs(filename)}
for filename in self.files_to_propagate
})
else:
covdata.add_lines({
filename: {
line: None
for line in self.coverage_data.lines(filename)
}
for filename in self.files_to_propagate
})
collector.save_data = original_save_data
collector.save_data = save_data
def evaluate_test_data(self, data):
if (time.time() - self.start_time >= HUNG_TEST_TIME_LIMIT):
fail_health_check(
self.settings,
('Your test has been running for at least five minutes. This '
'is probably not what you intended, so by default Hypothesis '
'turns it into an error.'), HealthCheck.hung_test)
try:
if self.collector is None:
result = self.test_runner(
data, reify_and_execute(
self.search_strategy,
self.test,
))
else: # pragma: no cover
# This should always be a no-op, but the coverage tracer has
# a bad habit of resurrecting itself.
original = sys.gettrace()
sys.settrace(None)
try:
self.collector.data = {}
result = self.test_runner(
data,
reify_and_execute(self.search_strategy,
self.test,
collector=self.collector))
finally:
sys.settrace(original)
covdata = CoverageData()
self.collector.save_data(covdata)
self.coverage_data.update(covdata)
for filename in covdata.measured_files():
if is_hypothesis_file(filename):
continue
data.tags.update(
arc(filename, source, target)
for source, target in covdata.arcs(filename))
if result is not None and self.settings.perform_health_check:
fail_health_check(
self.settings,
('Tests run under @given should return None, but '
'%s returned %r instead.') % (self.test.__name__, result),
HealthCheck.return_value)
self.at_least_one_success = True
return False
except UnsatisfiedAssumption:
data.mark_invalid()
except (
HypothesisDeprecationWarning,
FailedHealthCheck,
StopTest,
) + exceptions_to_reraise:
raise
except Exception as e:
escalate_hypothesis_internal_error()
data.__expected_traceback = traceback.format_exc()
data.__expected_exception = e
verbose_report(data.__expected_traceback)
error_class, _, tb = sys.exc_info()
origin = traceback.extract_tb(tb)[-1]
filename = origin[0]
lineno = origin[1]
data.mark_interesting((error_class, filename, lineno))
def run(self):
# Tell pytest to omit the body of this function from tracebacks
__tracebackhide__ = True
database_key = str_to_bytes(fully_qualified_name(self.test))
self.start_time = time.time()
global in_given
runner = ConjectureRunner(
self.evaluate_test_data,
settings=self.settings,
random=self.random,
database_key=database_key,
)
if in_given or self.collector is None:
runner.run()
else: # pragma: no cover
in_given = True
original_trace = sys.gettrace()
try:
sys.settrace(None)
runner.run()
finally:
in_given = False
sys.settrace(original_trace)
note_engine_for_statistics(runner)
run_time = time.time() - self.start_time
timed_out = runner.exit_reason == ExitReason.timeout
if runner.last_data is None:
return
if runner.interesting_examples:
self.falsifying_examples = sorted(
[d for d in runner.interesting_examples.values()],
key=lambda d: sort_key(d.buffer),
reverse=True)
else:
if timed_out:
note_deprecation((
'Your tests are hitting the settings timeout (%.2fs). '
'This functionality will go away in a future release '
'and you should not rely on it. Instead, try setting '
'max_examples to be some value lower than %d (the number '
'of examples your test successfully ran here). Or, if you '
'would prefer your tests to run to completion, regardless '
'of how long they take, you can set the timeout value to '
'hypothesis.unlimited.') % (self.settings.timeout,
runner.valid_examples),
self.settings)
if runner.valid_examples < min(
self.settings.min_satisfying_examples,
self.settings.max_examples,
) and not (runner.exit_reason == ExitReason.finished
and self.at_least_one_success):
if timed_out:
raise Timeout(
('Ran out of time before finding a satisfying '
'example for '
'%s. Only found %d examples in ' + '%.2fs.') %
(get_pretty_function_description(
self.test), runner.valid_examples, run_time))
else:
raise Unsatisfiable(
('Unable to satisfy assumptions of hypothesis '
'%s. Only %d examples considered '
'satisfied assumptions') % (
get_pretty_function_description(self.test),
runner.valid_examples,
))
if not self.falsifying_examples:
return
flaky = 0
self.__in_final_replay = True
for falsifying_example in self.falsifying_examples:
self.__was_flaky = False
raised_exception = False
try:
with self.settings:
self.test_runner(
ConjectureData.for_buffer(falsifying_example.buffer),
reify_and_execute(self.search_strategy,
self.test,
print_example=True,
is_final=True))
except (UnsatisfiedAssumption, StopTest):
report(traceback.format_exc())
self.__flaky(
'Unreliable assumption: An example which satisfied '
'assumptions on the first run now fails it.')
except:
if len(self.falsifying_examples) <= 1:
raise
raised_exception = True
report(traceback.format_exc())
if not raised_exception:
if (isinstance(falsifying_example.__expected_exception,
DeadlineExceeded)
and self.__test_runtime is not None):
report((
'Unreliable test timings! On an initial run, this '
'test took %.2fms, which exceeded the deadline of '
'%.2fms, but on a subsequent run it took %.2f ms, '
'which did not. If you expect this sort of '
'variability in your test timings, consider turning '
'deadlines off for this test by setting deadline=None.'
) % (falsifying_example.__expected_exception.runtime,
self.settings.deadline, self.__test_runtime))
else:
report(
'Failed to reproduce exception. Expected: \n' +
falsifying_example.__expected_traceback, )
filter_message = (
'Unreliable test data: Failed to reproduce a failure '
'and then when it came to recreating the example in '
'order to print the test data with a flaky result '
'the example was filtered out (by e.g. a '
'call to filter in your strategy) when we didn\'t '
'expect it to be.')
try:
self.test_runner(
ConjectureData.for_buffer(falsifying_example.buffer),
reify_and_execute(self.search_strategy,
test_is_flaky(
self.test,
self.repr_for_last_exception),
print_example=True,
is_final=True))
except (UnsatisfiedAssumption, StopTest):
self.__flaky(filter_message)
except Flaky as e:
if len(self.falsifying_examples) > 1:
self.__flaky(e.args[0])
else:
raise
if self.__was_flaky:
flaky += 1
# If we only have one example then we should have raised an error or
# flaky prior to this point.
assert len(self.falsifying_examples) > 1
if flaky > 0:
raise Flaky(
('Hypothesis found %d distinct failures, but %d of them '
'exhibited some sort of flaky behaviour.') %
(len(self.falsifying_examples), flaky))
else:
raise MultipleFailures(('Hypothesis found %d distinct failures.') %
(len(self.falsifying_examples, )))
def __flaky(self, message):
if len(self.falsifying_examples) <= 1:
raise Flaky(message)
else:
self.__was_flaky = True
report('Flaky example! ' + message)
class StateForActualGivenExecution(object):
def __init__(self, test_runner, search_strategy, test, settings, random):
self.test_runner = test_runner
self.search_strategy = search_strategy
self.settings = settings
self.at_least_one_success = False
self.last_exception = None
self.repr_for_last_exception = None
self.falsifying_examples = ()
self.__was_flaky = False
self.random = random
self.__warned_deadline = False
self.__existing_collector = None
self.__test_runtime = None
self.__in_final_replay = False
if self.settings.deadline is None:
self.test = test
else:
@proxies(test)
def timed_test(*args, **kwargs):
self.__test_runtime = None
start = time.time()
result = test(*args, **kwargs)
runtime = (time.time() - start) * 1000
self.__test_runtime = runtime
if self.settings.deadline is not_set:
if (
not self.__warned_deadline and
runtime >= 200
):
self.__warned_deadline = True
note_deprecation((
'Test took %.2fms to run. In future the default '
'deadline setting will be 200ms, which will '
'make this an error. You can set deadline to '
'an explicit value of e.g. %d to turn tests '
'slower than this into an error, or you can set '
'it to None to disable this check entirely.') % (
runtime, ceil(runtime / 100) * 100,
))
elif runtime >= self.current_deadline:
raise DeadlineExceeded(runtime, self.settings.deadline)
return result
self.test = timed_test
self.coverage_data = CoverageData()
self.files_to_propagate = set()
if settings.use_coverage and not IN_COVERAGE_TESTS: # pragma: no cover
if Collector._collectors:
self.hijack_collector(Collector._collectors[-1])
self.collector = Collector(
branch=True,
timid=FORCE_PURE_TRACER,
should_trace=self.should_trace,
check_include=hypothesis_check_include,
concurrency='thread',
warn=escalate_warning,
)
self.collector.reset()
# Hide the other collectors from this one so it doesn't attempt to
# pause them (we're doing trace function management ourselves so
# this will just cause problems).
self.collector._collectors = []
else:
self.collector = None
@property
def current_deadline(self):
base = self.settings.deadline
if self.__in_final_replay:
return base
else:
return base * 1.25
def should_trace(self, original_filename, frame): # pragma: no cover
disp = FileDisposition()
assert original_filename is not None
disp.original_filename = original_filename
disp.canonical_filename = encoded_filepath(
canonical_filename(original_filename))
disp.source_filename = disp.canonical_filename
disp.reason = ''
disp.file_tracer = None
disp.has_dynamic_filename = False
disp.trace = hypothesis_check_include(disp.canonical_filename)
if not disp.trace:
disp.reason = 'hypothesis internal reasons'
elif self.__existing_collector is not None:
check = self.__existing_collector.should_trace(
original_filename, frame)
if check.trace:
self.files_to_propagate.add(check.canonical_filename)
return disp
def hijack_collector(self, collector): # pragma: no cover
self.__existing_collector = collector
original_save_data = collector.save_data
def save_data(covdata):
original_save_data(covdata)
if collector.branch:
covdata.add_arcs({
filename: {
arc: None
for arc in self.coverage_data.arcs(filename)}
for filename in self.files_to_propagate
})
else:
covdata.add_lines({
filename: {
line: None
for line in self.coverage_data.lines(filename)}
for filename in self.files_to_propagate
})
collector.save_data = original_save_data
collector.save_data = save_data
def evaluate_test_data(self, data):
if (
time.time() - self.start_time >= HUNG_TEST_TIME_LIMIT
):
fail_health_check(self.settings, (
'Your test has been running for at least five minutes. This '
'is probably not what you intended, so by default Hypothesis '
'turns it into an error.'
), HealthCheck.hung_test)
try:
if self.collector is None:
result = self.test_runner(data, reify_and_execute(
self.search_strategy, self.test,
))
else: # pragma: no cover
# This should always be a no-op, but the coverage tracer has
# a bad habit of resurrecting itself.
original = sys.gettrace()
sys.settrace(None)
try:
self.collector.data = {}
result = self.test_runner(data, reify_and_execute(
self.search_strategy, self.test,
collector=self.collector
))
finally:
sys.settrace(original)
covdata = CoverageData()
self.collector.save_data(covdata)
self.coverage_data.update(covdata)
for filename in covdata.measured_files():
if is_hypothesis_file(filename):
continue
data.tags.update(
arc(filename, source, target)
for source, target in covdata.arcs(filename)
)
if result is not None and self.settings.perform_health_check:
fail_health_check(self.settings, (
'Tests run under @given should return None, but '
'%s returned %r instead.'
) % (self.test.__name__, result), HealthCheck.return_value)
self.at_least_one_success = True
return False
except UnsatisfiedAssumption:
data.mark_invalid()
except (
HypothesisDeprecationWarning, FailedHealthCheck,
StopTest,
) + exceptions_to_reraise:
raise
except Exception as e:
escalate_hypothesis_internal_error()
data.__expected_traceback = traceback.format_exc()
data.__expected_exception = e
verbose_report(data.__expected_traceback)
error_class, _, tb = sys.exc_info()
origin = traceback.extract_tb(tb)[-1]
filename = origin[0]
lineno = origin[1]
data.mark_interesting((error_class, filename, lineno))
def run(self):
# Tell pytest to omit the body of this function from tracebacks
__tracebackhide__ = True
database_key = str_to_bytes(fully_qualified_name(self.test))
self.start_time = time.time()
global in_given
runner = ConjectureRunner(
self.evaluate_test_data,
settings=self.settings, random=self.random,
database_key=database_key,
)
if in_given or self.collector is None:
runner.run()
else: # pragma: no cover
in_given = True
original_trace = sys.gettrace()
try:
sys.settrace(None)
runner.run()
finally:
in_given = False
sys.settrace(original_trace)
note_engine_for_statistics(runner)
run_time = time.time() - self.start_time
timed_out = runner.exit_reason == ExitReason.timeout
if runner.last_data is None:
return
if runner.interesting_examples:
self.falsifying_examples = sorted(
[d for d in runner.interesting_examples.values()],
key=lambda d: sort_key(d.buffer), reverse=True
)
else:
if timed_out:
note_deprecation((
'Your tests are hitting the settings timeout (%.2fs). '
'This functionality will go away in a future release '
'and you should not rely on it. Instead, try setting '
'max_examples to be some value lower than %d (the number '
'of examples your test successfully ran here). Or, if you '
'would prefer your tests to run to completion, regardless '
'of how long they take, you can set the timeout value to '
'hypothesis.unlimited.'
) % (
self.settings.timeout, runner.valid_examples),
self.settings)
if runner.valid_examples < min(
self.settings.min_satisfying_examples,
self.settings.max_examples,
) and not (
runner.exit_reason == ExitReason.finished and
self.at_least_one_success
):
if timed_out:
raise Timeout((
'Ran out of time before finding a satisfying '
'example for '
'%s. Only found %d examples in ' +
'%.2fs.'
) % (
get_pretty_function_description(self.test),
runner.valid_examples, run_time
))
else:
raise Unsatisfiable((
'Unable to satisfy assumptions of hypothesis '
'%s. Only %d examples considered '
'satisfied assumptions'
) % (
get_pretty_function_description(self.test),
runner.valid_examples,))
if not self.falsifying_examples:
return
flaky = 0
self.__in_final_replay = True
for falsifying_example in self.falsifying_examples:
self.__was_flaky = False
raised_exception = False
try:
with self.settings:
self.test_runner(
ConjectureData.for_buffer(falsifying_example.buffer),
reify_and_execute(
self.search_strategy, self.test,
print_example=True, is_final=True
))
except (UnsatisfiedAssumption, StopTest):
report(traceback.format_exc())
self.__flaky(
'Unreliable assumption: An example which satisfied '
'assumptions on the first run now fails it.'
)
except:
if len(self.falsifying_examples) <= 1:
raise
raised_exception = True
report(traceback.format_exc())
if not raised_exception:
if (
isinstance(
falsifying_example.__expected_exception,
DeadlineExceeded
) and self.__test_runtime is not None
):
report((
'Unreliable test timings! On an initial run, this '
'test took %.2fms, which exceeded the deadline of '
'%.2fms, but on a subsequent run it took %.2f ms, '
'which did not. If you expect this sort of '
'variability in your test timings, consider turning '
'deadlines off for this test by setting deadline=None.'
) % (
falsifying_example.__expected_exception.runtime,
self.settings.deadline, self.__test_runtime
))
else:
report(
'Failed to reproduce exception. Expected: \n' +
falsifying_example.__expected_traceback,
)
filter_message = (
'Unreliable test data: Failed to reproduce a failure '
'and then when it came to recreating the example in '
'order to print the test data with a flaky result '
'the example was filtered out (by e.g. a '
'call to filter in your strategy) when we didn\'t '
'expect it to be.'
)
try:
self.test_runner(
ConjectureData.for_buffer(falsifying_example.buffer),
reify_and_execute(
self.search_strategy,
test_is_flaky(
self.test, self.repr_for_last_exception),
print_example=True, is_final=True
))
except (UnsatisfiedAssumption, StopTest):
self.__flaky(filter_message)
except Flaky as e:
if len(self.falsifying_examples) > 1:
self.__flaky(e.args[0])
else:
raise
if self.__was_flaky:
flaky += 1
# If we only have one example then we should have raised an error or
# flaky prior to this point.
assert len(self.falsifying_examples) > 1
if flaky > 0:
raise Flaky((
'Hypothesis found %d distinct failures, but %d of them '
'exhibited some sort of flaky behaviour.') % (
len(self.falsifying_examples), flaky))
else:
raise MultipleFailures((
'Hypothesis found %d distinct failures.') % (
len(self.falsifying_examples,)))
def __flaky(self, message):
if len(self.falsifying_examples) <= 1:
raise Flaky(message)
else:
self.__was_flaky = True
report('Flaky example! ' + message)
def evaluate_test_data(self, data):
if (time.time() - self.start_time >= HUNG_TEST_TIME_LIMIT):
fail_health_check(
self.settings,
('Your test has been running for at least five minutes. This '
'is probably not what you intended, so by default Hypothesis '
'turns it into an error.'), HealthCheck.hung_test)
try:
if self.collector is None:
result = self.test_runner(
data, reify_and_execute(
self.search_strategy,
self.test,
))
else: # pragma: no cover
# This should always be a no-op, but the coverage tracer has
# a bad habit of resurrecting itself.
original = sys.gettrace()
sys.settrace(None)
try:
try:
self.collector.data = {}
self.collector.start()
result = self.test_runner(
data,
reify_and_execute(
self.search_strategy,
self.test,
))
finally:
self.collector.stop()
finally:
sys.settrace(original)
covdata = CoverageData()
self.collector.save_data(covdata)
self.coverage_data.update(covdata)
for filename in covdata.measured_files():
if is_hypothesis_file(filename):
continue
for lineno in covdata.lines(filename):
data.add_tag(Line(filename, lineno))
for source, target in covdata.arcs(filename):
data.add_tag(Arc(filename, source, target))
if result is not None and self.settings.perform_health_check:
fail_health_check(
self.settings,
('Tests run under @given should return None, but '
'%s returned %r instead.') % (self.test.__name__, result),
HealthCheck.return_value)
self.at_least_one_success = True
return False
except UnsatisfiedAssumption:
data.mark_invalid()
except (
HypothesisDeprecationWarning,
FailedHealthCheck,
StopTest,
) + exceptions_to_reraise:
raise
except Exception:
escalate_hypothesis_internal_error()
data.__expected_exception = traceback.format_exc()
verbose_report(data.__expected_exception)
error_class, _, tb = sys.exc_info()
origin = traceback.extract_tb(tb)[-1]
filename = origin[0]
lineno = origin[1]
data.mark_interesting((error_class, filename, lineno))
def evaluate_test_data(self, data):
if (
time.time() - self.start_time >= HUNG_TEST_TIME_LIMIT
):
fail_health_check(self.settings, (
'Your test has been running for at least five minutes. This '
'is probably not what you intended, so by default Hypothesis '
'turns it into an error.'
), HealthCheck.hung_test)
try:
if self.collector is None:
result = self.test_runner(data, reify_and_execute(
self.search_strategy, self.test,
))
else: # pragma: no cover
# This should always be a no-op, but the coverage tracer has
# a bad habit of resurrecting itself.
original = sys.gettrace()
sys.settrace(None)
try:
try:
self.collector.data = {}
self.collector.start()
result = self.test_runner(data, reify_and_execute(
self.search_strategy, self.test,
))
finally:
self.collector.stop()
finally:
sys.settrace(original)
covdata = CoverageData()
self.collector.save_data(covdata)
self.coverage_data.update(covdata)
for filename in covdata.measured_files():
if is_hypothesis_file(filename):
continue
for lineno in covdata.lines(filename):
data.add_tag(Line(filename, lineno))
for source, target in covdata.arcs(filename):
data.add_tag(Arc(filename, source, target))
if result is not None and self.settings.perform_health_check:
fail_health_check(self.settings, (
'Tests run under @given should return None, but '
'%s returned %r instead.'
) % (self.test.__name__, result), HealthCheck.return_value)
self.at_least_one_success = True
return False
except UnsatisfiedAssumption:
data.mark_invalid()
except (
HypothesisDeprecationWarning, FailedHealthCheck,
StopTest,
) + exceptions_to_reraise:
raise
except Exception as e:
escalate_hypothesis_internal_error()
data.__expected_traceback = traceback.format_exc()
data.__expected_exception = e
verbose_report(data.__expected_traceback)
error_class, _, tb = sys.exc_info()
origin = traceback.extract_tb(tb)[-1]
filename = origin[0]
lineno = origin[1]
data.mark_interesting((error_class, filename, lineno))
