class StateForActualGivenExecution(object): def __init__(self, test_runner, search_strategy, test, settings, random): self.test_runner = test_runner self.search_strategy = search_strategy self.settings = settings self.at_least_one_success = False self.last_exception = None self.repr_for_last_exception = None self.falsifying_examples = () self.__was_flaky = False self.random = random self.__warned_deadline = False self.__existing_collector = None self.__test_runtime = None self.__in_final_replay = False if self.settings.deadline is None: self.test = test else: @proxies(test) def timed_test(*args, **kwargs): self.__test_runtime = None start = time.time() result = test(*args, **kwargs) runtime = (time.time() - start) * 1000 self.__test_runtime = runtime if self.settings.deadline is not_set: if (not self.__warned_deadline and runtime >= 200): self.__warned_deadline = True note_deprecation( ('Test took %.2fms to run. In future the default ' 'deadline setting will be 200ms, which will ' 'make this an error. You can set deadline to ' 'an explicit value of e.g. %d to turn tests ' 'slower than this into an error, or you can set ' 'it to None to disable this check entirely.') % ( runtime, ceil(runtime / 100) * 100, )) elif runtime >= self.current_deadline: raise DeadlineExceeded(runtime, self.settings.deadline) return result self.test = timed_test self.coverage_data = CoverageData() self.files_to_propagate = set() if settings.use_coverage and not IN_COVERAGE_TESTS: # pragma: no cover if Collector._collectors: self.hijack_collector(Collector._collectors[-1]) self.collector = Collector( branch=True, timid=FORCE_PURE_TRACER, should_trace=self.should_trace, check_include=hypothesis_check_include, concurrency='thread', warn=escalate_warning, ) self.collector.reset() # Hide the other collectors from this one so it doesn't attempt to # pause them (we're doing trace function management ourselves so # this will just cause problems). self.collector._collectors = [] else: self.collector = None @property def current_deadline(self): base = self.settings.deadline if self.__in_final_replay: return base else: return base * 1.25 def should_trace(self, original_filename, frame): # pragma: no cover disp = FileDisposition() assert original_filename is not None disp.original_filename = original_filename disp.canonical_filename = encoded_filepath( canonical_filename(original_filename)) disp.source_filename = disp.canonical_filename disp.reason = '' disp.file_tracer = None disp.has_dynamic_filename = False disp.trace = hypothesis_check_include(disp.canonical_filename) if not disp.trace: disp.reason = 'hypothesis internal reasons' elif self.__existing_collector is not None: check = self.__existing_collector.should_trace( original_filename, frame) if check.trace: self.files_to_propagate.add(check.canonical_filename) return disp def hijack_collector(self, collector): # pragma: no cover self.__existing_collector = collector original_save_data = collector.save_data def save_data(covdata): original_save_data(covdata) if collector.branch: covdata.add_arcs({ filename: {arc: None for arc in self.coverage_data.arcs(filename)} for filename in self.files_to_propagate }) else: covdata.add_lines({ filename: { line: None for line in self.coverage_data.lines(filename) } for filename in self.files_to_propagate }) collector.save_data = original_save_data collector.save_data = save_data def evaluate_test_data(self, data): if (time.time() - self.start_time >= HUNG_TEST_TIME_LIMIT): fail_health_check( self.settings, ('Your test has been running for at least five minutes. This ' 'is probably not what you intended, so by default Hypothesis ' 'turns it into an error.'), HealthCheck.hung_test) try: if self.collector is None: result = self.test_runner( data, reify_and_execute( self.search_strategy, self.test, )) else: # pragma: no cover # This should always be a no-op, but the coverage tracer has # a bad habit of resurrecting itself. original = sys.gettrace() sys.settrace(None) try: self.collector.data = {} result = self.test_runner( data, reify_and_execute(self.search_strategy, self.test, collector=self.collector)) finally: sys.settrace(original) covdata = CoverageData() self.collector.save_data(covdata) self.coverage_data.update(covdata) for filename in covdata.measured_files(): if is_hypothesis_file(filename): continue data.tags.update( arc(filename, source, target) for source, target in covdata.arcs(filename)) if result is not None and self.settings.perform_health_check: fail_health_check( self.settings, ('Tests run under @given should return None, but ' '%s returned %r instead.') % (self.test.__name__, result), HealthCheck.return_value) self.at_least_one_success = True return False except UnsatisfiedAssumption: data.mark_invalid() except ( HypothesisDeprecationWarning, FailedHealthCheck, StopTest, ) + exceptions_to_reraise: raise except Exception as e: escalate_hypothesis_internal_error() data.__expected_traceback = traceback.format_exc() data.__expected_exception = e verbose_report(data.__expected_traceback) error_class, _, tb = sys.exc_info() origin = traceback.extract_tb(tb)[-1] filename = origin[0] lineno = origin[1] data.mark_interesting((error_class, filename, lineno)) def run(self): # Tell pytest to omit the body of this function from tracebacks __tracebackhide__ = True database_key = str_to_bytes(fully_qualified_name(self.test)) self.start_time = time.time() global in_given runner = ConjectureRunner( self.evaluate_test_data, settings=self.settings, random=self.random, database_key=database_key, ) if in_given or self.collector is None: runner.run() else: # pragma: no cover in_given = True original_trace = sys.gettrace() try: sys.settrace(None) runner.run() finally: in_given = False sys.settrace(original_trace) note_engine_for_statistics(runner) run_time = time.time() - self.start_time timed_out = runner.exit_reason == ExitReason.timeout if runner.last_data is None: return if runner.interesting_examples: self.falsifying_examples = sorted( [d for d in runner.interesting_examples.values()], key=lambda d: sort_key(d.buffer), reverse=True) else: if timed_out: note_deprecation(( 'Your tests are hitting the settings timeout (%.2fs). ' 'This functionality will go away in a future release ' 'and you should not rely on it. Instead, try setting ' 'max_examples to be some value lower than %d (the number ' 'of examples your test successfully ran here). Or, if you ' 'would prefer your tests to run to completion, regardless ' 'of how long they take, you can set the timeout value to ' 'hypothesis.unlimited.') % (self.settings.timeout, runner.valid_examples), self.settings) if runner.valid_examples < min( self.settings.min_satisfying_examples, self.settings.max_examples, ) and not (runner.exit_reason == ExitReason.finished and self.at_least_one_success): if timed_out: raise Timeout( ('Ran out of time before finding a satisfying ' 'example for ' '%s. Only found %d examples in ' + '%.2fs.') % (get_pretty_function_description( self.test), runner.valid_examples, run_time)) else: raise Unsatisfiable( ('Unable to satisfy assumptions of hypothesis ' '%s. Only %d examples considered ' 'satisfied assumptions') % ( get_pretty_function_description(self.test), runner.valid_examples, )) if not self.falsifying_examples: return flaky = 0 self.__in_final_replay = True for falsifying_example in self.falsifying_examples: self.__was_flaky = False raised_exception = False try: with self.settings: self.test_runner( ConjectureData.for_buffer(falsifying_example.buffer), reify_and_execute(self.search_strategy, self.test, print_example=True, is_final=True)) except (UnsatisfiedAssumption, StopTest): report(traceback.format_exc()) self.__flaky( 'Unreliable assumption: An example which satisfied ' 'assumptions on the first run now fails it.') except: if len(self.falsifying_examples) <= 1: raise raised_exception = True report(traceback.format_exc()) if not raised_exception: if (isinstance(falsifying_example.__expected_exception, DeadlineExceeded) and self.__test_runtime is not None): report(( 'Unreliable test timings! On an initial run, this ' 'test took %.2fms, which exceeded the deadline of ' '%.2fms, but on a subsequent run it took %.2f ms, ' 'which did not. If you expect this sort of ' 'variability in your test timings, consider turning ' 'deadlines off for this test by setting deadline=None.' ) % (falsifying_example.__expected_exception.runtime, self.settings.deadline, self.__test_runtime)) else: report( 'Failed to reproduce exception. Expected: \n' + falsifying_example.__expected_traceback, ) filter_message = ( 'Unreliable test data: Failed to reproduce a failure ' 'and then when it came to recreating the example in ' 'order to print the test data with a flaky result ' 'the example was filtered out (by e.g. a ' 'call to filter in your strategy) when we didn\'t ' 'expect it to be.') try: self.test_runner( ConjectureData.for_buffer(falsifying_example.buffer), reify_and_execute(self.search_strategy, test_is_flaky( self.test, self.repr_for_last_exception), print_example=True, is_final=True)) except (UnsatisfiedAssumption, StopTest): self.__flaky(filter_message) except Flaky as e: if len(self.falsifying_examples) > 1: self.__flaky(e.args[0]) else: raise if self.__was_flaky: flaky += 1 # If we only have one example then we should have raised an error or # flaky prior to this point. assert len(self.falsifying_examples) > 1 if flaky > 0: raise Flaky( ('Hypothesis found %d distinct failures, but %d of them ' 'exhibited some sort of flaky behaviour.') % (len(self.falsifying_examples), flaky)) else: raise MultipleFailures(('Hypothesis found %d distinct failures.') % (len(self.falsifying_examples, ))) def __flaky(self, message): if len(self.falsifying_examples) <= 1: raise Flaky(message) else: self.__was_flaky = True report('Flaky example! ' + message)
class StateForActualGivenExecution(object): def __init__(self, test_runner, search_strategy, test, settings, random, had_seed): self.test_runner = test_runner self.search_strategy = search_strategy self.settings = settings self.last_exception = None self.falsifying_examples = () self.__was_flaky = False self.random = random self.__warned_deadline = False self.__existing_collector = None self.__test_runtime = None self.__had_seed = had_seed self.test = test self.coverage_data = CoverageData() self.files_to_propagate = set() self.failed_normally = False self.used_examples_from_database = False if settings.use_coverage and not IN_COVERAGE_TESTS: # pragma: no cover if Collector._collectors: parent = Collector._collectors[-1] # We include any files the collector has already decided to # trace whether or not on re-investigation we still think it # wants to trace them. The reason for this is that in some # cases coverage gets the wrong answer when we run it # ourselves due to reasons that are our fault but are hard to # fix (we lie about where certain functions come from). # This causes us to not record the actual test bodies as # covered. But if we intended to trace test bodies then the # file must already have been traced when getting to this point # and so will already be in the collector's data. Hence we can # use that information to get the correct answer here. # See issue 997 for more context. self.files_to_propagate = set(parent.data) self.hijack_collector(parent) self.collector = Collector( branch=True, timid=FORCE_PURE_TRACER, should_trace=self.should_trace, check_include=hypothesis_check_include, concurrency='thread', warn=escalate_warning, ) self.collector.reset() # Hide the other collectors from this one so it doesn't attempt to # pause them (we're doing trace function management ourselves so # this will just cause problems). self.collector._collectors = [] else: self.collector = None def execute( self, data, print_example=False, is_final=False, expected_failure=None, collect=False, ): text_repr = [None] if self.settings.deadline is None: test = self.test else: @proxies(self.test) def test(*args, **kwargs): self.__test_runtime = None initial_draws = len(data.draw_times) start = benchmark_time() result = self.test(*args, **kwargs) finish = benchmark_time() internal_draw_time = sum(data.draw_times[initial_draws:]) runtime = (finish - start - internal_draw_time) * 1000 self.__test_runtime = runtime if self.settings.deadline is not_set: if (not self.__warned_deadline and runtime >= 200): self.__warned_deadline = True note_deprecation( ('Test took %.2fms to run. In future the default ' 'deadline setting will be 200ms, which will ' 'make this an error. You can set deadline to ' 'an explicit value of e.g. %d to turn tests ' 'slower than this into an error, or you can set ' 'it to None to disable this check entirely.') % ( runtime, ceil(runtime / 100) * 100, )) else: current_deadline = self.settings.deadline if not is_final: current_deadline *= 1.25 if runtime >= current_deadline: raise DeadlineExceeded(runtime, self.settings.deadline) return result def run(data): if not hasattr(data, 'can_reproduce_example_from_repr'): data.can_reproduce_example_from_repr = True with self.settings: with BuildContext(data, is_final=is_final): with deterministic_PRNG(): args, kwargs = data.draw(self.search_strategy) if expected_failure is not None: text_repr[0] = arg_string(test, args, kwargs) if print_example: example = '%s(%s)' % (test.__name__, arg_string(test, args, kwargs)) try: ast.parse(example) except SyntaxError: data.can_reproduce_example_from_repr = False report('Falsifying example: %s' % (example, )) elif current_verbosity() >= Verbosity.verbose: report(lambda: 'Trying example: %s(%s)' % (test.__name__, arg_string(test, args, kwargs))) if self.collector is None or not collect: with deterministic_PRNG(): return test(*args, **kwargs) else: # pragma: no cover try: self.collector.start() with deterministic_PRNG(): return test(*args, **kwargs) finally: self.collector.stop() result = self.test_runner(data, run) if expected_failure is not None: exception, traceback = expected_failure if (isinstance(exception, DeadlineExceeded) and self.__test_runtime is not None): report( ('Unreliable test timings! On an initial run, this ' 'test took %.2fms, which exceeded the deadline of ' '%.2fms, but on a subsequent run it took %.2f ms, ' 'which did not. If you expect this sort of ' 'variability in your test timings, consider turning ' 'deadlines off for this test by setting deadline=None.') % (exception.runtime, self.settings.deadline, self.__test_runtime)) else: report( 'Failed to reproduce exception. Expected: \n' + traceback, ) self.__flaky( ('Hypothesis %s(%s) produces unreliable results: Falsified' ' on the first call but did not on a subsequent one') % ( test.__name__, text_repr[0], )) return result def should_trace(self, original_filename, frame): # pragma: no cover disp = FileDisposition() assert original_filename is not None disp.original_filename = original_filename disp.canonical_filename = encoded_filepath( canonical_filename(original_filename)) disp.source_filename = disp.canonical_filename disp.reason = '' disp.file_tracer = None disp.has_dynamic_filename = False disp.trace = hypothesis_check_include(disp.canonical_filename) if not disp.trace: disp.reason = 'hypothesis internal reasons' elif self.__existing_collector is not None: check = self.__existing_collector.should_trace( original_filename, frame) if check.trace: self.files_to_propagate.add(check.canonical_filename) return disp def hijack_collector(self, collector): # pragma: no cover self.__existing_collector = collector original_save_data = collector.save_data def save_data(covdata): original_save_data(covdata) if collector.branch: covdata.add_arcs({ filename: { arc: None for arc in self.coverage_data.arcs(filename) or () } for filename in self.files_to_propagate }) else: covdata.add_lines({ filename: { line: None for line in self.coverage_data.lines(filename) or () } for filename in self.files_to_propagate }) collector.save_data = original_save_data collector.save_data = save_data def evaluate_test_data(self, data): try: if self.collector is None: result = self.execute(data) else: # pragma: no cover # This should always be a no-op, but the coverage tracer has # a bad habit of resurrecting itself. original = sys.gettrace() sys.settrace(None) try: self.collector.data = {} result = self.execute(data, collect=True) finally: sys.settrace(original) covdata = CoverageData() self.collector.save_data(covdata) self.coverage_data.update(covdata) for filename in covdata.measured_files(): if is_hypothesis_file(filename): continue data.tags.update( arc(filename, source, target) for source, target in covdata.arcs(filename)) if result is not None: fail_health_check( self.settings, ('Tests run under @given should return None, but ' '%s returned %r instead.') % (self.test.__name__, result), HealthCheck.return_value) return False except UnsatisfiedAssumption: data.mark_invalid() except ( HypothesisDeprecationWarning, FailedHealthCheck, StopTest, ) + exceptions_to_reraise: raise except Exception as e: escalate_hypothesis_internal_error() data.__expected_traceback = traceback.format_exc() data.__expected_exception = e verbose_report(data.__expected_traceback) error_class, _, tb = sys.exc_info() origin = traceback.extract_tb(tb)[-1] filename = origin[0] lineno = origin[1] data.mark_interesting((error_class, filename, lineno)) def run(self): # Tell pytest to omit the body of this function from tracebacks __tracebackhide__ = True if global_force_seed is None: database_key = str_to_bytes(fully_qualified_name(self.test)) else: database_key = None self.start_time = time.time() global in_given runner = ConjectureRunner( self.evaluate_test_data, settings=self.settings, random=self.random, database_key=database_key, ) if in_given or self.collector is None: runner.run() else: # pragma: no cover in_given = True original_trace = sys.gettrace() try: sys.settrace(None) runner.run() finally: in_given = False sys.settrace(original_trace) self.used_examples_from_database = \ runner.used_examples_from_database note_engine_for_statistics(runner) run_time = time.time() - self.start_time self.used_examples_from_database = runner.used_examples_from_database if runner.used_examples_from_database: if self.settings.derandomize: note_deprecation( 'In future derandomize will imply database=None, but your ' 'test is currently using examples from the database. To ' 'get the future behaviour, update your settings to ' 'include database=None.') if self.__had_seed: note_deprecation( 'In future use of @seed will imply database=None in your ' 'settings, but your test is currently using examples from ' 'the database. To get the future behaviour, update your ' 'settings for this test to include database=None.') timed_out = runner.exit_reason == ExitReason.timeout if runner.call_count == 0: return if runner.interesting_examples: self.falsifying_examples = sorted( [d for d in runner.interesting_examples.values()], key=lambda d: sort_key(d.buffer), reverse=True) else: if timed_out: note_deprecation(( 'Your tests are hitting the settings timeout (%.2fs). ' 'This functionality will go away in a future release ' 'and you should not rely on it. Instead, try setting ' 'max_examples to be some value lower than %d (the number ' 'of examples your test successfully ran here). Or, if you ' 'would prefer your tests to run to completion, regardless ' 'of how long they take, you can set the timeout value to ' 'hypothesis.unlimited.') % (self.settings.timeout, runner.valid_examples), self.settings) if runner.valid_examples == 0: if timed_out: raise Timeout( ('Ran out of time before finding a satisfying ' 'example for %s. Only found %d examples in %.2fs.') % (get_pretty_function_description( self.test), runner.valid_examples, run_time)) else: raise Unsatisfiable( 'Unable to satisfy assumptions of hypothesis %s.' % (get_pretty_function_description(self.test), )) if not self.falsifying_examples: return self.failed_normally = True flaky = 0 for falsifying_example in self.falsifying_examples: ran_example = ConjectureData.for_buffer(falsifying_example.buffer) self.__was_flaky = False assert falsifying_example.__expected_exception is not None try: self.execute(ran_example, print_example=True, is_final=True, expected_failure=( falsifying_example.__expected_exception, falsifying_example.__expected_traceback, )) except (UnsatisfiedAssumption, StopTest): report(traceback.format_exc()) self.__flaky( 'Unreliable assumption: An example which satisfied ' 'assumptions on the first run now fails it.') except BaseException: if len(self.falsifying_examples) <= 1: raise report(traceback.format_exc()) finally: # pragma: no cover # This section is in fact entirely covered by the tests in # test_reproduce_failure, but it seems to trigger a lovely set # of coverage bugs: The branches show up as uncovered (despite # definitely being covered - you can add an assert False else # branch to verify this and see it fail - and additionally the # second branch still complains about lack of coverage even if # you add a pragma: no cover to it! # See https://bitbucket.org/ned/coveragepy/issues/623/ if self.settings.print_blob is not PrintSettings.NEVER: failure_blob = encode_failure(falsifying_example.buffer) # Have to use the example we actually ran, not the original # falsifying example! Otherwise we won't catch problems # where the repr of the generated example doesn't parse. can_use_repr = ran_example.can_reproduce_example_from_repr if (self.settings.print_blob is PrintSettings.ALWAYS or (self.settings.print_blob is PrintSettings.INFER and not can_use_repr and len(failure_blob) < 200)): report(( '\n' 'You can reproduce this example by temporarily ' 'adding @reproduce_failure(%r, %r) as a decorator ' 'on your test case') % ( __version__, failure_blob, )) if self.__was_flaky: flaky += 1 # If we only have one example then we should have raised an error or # flaky prior to this point. assert len(self.falsifying_examples) > 1 if flaky > 0: raise Flaky( ('Hypothesis found %d distinct failures, but %d of them ' 'exhibited some sort of flaky behaviour.') % (len(self.falsifying_examples), flaky)) else: raise MultipleFailures(('Hypothesis found %d distinct failures.') % (len(self.falsifying_examples, ))) def __flaky(self, message): if len(self.falsifying_examples) <= 1: raise Flaky(message) else: self.__was_flaky = True report('Flaky example! ' + message)
class StateForActualGivenExecution(object): def __init__(self, test_runner, search_strategy, test, settings, random): self.test_runner = test_runner self.search_strategy = search_strategy self.settings = settings self.at_least_one_success = False self.last_exception = None self.repr_for_last_exception = None self.falsifying_examples = () self.__was_flaky = False self.random = random self.__warned_deadline = False self.__existing_collector = None self.__test_runtime = None self.__in_final_replay = False if self.settings.deadline is None: self.test = test else: @proxies(test) def timed_test(*args, **kwargs): self.__test_runtime = None start = time.time() result = test(*args, **kwargs) runtime = (time.time() - start) * 1000 self.__test_runtime = runtime if self.settings.deadline is not_set: if ( not self.__warned_deadline and runtime >= 200 ): self.__warned_deadline = True note_deprecation(( 'Test took %.2fms to run. In future the default ' 'deadline setting will be 200ms, which will ' 'make this an error. You can set deadline to ' 'an explicit value of e.g. %d to turn tests ' 'slower than this into an error, or you can set ' 'it to None to disable this check entirely.') % ( runtime, ceil(runtime / 100) * 100, )) elif runtime >= self.current_deadline: raise DeadlineExceeded(runtime, self.settings.deadline) return result self.test = timed_test self.coverage_data = CoverageData() self.files_to_propagate = set() if settings.use_coverage and not IN_COVERAGE_TESTS: # pragma: no cover if Collector._collectors: self.hijack_collector(Collector._collectors[-1]) self.collector = Collector( branch=True, timid=FORCE_PURE_TRACER, should_trace=self.should_trace, check_include=hypothesis_check_include, concurrency='thread', warn=escalate_warning, ) self.collector.reset() # Hide the other collectors from this one so it doesn't attempt to # pause them (we're doing trace function management ourselves so # this will just cause problems). self.collector._collectors = [] else: self.collector = None @property def current_deadline(self): base = self.settings.deadline if self.__in_final_replay: return base else: return base * 1.25 def should_trace(self, original_filename, frame): # pragma: no cover disp = FileDisposition() assert original_filename is not None disp.original_filename = original_filename disp.canonical_filename = encoded_filepath( canonical_filename(original_filename)) disp.source_filename = disp.canonical_filename disp.reason = '' disp.file_tracer = None disp.has_dynamic_filename = False disp.trace = hypothesis_check_include(disp.canonical_filename) if not disp.trace: disp.reason = 'hypothesis internal reasons' elif self.__existing_collector is not None: check = self.__existing_collector.should_trace( original_filename, frame) if check.trace: self.files_to_propagate.add(check.canonical_filename) return disp def hijack_collector(self, collector): # pragma: no cover self.__existing_collector = collector original_save_data = collector.save_data def save_data(covdata): original_save_data(covdata) if collector.branch: covdata.add_arcs({ filename: { arc: None for arc in self.coverage_data.arcs(filename)} for filename in self.files_to_propagate }) else: covdata.add_lines({ filename: { line: None for line in self.coverage_data.lines(filename)} for filename in self.files_to_propagate }) collector.save_data = original_save_data collector.save_data = save_data def evaluate_test_data(self, data): if ( time.time() - self.start_time >= HUNG_TEST_TIME_LIMIT ): fail_health_check(self.settings, ( 'Your test has been running for at least five minutes. This ' 'is probably not what you intended, so by default Hypothesis ' 'turns it into an error.' ), HealthCheck.hung_test) try: if self.collector is None: result = self.test_runner(data, reify_and_execute( self.search_strategy, self.test, )) else: # pragma: no cover # This should always be a no-op, but the coverage tracer has # a bad habit of resurrecting itself. original = sys.gettrace() sys.settrace(None) try: self.collector.data = {} result = self.test_runner(data, reify_and_execute( self.search_strategy, self.test, collector=self.collector )) finally: sys.settrace(original) covdata = CoverageData() self.collector.save_data(covdata) self.coverage_data.update(covdata) for filename in covdata.measured_files(): if is_hypothesis_file(filename): continue data.tags.update( arc(filename, source, target) for source, target in covdata.arcs(filename) ) if result is not None and self.settings.perform_health_check: fail_health_check(self.settings, ( 'Tests run under @given should return None, but ' '%s returned %r instead.' ) % (self.test.__name__, result), HealthCheck.return_value) self.at_least_one_success = True return False except UnsatisfiedAssumption: data.mark_invalid() except ( HypothesisDeprecationWarning, FailedHealthCheck, StopTest, ) + exceptions_to_reraise: raise except Exception as e: escalate_hypothesis_internal_error() data.__expected_traceback = traceback.format_exc() data.__expected_exception = e verbose_report(data.__expected_traceback) error_class, _, tb = sys.exc_info() origin = traceback.extract_tb(tb)[-1] filename = origin[0] lineno = origin[1] data.mark_interesting((error_class, filename, lineno)) def run(self): # Tell pytest to omit the body of this function from tracebacks __tracebackhide__ = True database_key = str_to_bytes(fully_qualified_name(self.test)) self.start_time = time.time() global in_given runner = ConjectureRunner( self.evaluate_test_data, settings=self.settings, random=self.random, database_key=database_key, ) if in_given or self.collector is None: runner.run() else: # pragma: no cover in_given = True original_trace = sys.gettrace() try: sys.settrace(None) runner.run() finally: in_given = False sys.settrace(original_trace) note_engine_for_statistics(runner) run_time = time.time() - self.start_time timed_out = runner.exit_reason == ExitReason.timeout if runner.last_data is None: return if runner.interesting_examples: self.falsifying_examples = sorted( [d for d in runner.interesting_examples.values()], key=lambda d: sort_key(d.buffer), reverse=True ) else: if timed_out: note_deprecation(( 'Your tests are hitting the settings timeout (%.2fs). ' 'This functionality will go away in a future release ' 'and you should not rely on it. Instead, try setting ' 'max_examples to be some value lower than %d (the number ' 'of examples your test successfully ran here). Or, if you ' 'would prefer your tests to run to completion, regardless ' 'of how long they take, you can set the timeout value to ' 'hypothesis.unlimited.' ) % ( self.settings.timeout, runner.valid_examples), self.settings) if runner.valid_examples < min( self.settings.min_satisfying_examples, self.settings.max_examples, ) and not ( runner.exit_reason == ExitReason.finished and self.at_least_one_success ): if timed_out: raise Timeout(( 'Ran out of time before finding a satisfying ' 'example for ' '%s. Only found %d examples in ' + '%.2fs.' ) % ( get_pretty_function_description(self.test), runner.valid_examples, run_time )) else: raise Unsatisfiable(( 'Unable to satisfy assumptions of hypothesis ' '%s. Only %d examples considered ' 'satisfied assumptions' ) % ( get_pretty_function_description(self.test), runner.valid_examples,)) if not self.falsifying_examples: return flaky = 0 self.__in_final_replay = True for falsifying_example in self.falsifying_examples: self.__was_flaky = False raised_exception = False try: with self.settings: self.test_runner( ConjectureData.for_buffer(falsifying_example.buffer), reify_and_execute( self.search_strategy, self.test, print_example=True, is_final=True )) except (UnsatisfiedAssumption, StopTest): report(traceback.format_exc()) self.__flaky( 'Unreliable assumption: An example which satisfied ' 'assumptions on the first run now fails it.' ) except: if len(self.falsifying_examples) <= 1: raise raised_exception = True report(traceback.format_exc()) if not raised_exception: if ( isinstance( falsifying_example.__expected_exception, DeadlineExceeded ) and self.__test_runtime is not None ): report(( 'Unreliable test timings! On an initial run, this ' 'test took %.2fms, which exceeded the deadline of ' '%.2fms, but on a subsequent run it took %.2f ms, ' 'which did not. If you expect this sort of ' 'variability in your test timings, consider turning ' 'deadlines off for this test by setting deadline=None.' ) % ( falsifying_example.__expected_exception.runtime, self.settings.deadline, self.__test_runtime )) else: report( 'Failed to reproduce exception. Expected: \n' + falsifying_example.__expected_traceback, ) filter_message = ( 'Unreliable test data: Failed to reproduce a failure ' 'and then when it came to recreating the example in ' 'order to print the test data with a flaky result ' 'the example was filtered out (by e.g. a ' 'call to filter in your strategy) when we didn\'t ' 'expect it to be.' ) try: self.test_runner( ConjectureData.for_buffer(falsifying_example.buffer), reify_and_execute( self.search_strategy, test_is_flaky( self.test, self.repr_for_last_exception), print_example=True, is_final=True )) except (UnsatisfiedAssumption, StopTest): self.__flaky(filter_message) except Flaky as e: if len(self.falsifying_examples) > 1: self.__flaky(e.args[0]) else: raise if self.__was_flaky: flaky += 1 # If we only have one example then we should have raised an error or # flaky prior to this point. assert len(self.falsifying_examples) > 1 if flaky > 0: raise Flaky(( 'Hypothesis found %d distinct failures, but %d of them ' 'exhibited some sort of flaky behaviour.') % ( len(self.falsifying_examples), flaky)) else: raise MultipleFailures(( 'Hypothesis found %d distinct failures.') % ( len(self.falsifying_examples,))) def __flaky(self, message): if len(self.falsifying_examples) <= 1: raise Flaky(message) else: self.__was_flaky = True report('Flaky example! ' + message)