def evaluate_test_data(self, data): try: result = self.execute(data) if result is not None: fail_health_check( self.settings, ('Tests run under @given should return None, but ' '%s returned %r instead.') % (self.test.__name__, result), HealthCheck.return_value) except UnsatisfiedAssumption: data.mark_invalid() except ( HypothesisDeprecationWarning, FailedHealthCheck, StopTest, ) + EXCEPTIONS_TO_RERAISE: raise except EXCEPTIONS_TO_FAIL as e: escalate_hypothesis_internal_error() tb = get_trimmed_traceback() data.__expected_traceback = ''.join( traceback.format_exception(type(e), e, tb)) data.__expected_exception = e verbose_report(data.__expected_traceback) origin = traceback.extract_tb(tb)[-1] filename = origin[0] lineno = origin[1] data.mark_interesting((type(e), filename, lineno))
def evaluate_test_data(self, data): if (time.time() - self.start_time >= HUNG_TEST_TIME_LIMIT): fail_health_check( self.settings, ('Your test has been running for at least five minutes. This ' 'is probably not what you intended, so by default Hypothesis ' 'turns it into an error.'), HealthCheck.hung_test) try: result = self.test_runner( data, reify_and_execute( self.search_strategy, self.test, )) if result is not None and self.settings.perform_health_check: fail_health_check( self.settings, ('Tests run under @given should return None, but ' '%s returned %r instead.') % (self.test.__name__, result), HealthCheck.return_value) self.at_least_one_success = True return False except UnsatisfiedAssumption: data.mark_invalid() except ( HypothesisDeprecationWarning, FailedHealthCheck, StopTest, ) + exceptions_to_reraise: raise except Exception: escalate_hypothesis_internal_error() self.last_exception = traceback.format_exc() verbose_report(self.last_exception) data.mark_interesting()
def evaluate_test_data(self, data): try: result = self.execute(data) if result is not None: fail_health_check(self.settings, ( 'Tests run under @given should return None, but ' '%s returned %r instead.' ) % (self.test.__name__, result), HealthCheck.return_value) return False except UnsatisfiedAssumption: data.mark_invalid() except ( HypothesisDeprecationWarning, FailedHealthCheck, StopTest, ) + EXCEPTIONS_TO_RERAISE: raise except Exception as e: escalate_hypothesis_internal_error() data.__expected_traceback = traceback.format_exc() data.__expected_exception = e verbose_report(data.__expected_traceback) error_class, _, tb = sys.exc_info() origin = traceback.extract_tb(tb)[-1] filename = origin[0] lineno = origin[1] data.mark_interesting((error_class, filename, lineno))
def evaluate_test_data(self, data): try: result = self.test_runner( data, reify_and_execute( self.search_strategy, self.test, )) if result is not None and self.settings.perform_health_check: fail_health_check( self.settings, ('Tests run under @given should return None, but ' '%s returned %r instead.') % (self.test.__name__, result), HealthCheck.return_value) self.at_least_one_success = True return False except UnsatisfiedAssumption: data.mark_invalid() except ( HypothesisDeprecationWarning, FailedHealthCheck, StopTest, ): raise except Exception: escalate_hypothesis_internal_error() self.last_exception = traceback.format_exc() verbose_report(self.last_exception) data.mark_interesting()
def test_does_not_escalate_errors_in_hypothesis_file_if_disabled(monkeypatch): monkeypatch.setattr(esc, 'is_hypothesis_file', lambda x: True) monkeypatch.setattr(esc, 'PREVENT_ESCALATION', True) try: assert False except AssertionError: esc.escalate_hypothesis_internal_error()
def test_does_escalate_errors_in_hypothesis_file(monkeypatch): monkeypatch.setattr(esc, 'is_hypothesis_file', lambda x: True) with pytest.raises(AssertionError): try: assert False except AssertionError: esc.escalate_hypothesis_internal_error()
def test_does_escalate_errors_in_hypothesis_file(monkeypatch): monkeypatch.setattr(esc, "is_hypothesis_file", lambda x: True) with pytest.raises(AssertionError): try: assert False except AssertionError: esc.escalate_hypothesis_internal_error()
def test_does_not_escalate_errors_in_hypothesis_file_if_disabled(monkeypatch): monkeypatch.setattr(esc, "is_hypothesis_file", lambda x: True) monkeypatch.setattr(esc, "PREVENT_ESCALATION", True) try: assert False except AssertionError: esc.escalate_hypothesis_internal_error()
def _execute_once_for_engine(self, data): """Wrapper around ``execute_once`` that intercepts test failure exceptions and single-test control exceptions, and turns them into appropriate method calls to `data` instead. This allows the engine to assume that any exception other than ``StopTest`` must be a fatal error, and should stop the entire engine. """ try: result = self.execute_once(data) if result is not None: fail_health_check( self.settings, ("Tests run under @given should return None, but " "%s returned %r instead.") % (self.test.__name__, result), HealthCheck.return_value, ) except UnsatisfiedAssumption: # An "assume" check failed, so instead we inform the engine that # this test run was invalid. data.mark_invalid() except StopTest: # The engine knows how to handle this control exception, so it's # OK to re-raise it. raise except ( HypothesisDeprecationWarning, FailedHealthCheck, ) + skip_exceptions_to_reraise(): # These are fatal errors or control exceptions that should stop the # engine, so we re-raise them. raise except failure_exceptions_to_catch() as e: # If the error was raised by Hypothesis-internal code, re-raise it # as a fatal error instead of treating it as a test failure. escalate_hypothesis_internal_error() if data.frozen: # This can happen if an error occurred in a finally # block somewhere, suppressing our original StopTest. # We raise a new one here to resume normal operation. raise StopTest(data.testcounter) else: # The test failed by raising an exception, so we inform the # engine that this test run was interesting. This is the normal # path for test runs that fail. tb = get_trimmed_traceback() info = data.extra_information info.__expected_traceback = "".join( traceback.format_exception(type(e), e, tb)) info.__expected_exception = e verbose_report(info.__expected_traceback) origin = traceback.extract_tb(tb)[-1] filename = origin[0] lineno = origin[1] data.mark_interesting((type(e), filename, lineno))
def evaluate_test_data(self, data): try: if self.collector is None: result = self.execute(data) else: # pragma: no cover # This should always be a no-op, but the coverage tracer has # a bad habit of resurrecting itself. original = sys.gettrace() sys.settrace(None) try: self.collector.data = {} result = self.execute(data, collect=True) finally: sys.settrace(original) covdata = CoverageData() self.collector.save_data(covdata) self.coverage_data.update(covdata) for filename in covdata.measured_files(): if is_hypothesis_file(filename): continue data.tags.update( arc(filename, source, target) for source, target in covdata.arcs(filename)) if result is not None and self.settings.perform_health_check: fail_health_check( self.settings, ('Tests run under @given should return None, but ' '%s returned %r instead.') % (self.test.__name__, result), HealthCheck.return_value) self.at_least_one_success = True return False except UnsatisfiedAssumption: data.mark_invalid() except ( HypothesisDeprecationWarning, FailedHealthCheck, StopTest, ) + exceptions_to_reraise: raise except Exception as e: escalate_hypothesis_internal_error() data.__expected_traceback = traceback.format_exc() data.__expected_exception = e verbose_report(data.__expected_traceback) error_class, _, tb = sys.exc_info() origin = traceback.extract_tb(tb)[-1] filename = origin[0] lineno = origin[1] data.mark_interesting((error_class, filename, lineno))
def evaluate_test_data(self, data): try: result = self.execute(data) if result is not None: fail_health_check( self.settings, ( "Tests run under @given should return None, but " "%s returned %r instead." ) % (self.test.__name__, result), HealthCheck.return_value, ) except UnsatisfiedAssumption: data.mark_invalid() except ( HypothesisDeprecationWarning, FailedHealthCheck, StopTest, ) + skip_exceptions_to_reraise(): raise except failure_exceptions_to_catch() as e: escalate_hypothesis_internal_error() if data.frozen: # This can happen if an error occurred in a finally # block somewhere, suppressing our original StopTest. # We raise a new one here to resume normal operation. raise StopTest(data.testcounter) else: tb = get_trimmed_traceback() info = data.extra_information info.__expected_traceback = "".join( traceback.format_exception(type(e), e, tb) ) info.__expected_exception = e verbose_report(info.__expected_traceback) origin = traceback.extract_tb(tb)[-1] filename = origin[0] lineno = origin[1] data.mark_interesting((type(e), filename, lineno))
def evaluate_test_data(self, data): try: result = self.execute(data) if result is not None: fail_health_check( self.settings, ( "Tests run under @given should return None, but " "%s returned %r instead." ) % (self.test.__name__, result), HealthCheck.return_value, ) except UnsatisfiedAssumption: data.mark_invalid() except ( HypothesisDeprecationWarning, FailedHealthCheck, StopTest, ) + skip_exceptions_to_reraise(): raise except failure_exceptions_to_catch() as e: escalate_hypothesis_internal_error() if data.frozen: # This can happen if an error occurred in a finally # block somewhere, suppressing our original StopTest. # We raise a new one here to resume normal operation. raise StopTest(data.testcounter) else: tb = get_trimmed_traceback() info = data.extra_information info.__expected_traceback = "".join( traceback.format_exception(type(e), e, tb) ) info.__expected_exception = e verbose_report(info.__expected_traceback) origin = traceback.extract_tb(tb)[-1] filename = origin[0] lineno = origin[1] data.mark_interesting((type(e), filename, lineno))
def evaluate_test_data(self, data): if (time.time() - self.start_time >= HUNG_TEST_TIME_LIMIT): fail_health_check( self.settings, ('Your test has been running for at least five minutes. This ' 'is probably not what you intended, so by default Hypothesis ' 'turns it into an error.'), HealthCheck.hung_test) try: if self.collector is None: result = self.test_runner( data, reify_and_execute( self.search_strategy, self.test, )) else: # pragma: no cover # This should always be a no-op, but the coverage tracer has # a bad habit of resurrecting itself. original = sys.gettrace() sys.settrace(None) try: try: self.collector.data = {} self.collector.start() result = self.test_runner( data, reify_and_execute( self.search_strategy, self.test, )) finally: self.collector.stop() finally: sys.settrace(original) covdata = CoverageData() self.collector.save_data(covdata) self.coverage_data.update(covdata) for filename in covdata.measured_files(): if is_hypothesis_file(filename): continue for lineno in covdata.lines(filename): data.add_tag(Line(filename, lineno)) for source, target in covdata.arcs(filename): data.add_tag(Arc(filename, source, target)) if result is not None and self.settings.perform_health_check: fail_health_check( self.settings, ('Tests run under @given should return None, but ' '%s returned %r instead.') % (self.test.__name__, result), HealthCheck.return_value) self.at_least_one_success = True return False except UnsatisfiedAssumption: data.mark_invalid() except ( HypothesisDeprecationWarning, FailedHealthCheck, StopTest, ) + exceptions_to_reraise: raise except Exception: escalate_hypothesis_internal_error() data.__expected_exception = traceback.format_exc() verbose_report(data.__expected_exception) error_class, _, tb = sys.exc_info() origin = traceback.extract_tb(tb)[-1] filename = origin[0] lineno = origin[1] data.mark_interesting((error_class, filename, lineno))
def perform_health_checks(random, settings, test_runner, search_strategy): # Tell pytest to omit the body of this function from tracebacks __tracebackhide__ = True if not settings.perform_health_check: return if not Settings.default.perform_health_check: return health_check_random = Random(random.getrandbits(128)) # We "pre warm" the health check with one draw to give it some # time to calculate any cached data. This prevents the case # where the first draw of the health check takes ages because # of loading unicode data the first time. data = ConjectureData( max_length=settings.buffer_size, draw_bytes=lambda data, n: uniform(health_check_random, n) ) with Settings(settings, verbosity=Verbosity.quiet): try: test_runner(data, reify_and_execute( search_strategy, lambda *args, **kwargs: None, )) except BaseException: pass count = 0 overruns = 0 filtered_draws = 0 start = time.time() while ( count < 10 and time.time() < start + 1 and filtered_draws < 50 and overruns < 20 ): try: data = ConjectureData( max_length=settings.buffer_size, draw_bytes=lambda data, n: uniform(health_check_random, n) ) with Settings(settings, verbosity=Verbosity.quiet): test_runner(data, reify_and_execute( search_strategy, lambda *args, **kwargs: None, )) count += 1 except UnsatisfiedAssumption: filtered_draws += 1 except StopTest: if data.status == Status.INVALID: filtered_draws += 1 else: assert data.status == Status.OVERRUN overruns += 1 except InvalidArgument: raise except Exception: escalate_hypothesis_internal_error() if ( HealthCheck.exception_in_generation in settings.suppress_health_check ): raise report(traceback.format_exc()) if test_runner is default_new_style_executor: fail_health_check( settings, 'An exception occurred during data ' 'generation in initial health check. ' 'This indicates a bug in the strategy. ' 'This could either be a Hypothesis bug or ' "an error in a function you've passed to " 'it to construct your data.', HealthCheck.exception_in_generation, ) else: fail_health_check( settings, 'An exception occurred during data ' 'generation in initial health check. ' 'This indicates a bug in the strategy. ' 'This could either be a Hypothesis bug or ' 'an error in a function you\'ve passed to ' 'it to construct your data. Additionally, ' 'you have a custom executor, which means ' 'that this could be your executor failing ' 'to handle a function which returns None. ', HealthCheck.exception_in_generation, ) if overruns >= 20 or ( not count and overruns > 0 ): fail_health_check(settings, ( 'Examples routinely exceeded the max allowable size. ' '(%d examples overran while generating %d valid ones)' '. Generating examples this large will usually lead to' ' bad results. You should try setting average_size or ' 'max_size parameters on your collections and turning ' 'max_leaves down on recursive() calls.') % ( overruns, count ), HealthCheck.data_too_large) if filtered_draws >= 50 or ( not count and filtered_draws > 0 ): fail_health_check(settings, ( 'It looks like your strategy is filtering out a lot ' 'of data. Health check found %d filtered examples but ' 'only %d good ones. This will make your tests much ' 'slower, and also will probably distort the data ' 'generation quite a lot. You should adapt your ' 'strategy to filter less. This can also be caused by ' 'a low max_leaves parameter in recursive() calls') % ( filtered_draws, count ), HealthCheck.filter_too_much) runtime = time.time() - start if runtime > 1.0 or count < 10: fail_health_check(settings, ( 'Data generation is extremely slow: Only produced ' '%d valid examples in %.2f seconds (%d invalid ones ' 'and %d exceeded maximum size). Try decreasing ' "size of the data you're generating (with e.g." 'average_size or max_leaves parameters).' ) % (count, runtime, filtered_draws, overruns), HealthCheck.too_slow, )
def test_does_not_escalate_errors_in_non_hypothesis_file(): try: assert False except AssertionError: esc.escalate_hypothesis_internal_error()
def evaluate_test_data(self, data): if ( time.time() - self.start_time >= HUNG_TEST_TIME_LIMIT ): fail_health_check(self.settings, ( 'Your test has been running for at least five minutes. This ' 'is probably not what you intended, so by default Hypothesis ' 'turns it into an error.' ), HealthCheck.hung_test) try: if self.collector is None: result = self.test_runner(data, reify_and_execute( self.search_strategy, self.test, )) else: # pragma: no cover # This should always be a no-op, but the coverage tracer has # a bad habit of resurrecting itself. original = sys.gettrace() sys.settrace(None) try: self.collector.data = {} result = self.test_runner(data, reify_and_execute( self.search_strategy, self.test, collector=self.collector )) finally: sys.settrace(original) covdata = CoverageData() self.collector.save_data(covdata) self.coverage_data.update(covdata) for filename in covdata.measured_files(): if is_hypothesis_file(filename): continue data.tags.update( arc(filename, source, target) for source, target in covdata.arcs(filename) ) if result is not None and self.settings.perform_health_check: fail_health_check(self.settings, ( 'Tests run under @given should return None, but ' '%s returned %r instead.' ) % (self.test.__name__, result), HealthCheck.return_value) self.at_least_one_success = True return False except UnsatisfiedAssumption: data.mark_invalid() except ( HypothesisDeprecationWarning, FailedHealthCheck, StopTest, ) + exceptions_to_reraise: raise except Exception as e: escalate_hypothesis_internal_error() data.__expected_traceback = traceback.format_exc() data.__expected_exception = e verbose_report(data.__expected_traceback) error_class, _, tb = sys.exc_info() origin = traceback.extract_tb(tb)[-1] filename = origin[0] lineno = origin[1] data.mark_interesting((error_class, filename, lineno))
def test_does_not_escalate_errors_in_non_hypothesis_file(): try: assert False except AssertionError: esc.escalate_hypothesis_internal_error()
def perform_health_checks(random, settings, test_runner, search_strategy): if not settings.perform_health_check: return if not Settings.default.perform_health_check: return health_check_random = Random(random.getrandbits(128)) # We "pre warm" the health check with one draw to give it some # time to calculate any cached data. This prevents the case # where the first draw of the health check takes ages because # of loading unicode data the first time. data = ConjectureData(max_length=settings.buffer_size, draw_bytes=lambda data, n, distribution: distribution(health_check_random, n)) with Settings(settings, verbosity=Verbosity.quiet): try: test_runner( data, reify_and_execute( search_strategy, lambda *args, **kwargs: None, )) except BaseException: pass count = 0 overruns = 0 filtered_draws = 0 start = time.time() while (count < 10 and time.time() < start + 1 and filtered_draws < 50 and overruns < 20): try: data = ConjectureData(max_length=settings.buffer_size, draw_bytes=lambda data, n, distribution: distribution(health_check_random, n)) with Settings(settings, verbosity=Verbosity.quiet): test_runner( data, reify_and_execute( search_strategy, lambda *args, **kwargs: None, )) count += 1 except UnsatisfiedAssumption: filtered_draws += 1 except StopTest: if data.status == Status.INVALID: filtered_draws += 1 else: assert data.status == Status.OVERRUN overruns += 1 except InvalidArgument: raise except Exception: escalate_hypothesis_internal_error() if (HealthCheck.exception_in_generation in settings.suppress_health_check): raise report(traceback.format_exc()) if test_runner is default_new_style_executor: fail_health_check( settings, 'An exception occurred during data ' 'generation in initial health check. ' 'This indicates a bug in the strategy. ' 'This could either be a Hypothesis bug or ' "an error in a function you've passed to " 'it to construct your data.', HealthCheck.exception_in_generation, ) else: fail_health_check( settings, 'An exception occurred during data ' 'generation in initial health check. ' 'This indicates a bug in the strategy. ' 'This could either be a Hypothesis bug or ' 'an error in a function you\'ve passed to ' 'it to construct your data. Additionally, ' 'you have a custom executor, which means ' 'that this could be your executor failing ' 'to handle a function which returns None. ', HealthCheck.exception_in_generation, ) if overruns >= 20 or (not count and overruns > 0): fail_health_check( settings, ('Examples routinely exceeded the max allowable size. ' '(%d examples overran while generating %d valid ones)' '. Generating examples this large will usually lead to' ' bad results. You should try setting average_size or ' 'max_size parameters on your collections and turning ' 'max_leaves down on recursive() calls.') % (overruns, count), HealthCheck.data_too_large) if filtered_draws >= 50 or (not count and filtered_draws > 0): fail_health_check( settings, ('It looks like your strategy is filtering out a lot ' 'of data. Health check found %d filtered examples but ' 'only %d good ones. This will make your tests much ' 'slower, and also will probably distort the data ' 'generation quite a lot. You should adapt your ' 'strategy to filter less. This can also be caused by ' 'a low max_leaves parameter in recursive() calls') % (filtered_draws, count), HealthCheck.filter_too_much) runtime = time.time() - start if runtime > 1.0 or count < 10: fail_health_check( settings, ('Data generation is extremely slow: Only produced ' '%d valid examples in %.2f seconds (%d invalid ones ' 'and %d exceeded maximum size). Try decreasing ' "size of the data you're generating (with e.g." 'average_size or max_leaves parameters).') % (count, runtime, filtered_draws, overruns), HealthCheck.too_slow, )