def test_stops_after_max_examples_when_generating(): seen = [] def f(data): seen.append(data.draw_bytes(1)) runner = TestRunner(f, settings=settings(max_examples=1, database=None)) runner.run() assert len(seen) == 1
def test_run_with_timeout_while_boring(): def f(data): time.sleep(0.1) runner = TestRunner(f, settings=settings(database=None, timeout=0.2,)) start = time.time() runner.run() assert time.time() <= start + 1 assert runner.last_data.status == Status.VALID
def run_to_buffer(f): runner = TestRunner(f, settings=settings( max_examples=5000, max_iterations=10000, max_shrinks=MAX_SHRINKS, buffer_size=1024, database=None, )) runner.run() assert runner.last_data.status == Status.INTERESTING return hbytes(runner.last_data.buffer)
def test_can_navigate_to_a_valid_example(): def f(data): i = int_from_bytes(data.draw_bytes(2)) data.draw_bytes(i) data.mark_interesting() runner = TestRunner(f, settings=settings(max_examples=5000, max_iterations=10000, buffer_size=2, database=None)) runner.run() assert runner.last_data.status == Status.INTERESTING return hbytes(runner.last_data.buffer)
def test_phases_can_disable_shrinking(): seen = set() def f(data): seen.add(hbytes(data.draw_bytes(32))) data.mark_interesting() runner = TestRunner(f, settings=settings(database=None, phases=(Phase.reuse, Phase.generate))) runner.run() assert len(seen) == 1
def test_max_shrinks_can_disable_shrinking(): seen = set() def f(data): seen.add(hbytes(data.draw_bytes(32))) data.mark_interesting() runner = TestRunner(f, settings=settings(database=None, max_shrinks=0,)) runner.run() assert len(seen) == 1
def test_run_with_timeout_while_boring(): def f(data): time.sleep(0.1) runner = TestRunner(f, settings=settings( database=None, timeout=0.2, )) start = time.time() runner.run() assert time.time() <= start + 1 assert runner.last_data.status == Status.VALID
def test_run_with_timeout_while_shrinking(): def f(data): time.sleep(0.1) x = data.draw_bytes(32) if any(x): data.mark_interesting() runner = TestRunner(f, settings=settings(database=None, timeout=0.2,)) start = time.time() runner.run() assert time.time() <= start + 1 assert runner.last_data.status == Status.INTERESTING
def test_stops_after_max_examples_when_generating(): seen = [] def f(data): seen.append(data.draw_bytes(1)) runner = TestRunner(f, settings=settings( max_examples=1, database=None, )) runner.run() assert len(seen) == 1
def test_max_shrinks_can_disable_shrinking(): seen = set() def f(data): seen.add(hbytes(data.draw_bytes(32))) data.mark_interesting() runner = TestRunner(f, settings=settings( database=None, max_shrinks=0, )) runner.run() assert len(seen) == 1
def test_detects_flakiness(): failed_once = [False] count = [0] def tf(data): data.draw_bytes(1) count[0] += 1 if not failed_once[0]: failed_once[0] = True data.mark_interesting() runner = TestRunner(tf) runner.run() assert count == [2]
def test_phases_can_disable_shrinking(): seen = set() def f(data): seen.add(hbytes(data.draw_bytes(32))) data.mark_interesting() runner = TestRunner(f, settings=settings( database=None, phases=(Phase.reuse, Phase.generate), )) runner.run() assert len(seen) == 1
def test_garbage_collects_the_database(): key = b'hi there' n = 200 db = ExampleDatabase(':memory:') assert list(db.fetch(key)) == [] seen = set() go = True def f(data): x = hbytes(data.draw_bytes(512)) if not go: return if sum(x) >= 5000 and len(seen) < n: seen.add(x) if x in seen: data.mark_interesting() runner = TestRunner(f, settings=settings(database=db, max_shrinks=2 * n), database_key=key) runner.run() assert runner.last_data.status == Status.INTERESTING assert len(seen) == n assert set(db.fetch(key)) == seen go = False runner = TestRunner(f, settings=settings(database=db, max_shrinks=2 * n), database_key=key) runner.run() assert 0 < len(set(db.fetch(key))) < n
def test_can_load_data_from_a_corpus(): key = b'hi there' db = ExampleDatabase() value = b'=\xc3\xe4l\x81\xe1\xc2H\xc9\xfb\x1a\xb6bM\xa8\x7f' db.save(key, value) def f(data): if data.draw_bytes(len(value)) == value: data.mark_interesting() runner = TestRunner( f, settings=settings(database=db), database_key=key) runner.run() assert runner.last_data.status == Status.INTERESTING assert runner.last_data.buffer == value assert len(list(db.fetch(key))) == 1
def x(data): rnd = Random(hbytes(data.draw_bytes(8))) def g(d2): while True: b = d2.draw_bytes(1)[0] result = data.draw_bytes(b) if 255 in result: d2.mark_interesting() if 0 in result: d2.mark_invalid() runner = TestRunner(g, random=rnd) runner.run() if runner.last_data.status == Status.INTERESTING: data.mark_interesting()
def test_can_load_data_from_a_corpus(): key = b'hi there' db = ExampleDatabase() value = b'=\xc3\xe4l\x81\xe1\xc2H\xc9\xfb\x1a\xb6bM\xa8\x7f' db.save(key, value) def f(data): if data.draw_bytes(len(value)) == value: data.mark_interesting() runner = TestRunner(f, settings=settings(database=db), database_key=key) runner.run() assert runner.last_data.status == Status.INTERESTING assert runner.last_data.buffer == value assert len(list(db.fetch(key))) == 1
def test_stops_after_max_examples_when_reading(): key = b"key" db = ExampleDatabase(":memory:") for i in range(10): db.save(key, hbytes([i])) seen = [] def f(data): seen.append(data.draw_bytes(1)) runner = TestRunner(f, settings=settings(max_examples=1, database=db), database_key=key) runner.run() assert len(seen) == 1
def test_run_with_timeout_while_shrinking(): def f(data): time.sleep(0.1) x = data.draw_bytes(32) if any(x): data.mark_interesting() runner = TestRunner(f, settings=settings( database=None, timeout=0.2, )) start = time.time() runner.run() assert time.time() <= start + 1 assert runner.last_data.status == Status.INTERESTING
def test_terminates_shrinks(): shrinks = [-1] def tf(data): x = hbytes(data.draw_bytes(100)) if sum(x) >= 500: shrinks[0] += 1 data.mark_interesting() runner = TestRunner(tf, settings=settings(max_examples=5000, max_iterations=10000, max_shrinks=10, database=None)) runner.run() assert runner.last_data.status == Status.INTERESTING # There's an extra non-shrinking check step to abort in the presence of # flakiness assert shrinks[0] == 11
def test_can_navigate_to_a_valid_example(): def f(data): i = int_from_bytes(data.draw_bytes(2)) data.draw_bytes(i) data.mark_interesting() runner = TestRunner(f, settings=settings( max_examples=5000, max_iterations=10000, buffer_size=2, database=None, )) runner.run() assert runner.last_data.status == Status.INTERESTING return hbytes(runner.last_data.buffer)
def test_stops_after_max_iterations_when_generating(): key = b"key" value = b"rubber baby buggy bumpers" max_iterations = 100 db = ExampleDatabase(":memory:") db.save(key, value) seen = [] def f(data): seen.append(data.draw_bytes(len(value))) data.mark_invalid() runner = TestRunner( f, settings=settings(max_examples=1, max_iterations=max_iterations, database=db), database_key=key ) runner.run() assert len(seen) == max_iterations assert value in seen
def test_stops_after_max_examples_when_reading(): key = b'key' db = ExampleDatabase(':memory:') for i in range(10): db.save(key, hbytes([i])) seen = [] def f(data): seen.append(data.draw_bytes(1)) runner = TestRunner(f, settings=settings( max_examples=1, database=db, ), database_key=key) runner.run() assert len(seen) == 1
def test_stops_after_max_iterations_when_reading(): key = b'key' max_iterations = 1 db = ExampleDatabase(':memory:') for i in range(10): db.save(key, hbytes([i])) seen = [] def f(data): seen.append(data.draw_bytes(1)) data.mark_invalid() runner = TestRunner(f, settings=settings( max_examples=1, max_iterations=max_iterations, database=db, ), database_key=key) runner.run() assert len(seen) == max_iterations
def test_saves_data_while_shrinking(): key = b'hi there' n = 5 db = ExampleDatabase(':memory:') assert list(db.fetch(key)) == [] seen = set() def f(data): x = data.draw_bytes(512) if sum(x) >= 5000 and len(seen) < n: seen.add(hbytes(x)) if hbytes(x) in seen: data.mark_interesting() runner = TestRunner(f, settings=settings(database=db), database_key=key) runner.run() assert runner.last_data.status == Status.INTERESTING assert len(seen) == n in_db = set(db.fetch(key)) assert in_db.issubset(seen) assert in_db == seen
def test_saves_data_while_shrinking(): key = b'hi there' n = 5 db = ExampleDatabase(':memory:') assert list(db.fetch(key)) == [] seen = set() def f(data): x = data.draw_bytes(512) if sum(x) >= 5000 and len(seen) < n: seen.add(hbytes(x)) if hbytes(x) in seen: data.mark_interesting() runner = TestRunner( f, settings=settings(database=db), database_key=key) runner.run() assert runner.last_data.status == Status.INTERESTING assert len(seen) == n in_db = set(db.fetch(key)) assert in_db.issubset(seen) assert in_db == seen
def test_terminates_shrinks(): shrinks = [-1] def tf(data): x = hbytes(data.draw_bytes(100)) if sum(x) >= 500: shrinks[0] += 1 data.mark_interesting() runner = TestRunner(tf, settings=settings( max_examples=5000, max_iterations=10000, max_shrinks=10, database=None, )) runner.run() assert runner.last_data.status == Status.INTERESTING # There's an extra non-shrinking check step to abort in the presence of # flakiness assert shrinks[0] == 11
def test_garbage_collects_the_database(): key = b'hi there' n = 200 db = ExampleDatabase(':memory:') assert list(db.fetch(key)) == [] seen = set() go = True def f(data): x = hbytes(data.draw_bytes(512)) if not go: return if sum(x) >= 5000 and len(seen) < n: seen.add(x) if x in seen: data.mark_interesting() runner = TestRunner( f, settings=settings(database=db, max_shrinks=2 * n), database_key=key) runner.run() assert runner.last_data.status == Status.INTERESTING assert len(seen) == n assert set(db.fetch(key)) == seen go = False runner = TestRunner( f, settings=settings(database=db, max_shrinks=2 * n), database_key=key) runner.run() assert 0 < len(set(db.fetch(key))) < n
def run_test(): if condition is None: _condition = lambda x: True condition_string = u'' else: _condition = condition condition_string = strip_lambda( reflection.get_pretty_function_description(condition)) count = [0] successful_runs = [0] def test_function(data): try: value = data.draw(specifier) except UnsatisfiedAssumption: data.mark_invalid() if not _condition(value): data.mark_invalid() successful_runs[0] += 1 if predicate(value): count[0] += 1 TestRunner(test_function, settings=Settings( max_examples=MAX_RUNS, max_iterations=MAX_RUNS * 10, )).run() successful_runs = successful_runs[0] count = count[0] if successful_runs < MIN_RUNS: raise ConditionTooHard( (u'Unable to find enough examples satisfying predicate %s ' u'only found %d but required at least %d for validity') % (condition_string, successful_runs, MIN_RUNS)) result = Result( count, successful_runs, q, predicate, condition_string, specifier, ) p = cumulative_binomial_probability(successful_runs, q, count) run_test.test_result = result # The test passes if we fail to reject the null hypothesis that # the probability is at least q if p < REQUIRED_P: result.failed = True raise HypothesisFalsified(result.description() + u' rejected')
def test_stops_after_max_iterations_when_generating(): key = b'key' value = b'rubber baby buggy bumpers' max_iterations = 100 db = ExampleDatabase(':memory:') db.save(key, value) seen = [] def f(data): seen.append(data.draw_bytes(len(value))) data.mark_invalid() runner = TestRunner(f, settings=settings( max_examples=1, max_iterations=max_iterations, database=db, ), database_key=key) runner.run() assert len(seen) == max_iterations assert value in seen
def find(specifier, condition, settings=None, random=None, database_key=None): settings = settings or Settings(max_examples=2000, min_satisfying_examples=0, max_shrinks=2000) if database_key is None and settings.database is not None: database_key = function_digest(condition) if not isinstance(specifier, SearchStrategy): raise InvalidArgument("Expected SearchStrategy but got %r of type %s" % (specifier, type(specifier).__name__)) search = specifier random = random or new_random() successful_examples = [0] last_data = [None] def template_condition(data): with BuildContext(): try: data.is_find = True result = data.draw(search) data.note(result) success = condition(result) except UnsatisfiedAssumption: data.mark_invalid() if success: successful_examples[0] += 1 if settings.verbosity == Verbosity.verbose: if not successful_examples[0]: report(lambda: u"Trying example %s" % (nicerepr(result),)) elif success: if successful_examples[0] == 1: report(lambda: u"Found satisfying example %s" % (nicerepr(result),)) else: report(lambda: u"Shrunk example to %s" % (nicerepr(result),)) last_data[0] = data if success and not data.frozen: data.mark_interesting() from hypothesis.internal.conjecture.engine import TestRunner from hypothesis.internal.conjecture.data import TestData, Status start = time.time() runner = TestRunner(template_condition, settings=settings, random=random, database_key=database_key) runner.run() run_time = time.time() - start if runner.last_data.status == Status.INTERESTING: with BuildContext(): return TestData.for_buffer(runner.last_data.buffer).draw(search) if runner.valid_examples <= settings.min_satisfying_examples: if settings.timeout > 0 and run_time > settings.timeout: raise Timeout( ( "Ran out of time before finding enough valid examples for " "%s. Only %d valid examples found in %.2f seconds." ) % (get_pretty_function_description(condition), runner.valid_examples, run_time) ) else: raise Unsatisfiable( ("Unable to satisfy assumptions of " "%s. Only %d examples considered satisfied assumptions") % (get_pretty_function_description(condition), runner.valid_examples) ) raise NoSuchExample(get_pretty_function_description(condition))
def wrapped_test(*arguments, **kwargs): settings = wrapped_test._hypothesis_internal_use_settings if wrapped_test._hypothesis_internal_use_seed is not None: random = Random(wrapped_test._hypothesis_internal_use_seed) elif settings.derandomize: random = Random(function_digest(test)) else: random = new_random() import hypothesis.strategies as sd selfy = None arguments, kwargs = convert_positional_arguments(wrapped_test, arguments, kwargs) # If the test function is a method of some kind, the bound object # will be the first named argument if there are any, otherwise the # first vararg (if any). if argspec.args: selfy = kwargs.get(argspec.args[0]) elif arguments: selfy = arguments[0] test_runner = new_style_executor(selfy) for example in reversed(getattr(wrapped_test, "hypothesis_explicit_examples", ())): if example.args: if len(example.args) > len(original_argspec.args): raise InvalidArgument( "example has too many arguments for test. " "Expected at most %d but got %d" % (len(original_argspec.args), len(example.args)) ) example_kwargs = dict(zip(original_argspec.args[-len(example.args) :], example.args)) else: example_kwargs = example.kwargs if Phase.explicit not in settings.phases: continue example_kwargs.update(kwargs) # Note: Test may mutate arguments and we can't rerun explicit # examples, so we have to calculate the failure message at this # point rather than than later. message_on_failure = "Falsifying example: %s(%s)" % ( test.__name__, arg_string(test, arguments, example_kwargs), ) try: with BuildContext() as b: test_runner(None, lambda data: test(*arguments, **example_kwargs)) except BaseException: report(message_on_failure) for n in b.notes: report(n) raise if settings.max_examples <= 0: return arguments = tuple(arguments) given_specifier = sd.tuples( sd.just(arguments), sd.fixed_dictionaries(generator_kwargs).map(lambda args: dict(args, **kwargs)) ) def fail_health_check(message, label): if label in settings.suppress_health_check: return message += ( "\nSee https://hypothesis.readthedocs.io/en/latest/health" "checks.html for more information about this. " ) message += ( "If you want to disable just this health check, add %s " "to the suppress_health_check settings for this test." ) % (label,) raise FailedHealthCheck(message) search_strategy = given_specifier if selfy is not None: search_strategy = WithRunner(search_strategy, selfy) search_strategy.validate() perform_health_check = settings.perform_health_check perform_health_check &= Settings.default.perform_health_check from hypothesis.internal.conjecture.data import TestData, Status, StopTest if not (Phase.reuse in settings.phases or Phase.generate in settings.phases): return if perform_health_check: health_check_random = Random(random.getrandbits(128)) # We "pre warm" the health check with one draw to give it some # time to calculate any cached data. This prevents the case # where the first draw of the health check takes ages because # of loading unicode data the first time. data = TestData( max_length=settings.buffer_size, draw_bytes=lambda data, n, distribution: distribution(health_check_random, n), ) with Settings(settings, verbosity=Verbosity.quiet): try: test_runner(data, reify_and_execute(search_strategy, lambda *args, **kwargs: None)) except BaseException: pass count = 0 overruns = 0 filtered_draws = 0 start = time.time() while count < 10 and time.time() < start + 1 and filtered_draws < 50 and overruns < 20: try: data = TestData( max_length=settings.buffer_size, draw_bytes=lambda data, n, distribution: distribution(health_check_random, n), ) with Settings(settings, verbosity=Verbosity.quiet): test_runner(data, reify_and_execute(search_strategy, lambda *args, **kwargs: None)) count += 1 except UnsatisfiedAssumption: filtered_draws += 1 except StopTest: if data.status == Status.INVALID: filtered_draws += 1 else: assert data.status == Status.OVERRUN overruns += 1 except InvalidArgument: raise except Exception: if HealthCheck.exception_in_generation in settings.suppress_health_check: raise report(traceback.format_exc()) if test_runner is default_new_style_executor: fail_health_check( "An exception occurred during data " "generation in initial health check. " "This indicates a bug in the strategy. " "This could either be a Hypothesis bug or " "an error in a function you've passed to " "it to construct your data.", HealthCheck.exception_in_generation, ) else: fail_health_check( "An exception occurred during data " "generation in initial health check. " "This indicates a bug in the strategy. " "This could either be a Hypothesis bug or " "an error in a function you've passed to " "it to construct your data. Additionally, " "you have a custom executor, which means " "that this could be your executor failing " "to handle a function which returns None. ", HealthCheck.exception_in_generation, ) if overruns >= 20 or (not count and overruns > 0): fail_health_check( ( "Examples routinely exceeded the max allowable size. " "(%d examples overran while generating %d valid ones)" ". Generating examples this large will usually lead to" " bad results. You should try setting average_size or " "max_size parameters on your collections and turning " "max_leaves down on recursive() calls." ) % (overruns, count), HealthCheck.data_too_large, ) if filtered_draws >= 50 or (not count and filtered_draws > 0): fail_health_check( ( "It looks like your strategy is filtering out a lot " "of data. Health check found %d filtered examples but " "only %d good ones. This will make your tests much " "slower, and also will probably distort the data " "generation quite a lot. You should adapt your " "strategy to filter less. This can also be caused by " "a low max_leaves parameter in recursive() calls" ) % (filtered_draws, count), HealthCheck.filter_too_much, ) runtime = time.time() - start if runtime > 1.0 or count < 10: fail_health_check( ( "Data generation is extremely slow: Only produced " "%d valid examples in %.2f seconds (%d invalid ones " "and %d exceeded maximum size). Try decreasing " "size of the data you're generating (with e.g." "average_size or max_leaves parameters)." ) % (count, runtime, filtered_draws, overruns), HealthCheck.too_slow, ) last_exception = [None] repr_for_last_exception = [None] def evaluate_test_data(data): try: result = test_runner(data, reify_and_execute(search_strategy, test)) if result is not None and settings.perform_health_check: fail_health_check( ("Tests run under @given should return None, but " "%s returned %r instead.") % (test.__name__, result), HealthCheck.return_value, ) return False except UnsatisfiedAssumption: data.mark_invalid() except (HypothesisDeprecationWarning, FailedHealthCheck, StopTest): raise except Exception: last_exception[0] = traceback.format_exc() verbose_report(last_exception[0]) data.mark_interesting() from hypothesis.internal.conjecture.engine import TestRunner falsifying_example = None database_key = str_to_bytes(fully_qualified_name(test)) start_time = time.time() runner = TestRunner(evaluate_test_data, settings=settings, random=random, database_key=database_key) runner.run() run_time = time.time() - start_time timed_out = settings.timeout > 0 and run_time >= settings.timeout if runner.last_data is None: return if runner.last_data.status == Status.INTERESTING: falsifying_example = runner.last_data.buffer if settings.database is not None: settings.database.save(database_key, falsifying_example) else: if runner.valid_examples < min(settings.min_satisfying_examples, settings.max_examples): if timed_out: raise Timeout( ( "Ran out of time before finding a satisfying " "example for " "%s. Only found %d examples in " + "%.2fs." ) % (get_pretty_function_description(test), runner.valid_examples, run_time) ) else: raise Unsatisfiable( ( "Unable to satisfy assumptions of hypothesis " "%s. Only %d examples considered " "satisfied assumptions" ) % (get_pretty_function_description(test), runner.valid_examples) ) return assert last_exception[0] is not None try: with settings: test_runner( TestData.for_buffer(falsifying_example), reify_and_execute(search_strategy, test, print_example=True, is_final=True), ) except (UnsatisfiedAssumption, StopTest): report(traceback.format_exc()) raise Flaky( "Unreliable assumption: An example which satisfied " "assumptions on the first run now fails it." ) report("Failed to reproduce exception. Expected: \n" + last_exception[0]) filter_message = ( "Unreliable test data: Failed to reproduce a failure " "and then when it came to recreating the example in " "order to print the test data with a flaky result " "the example was filtered out (by e.g. a " "call to filter in your strategy) when we didn't " "expect it to be." ) try: test_runner( TestData.for_buffer(falsifying_example), reify_and_execute( search_strategy, test_is_flaky(test, repr_for_last_exception[0]), print_example=True, is_final=True, ), ) except (UnsatisfiedAssumption, StopTest): raise Flaky(filter_message)
def wrapped_test(*arguments, **kwargs): settings = wrapped_test._hypothesis_internal_use_settings if wrapped_test._hypothesis_internal_use_seed is not None: random = Random( wrapped_test._hypothesis_internal_use_seed) elif settings.derandomize: random = Random(function_digest(test)) else: random = new_random() import hypothesis.strategies as sd selfy = None arguments, kwargs = convert_positional_arguments( wrapped_test, arguments, kwargs) # If the test function is a method of some kind, the bound object # will be the first named argument if there are any, otherwise the # first vararg (if any). if argspec.args: selfy = kwargs.get(argspec.args[0]) elif arguments: selfy = arguments[0] test_runner = new_style_executor(selfy) for example in reversed(getattr( wrapped_test, 'hypothesis_explicit_examples', () )): if example.args: if len(example.args) > len(original_argspec.args): raise InvalidArgument( 'example has too many arguments for test. ' 'Expected at most %d but got %d' % ( len(original_argspec.args), len(example.args))) example_kwargs = dict(zip( original_argspec.args[-len(example.args):], example.args )) else: example_kwargs = example.kwargs example_kwargs.update(kwargs) # Note: Test may mutate arguments and we can't rerun explicit # examples, so we have to calculate the failure message at this # point rather than than later. message_on_failure = 'Falsifying example: %s(%s)' % ( test.__name__, arg_string(test, arguments, example_kwargs) ) try: with BuildContext() as b: test_runner( None, lambda data: test(*arguments, **example_kwargs) ) except BaseException: report(message_on_failure) for n in b.notes: report(n) raise if settings.max_examples <= 0: return arguments = tuple(arguments) given_specifier = sd.tuples( sd.just(arguments), sd.fixed_dictionaries(generator_kwargs).map( lambda args: dict(args, **kwargs) ) ) def fail_health_check(message): message += ( '\nSee http://hypothesis.readthedocs.org/en/latest/health' 'checks.html for more information about this.' ) raise FailedHealthCheck(message) search_strategy = given_specifier search_strategy.validate() perform_health_check = settings.perform_health_check perform_health_check &= Settings.default.perform_health_check from hypothesis.internal.conjecture.data import TestData, Status, \ StopTest if perform_health_check: initial_state = getglobalrandomstate() health_check_random = Random(random.getrandbits(128)) # We "pre warm" the health check with one draw to give it some # time to calculate any cached data. This prevents the case # where the first draw of the health check takes ages because # of loading unicode data the first time. data = TestData( max_length=settings.buffer_size, draw_bytes=lambda data, n, distribution: distribution(health_check_random, n) ) with Settings(settings, verbosity=Verbosity.quiet): try: test_runner(data, reify_and_execute( search_strategy, lambda *args, **kwargs: None, )) except BaseException: pass count = 0 overruns = 0 filtered_draws = 0 start = time.time() while ( count < 10 and time.time() < start + 1 and filtered_draws < 50 and overruns < 20 ): try: data = TestData( max_length=settings.buffer_size, draw_bytes=lambda data, n, distribution: distribution(health_check_random, n) ) with Settings(settings, verbosity=Verbosity.quiet): test_runner(data, reify_and_execute( search_strategy, lambda *args, **kwargs: None, )) count += 1 except UnsatisfiedAssumption: filtered_draws += 1 except StopTest: if data.status == Status.INVALID: filtered_draws += 1 else: assert data.status == Status.OVERRUN overruns += 1 except Exception: report(traceback.format_exc()) if test_runner is default_new_style_executor: fail_health_check( 'An exception occurred during data ' 'generation in initial health check. ' 'This indicates a bug in the strategy. ' 'This could either be a Hypothesis bug or ' "an error in a function yo've passed to " 'it to construct your data.' ) else: fail_health_check( 'An exception occurred during data ' 'generation in initial health check. ' 'This indicates a bug in the strategy. ' 'This could either be a Hypothesis bug or ' 'an error in a function you\'ve passed to ' 'it to construct your data. Additionally, ' 'you have a custom executor, which means ' 'that this could be your executor failing ' 'to handle a function which returns None. ' ) if overruns >= 20 or ( not count and overruns > 0 ): fail_health_check(( 'Examples routinely exceeded the max allowable size. ' '(%d examples overran while generating %d valid ones)' '. Generating examples this large will usually lead to' ' bad results. You should try setting average_size or ' 'max_size parameters on your collections and turning ' 'max_leaves down on recursive() calls.') % ( overruns, count )) if filtered_draws >= 50 or ( not count and filtered_draws > 0 ): fail_health_check(( 'It looks like your strategy is filtering out a lot ' 'of data. Health check found %d filtered examples but ' 'only %d good ones. This will make your tests much ' 'slower, and also will probably distort the data ' 'generation quite a lot. You should adapt your ' 'strategy to filter less. This can also be caused by ' 'a low max_leaves parameter in recursive() calls') % ( filtered_draws, count )) runtime = time.time() - start if runtime > 1.0 or count < 10: fail_health_check(( 'Data generation is extremely slow: Only produced ' '%d valid examples in %.2f seconds (%d invalid ones ' 'and %d exceeded maximum size). Try decreasing ' "size of the data you're generating (with e.g." 'average_size or max_leaves parameters).' ) % (count, runtime, filtered_draws, overruns)) if getglobalrandomstate() != initial_state: fail_health_check( 'Data generation depends on global random module. ' 'This makes results impossible to replay, which ' 'prevents Hypothesis from working correctly. ' 'If you want to use methods from random, use ' 'randoms() from hypothesis.strategies to get an ' 'instance of Random you can use. Alternatively, you ' 'can use the random_module() strategy to explicitly ' 'seed the random module.' ) last_exception = [None] repr_for_last_exception = [None] performed_random_check = [False] def evaluate_test_data(data): if perform_health_check and not performed_random_check[0]: initial_state = getglobalrandomstate() performed_random_check[0] = True else: initial_state = None try: result = test_runner(data, reify_and_execute( search_strategy, test, )) if result is not None and settings.perform_health_check: raise FailedHealthCheck(( 'Tests run under @given should return None, but ' '%s returned %r instead.' ) % (test.__name__, result), settings) return False except UnsatisfiedAssumption: data.mark_invalid() except ( HypothesisDeprecationWarning, FailedHealthCheck, StopTest, ): raise except Exception: last_exception[0] = traceback.format_exc() verbose_report(last_exception[0]) data.mark_interesting() finally: if ( initial_state is not None and getglobalrandomstate() != initial_state ): fail_health_check( 'Your test used the global random module. ' 'This is unlikely to work correctly. You should ' 'consider using the randoms() strategy from ' 'hypothesis.strategies instead. Alternatively, ' 'you can use the random_module() strategy to ' 'explicitly seed the random module.') from hypothesis.internal.conjecture.engine import TestRunner falsifying_example = None database_key = str_to_bytes(fully_qualified_name(test)) start_time = time.time() runner = TestRunner( evaluate_test_data, settings=settings, random=random, database_key=database_key, ) runner.run() run_time = time.time() - start_time timed_out = ( settings.timeout > 0 and run_time >= settings.timeout ) if runner.last_data.status == Status.INTERESTING: falsifying_example = runner.last_data.buffer if settings.database is not None: settings.database.save( database_key, falsifying_example ) else: if runner.valid_examples < min( settings.min_satisfying_examples, settings.max_examples, ): if timed_out: raise Timeout(( 'Ran out of time before finding a satisfying ' 'example for ' '%s. Only found %d examples in ' + '%.2fs.' ) % ( get_pretty_function_description(test), runner.valid_examples, run_time )) else: raise Unsatisfiable(( 'Unable to satisfy assumptions of hypothesis ' '%s. Only %d examples considered ' 'satisfied assumptions' ) % ( get_pretty_function_description(test), runner.valid_examples,)) return assert last_exception[0] is not None try: with settings: test_runner( TestData.for_buffer(falsifying_example), reify_and_execute( search_strategy, test, print_example=True, is_final=True )) except (UnsatisfiedAssumption, StopTest): report(traceback.format_exc()) raise Flaky( 'Unreliable assumption: An example which satisfied ' 'assumptions on the first run now fails it.' ) report( 'Failed to reproduce exception. Expected: \n' + last_exception[0], ) filter_message = ( 'Unreliable test data: Failed to reproduce a failure ' 'and then when it came to recreating the example in ' 'order to print the test data with a flaky result ' 'the example was filtered out (by e.g. a ' 'call to filter in your strategy) when we didn\'t ' 'expect it to be.' ) try: test_runner( TestData.for_buffer(falsifying_example), reify_and_execute( search_strategy, test_is_flaky(test, repr_for_last_exception[0]), print_example=True, is_final=True )) except (UnsatisfiedAssumption, StopTest): raise Flaky(filter_message)
def wrapped_test(*arguments, **kwargs): settings = wrapped_test._hypothesis_internal_use_settings if wrapped_test._hypothesis_internal_use_seed is not None: random = Random(wrapped_test._hypothesis_internal_use_seed) elif settings.derandomize: random = Random(function_digest(test)) else: random = new_random() import hypothesis.strategies as sd selfy = None arguments, kwargs = convert_positional_arguments( wrapped_test, arguments, kwargs) # If the test function is a method of some kind, the bound object # will be the first named argument if there are any, otherwise the # first vararg (if any). if argspec.args: selfy = kwargs.get(argspec.args[0]) elif arguments: selfy = arguments[0] test_runner = new_style_executor(selfy) for example in reversed( getattr(wrapped_test, 'hypothesis_explicit_examples', ())): if example.args: if len(example.args) > len(original_argspec.args): raise InvalidArgument( 'example has too many arguments for test. ' 'Expected at most %d but got %d' % (len(original_argspec.args), len(example.args))) example_kwargs = dict( zip(original_argspec.args[-len(example.args):], example.args)) else: example_kwargs = example.kwargs if Phase.explicit not in settings.phases: continue example_kwargs.update(kwargs) # Note: Test may mutate arguments and we can't rerun explicit # examples, so we have to calculate the failure message at this # point rather than than later. message_on_failure = 'Falsifying example: %s(%s)' % ( test.__name__, arg_string(test, arguments, example_kwargs)) try: with BuildContext() as b: test_runner( None, lambda data: test(*arguments, **example_kwargs)) except BaseException: report(message_on_failure) for n in b.notes: report(n) raise if settings.max_examples <= 0: return arguments = tuple(arguments) given_specifier = sd.tuples( sd.just(arguments), sd.fixed_dictionaries(generator_kwargs).map( lambda args: dict(args, **kwargs))) def fail_health_check(message, label): if label in settings.suppress_health_check: return message += ( '\nSee http://hypothesis.readthedocs.org/en/latest/health' 'checks.html for more information about this. ') message += ( 'If you want to disable just this health check, add %s ' 'to the suppress_health_check settings for this test.') % ( label, ) raise FailedHealthCheck(message) search_strategy = given_specifier if selfy is not None: search_strategy = WithRunner(search_strategy, selfy) search_strategy.validate() perform_health_check = settings.perform_health_check perform_health_check &= Settings.default.perform_health_check from hypothesis.internal.conjecture.data import TestData, Status, \ StopTest if not (Phase.reuse in settings.phases or Phase.generate in settings.phases): return if perform_health_check: health_check_random = Random(random.getrandbits(128)) # We "pre warm" the health check with one draw to give it some # time to calculate any cached data. This prevents the case # where the first draw of the health check takes ages because # of loading unicode data the first time. data = TestData(max_length=settings.buffer_size, draw_bytes=lambda data, n, distribution: distribution(health_check_random, n)) with Settings(settings, verbosity=Verbosity.quiet): try: test_runner( data, reify_and_execute( search_strategy, lambda *args, **kwargs: None, )) except BaseException: pass count = 0 overruns = 0 filtered_draws = 0 start = time.time() while (count < 10 and time.time() < start + 1 and filtered_draws < 50 and overruns < 20): try: data = TestData( max_length=settings.buffer_size, draw_bytes=lambda data, n, distribution: distribution(health_check_random, n)) with Settings(settings, verbosity=Verbosity.quiet): test_runner( data, reify_and_execute( search_strategy, lambda *args, **kwargs: None, )) count += 1 except UnsatisfiedAssumption: filtered_draws += 1 except StopTest: if data.status == Status.INVALID: filtered_draws += 1 else: assert data.status == Status.OVERRUN overruns += 1 except InvalidArgument: raise except Exception: if (HealthCheck.exception_in_generation in settings.suppress_health_check): raise report(traceback.format_exc()) if test_runner is default_new_style_executor: fail_health_check( 'An exception occurred during data ' 'generation in initial health check. ' 'This indicates a bug in the strategy. ' 'This could either be a Hypothesis bug or ' "an error in a function you've passed to " 'it to construct your data.', HealthCheck.exception_in_generation, ) else: fail_health_check( 'An exception occurred during data ' 'generation in initial health check. ' 'This indicates a bug in the strategy. ' 'This could either be a Hypothesis bug or ' 'an error in a function you\'ve passed to ' 'it to construct your data. Additionally, ' 'you have a custom executor, which means ' 'that this could be your executor failing ' 'to handle a function which returns None. ', HealthCheck.exception_in_generation, ) if overruns >= 20 or (not count and overruns > 0): fail_health_check(( 'Examples routinely exceeded the max allowable size. ' '(%d examples overran while generating %d valid ones)' '. Generating examples this large will usually lead to' ' bad results. You should try setting average_size or ' 'max_size parameters on your collections and turning ' 'max_leaves down on recursive() calls.') % (overruns, count), HealthCheck.data_too_large) if filtered_draws >= 50 or (not count and filtered_draws > 0): fail_health_check(( 'It looks like your strategy is filtering out a lot ' 'of data. Health check found %d filtered examples but ' 'only %d good ones. This will make your tests much ' 'slower, and also will probably distort the data ' 'generation quite a lot. You should adapt your ' 'strategy to filter less. This can also be caused by ' 'a low max_leaves parameter in recursive() calls') % (filtered_draws, count), HealthCheck.filter_too_much) runtime = time.time() - start if runtime > 1.0 or count < 10: fail_health_check( ('Data generation is extremely slow: Only produced ' '%d valid examples in %.2f seconds (%d invalid ones ' 'and %d exceeded maximum size). Try decreasing ' "size of the data you're generating (with e.g." 'average_size or max_leaves parameters).') % (count, runtime, filtered_draws, overruns), HealthCheck.too_slow, ) last_exception = [None] repr_for_last_exception = [None] def evaluate_test_data(data): try: result = test_runner( data, reify_and_execute( search_strategy, test, )) if result is not None and settings.perform_health_check: fail_health_check( ('Tests run under @given should return None, but ' '%s returned %r instead.') % (test.__name__, result), HealthCheck.return_value) return False except UnsatisfiedAssumption: data.mark_invalid() except ( HypothesisDeprecationWarning, FailedHealthCheck, StopTest, ): raise except Exception: last_exception[0] = traceback.format_exc() verbose_report(last_exception[0]) data.mark_interesting() from hypothesis.internal.conjecture.engine import TestRunner falsifying_example = None database_key = str_to_bytes(fully_qualified_name(test)) start_time = time.time() runner = TestRunner( evaluate_test_data, settings=settings, random=random, database_key=database_key, ) runner.run() run_time = time.time() - start_time timed_out = (settings.timeout > 0 and run_time >= settings.timeout) if runner.last_data is None: return if runner.last_data.status == Status.INTERESTING: falsifying_example = runner.last_data.buffer if settings.database is not None: settings.database.save(database_key, falsifying_example) else: if runner.valid_examples < min( settings.min_satisfying_examples, settings.max_examples, ): if timed_out: raise Timeout( ('Ran out of time before finding a satisfying ' 'example for ' '%s. Only found %d examples in ' + '%.2fs.') % (get_pretty_function_description(test), runner.valid_examples, run_time)) else: raise Unsatisfiable( ('Unable to satisfy assumptions of hypothesis ' '%s. Only %d examples considered ' 'satisfied assumptions') % ( get_pretty_function_description(test), runner.valid_examples, )) return assert last_exception[0] is not None try: with settings: test_runner( TestData.for_buffer(falsifying_example), reify_and_execute(search_strategy, test, print_example=True, is_final=True)) except (UnsatisfiedAssumption, StopTest): report(traceback.format_exc()) raise Flaky( 'Unreliable assumption: An example which satisfied ' 'assumptions on the first run now fails it.') report( 'Failed to reproduce exception. Expected: \n' + last_exception[0], ) filter_message = ( 'Unreliable test data: Failed to reproduce a failure ' 'and then when it came to recreating the example in ' 'order to print the test data with a flaky result ' 'the example was filtered out (by e.g. a ' 'call to filter in your strategy) when we didn\'t ' 'expect it to be.') try: test_runner( TestData.for_buffer(falsifying_example), reify_and_execute(search_strategy, test_is_flaky( test, repr_for_last_exception[0]), print_example=True, is_final=True)) except (UnsatisfiedAssumption, StopTest): raise Flaky(filter_message)
def find(specifier, condition, settings=None, random=None, database_key=None): settings = settings or Settings( max_examples=2000, min_satisfying_examples=0, max_shrinks=2000, ) if database_key is None and settings.database is not None: database_key = function_digest(condition) if not isinstance(specifier, SearchStrategy): raise InvalidArgument('Expected SearchStrategy but got %r of type %s' % (specifier, type(specifier).__name__)) search = specifier random = random or new_random() successful_examples = [0] last_data = [None] def template_condition(data): with BuildContext(): try: data.is_find = True result = data.draw(search) data.note(result) success = condition(result) except UnsatisfiedAssumption: data.mark_invalid() if success: successful_examples[0] += 1 if settings.verbosity == Verbosity.verbose: if not successful_examples[0]: report(lambda: u'Trying example %s' % (nicerepr(result), )) elif success: if successful_examples[0] == 1: report(lambda: u'Found satisfying example %s' % (nicerepr(result), )) else: report(lambda: u'Shrunk example to %s' % (nicerepr(result), )) last_data[0] = data if success and not data.frozen: data.mark_interesting() from hypothesis.internal.conjecture.engine import TestRunner from hypothesis.internal.conjecture.data import TestData, Status start = time.time() runner = TestRunner( template_condition, settings=settings, random=random, database_key=database_key, ) runner.run() run_time = time.time() - start if runner.last_data.status == Status.INTERESTING: with BuildContext(): return TestData.for_buffer(runner.last_data.buffer).draw(search) if runner.valid_examples <= settings.min_satisfying_examples: if settings.timeout > 0 and run_time > settings.timeout: raise Timeout( ('Ran out of time before finding enough valid examples for ' '%s. Only %d valid examples found in %.2f seconds.') % (get_pretty_function_description(condition), runner.valid_examples, run_time)) else: raise Unsatisfiable( ('Unable to satisfy assumptions of ' '%s. Only %d examples considered satisfied assumptions') % ( get_pretty_function_description(condition), runner.valid_examples, )) raise NoSuchExample(get_pretty_function_description(condition))