def __init__(self, specifier, settings): self.specifier = specifier self.leaf_strategy = strategy(specifier.leaf_values, settings) self.branch_key_strategy = strategy(specifier.branch_keys, settings) self.branch_label_strategy = strategy(specifier.branch_labels, settings) self.child_strategy = (lists(tuples(self.branch_key_strategy, self)))
def __init__(self, specifier, settings): self.specifier = specifier self.leaf_strategy = strategy(specifier.leaf_values, settings) self.branch_key_strategy = strategy( specifier.branch_keys, settings) self.branch_label_strategy = strategy( specifier.branch_labels, settings) self.child_strategy = ( lists(tuples(self.branch_key_strategy, self)))
def define_rule(cls, targets, function, arguments): converted_arguments = {} for k, v in arguments.items(): if not isinstance(v, Bundle): v = strategy(v) converted_arguments[k] = v if cls in cls._rules_per_class: target = cls._rules_per_class[cls] else: target = cls._base_rules_per_class.setdefault(cls, []) return target.append( Rule(targets, function, converted_arguments) )
def find(specifier, condition, settings=None, random=None, storage=None): settings = settings or Settings(max_examples=2000, min_satisfying_examples=0, max_shrinks=2000) from hypothesis.internal.strategymethod import strategy search = strategy(specifier, settings) if storage is None and settings.database is not None: storage = settings.database.storage( u"find(%s)" % (binascii.hexlify(function_digest(condition)).decode(u"ascii"),) ) random = random or new_random() successful_examples = [0] def template_condition(template): with BuildContext(): result = search.reify(template) success = condition(result) if success: successful_examples[0] += 1 if not successful_examples[0]: verbose_report(lambda: u"Trying example %s" % (repr(result),)) elif success: if successful_examples[0] == 1: verbose_report(lambda: u"Found satisfying example %s" % (repr(result),)) else: verbose_report(lambda: u"Shrunk example to %s" % (repr(result),)) return success template_condition.__name__ = condition.__name__ tracker = Tracker() try: template = best_satisfying_template( search, random, template_condition, settings, tracker=tracker, max_parameter_tries=2, storage=storage ) with BuildContext(is_final=True, close_on_capture=False): return search.reify(template) except Timeout: raise except NoSuchExample: if search.template_upper_bound <= len(tracker): raise DefinitelyNoSuchExample(get_pretty_function_description(condition), search.template_upper_bound) raise NoSuchExample(get_pretty_function_description(condition))
def pack(self, source_and_morpher): source, morpher = source_and_morpher return morpher.become(strategy(self.expand(source)))
def wrapped_test(*arguments, **kwargs): if settings.derandomize: random = Random(function_digest(test)) else: random = provided_random or new_random() import hypothesis.strategies as sd from hypothesis.internal.strategymethod import strategy selfy = None arguments, kwargs = convert_positional_arguments( wrapped_test, arguments, kwargs) for arg in hypothesis_owned_arguments: try: value = kwargs[arg] except KeyError: continue if not isinstance(value, HypothesisProvided): note_deprecation( 'Passing in explicit values to override Hypothesis ' 'provided values is deprecated and will no longer ' 'work in Hypothesis 2.0. If you need to do this, ' 'extract a common function and call that from a ' 'Hypothesis based test.', settings ) # Anything in unused_kwargs hasn't been injected through # argspec.defaults, so we need to add them. for k in unused_kwargs: if k not in kwargs: kwargs[k] = unused_kwargs[k] # If the test function is a method of some kind, the bound object # will be the first named argument if there are any, otherwise the # first vararg (if any). if argspec.args: selfy = kwargs.get(argspec.args[0]) elif arguments: selfy = arguments[0] if isinstance(selfy, HypothesisProvided): selfy = None test_runner = executor(selfy) for example in reversed(getattr( wrapped_test, u'hypothesis_explicit_examples', () )): if example.args: example_kwargs = dict(zip( argspec.args[-len(example.args):], example.args )) else: example_kwargs = dict(example.kwargs) for k, v in kwargs.items(): if not isinstance(v, HypothesisProvided): example_kwargs[k] = v # Note: Test may mutate arguments and we can't rerun explicit # examples, so we have to calculate the failure message at this # point rather than than later. message_on_failure = u'Falsifying example: %s(%s)' % ( test.__name__, arg_string(test, arguments, example_kwargs) ) try: with BuildContext() as b: test_runner( lambda: test(*arguments, **example_kwargs) ) except BaseException: report(message_on_failure) for n in b.notes: report(n) raise if not any( isinstance(x, HypothesisProvided) for xs in (arguments, kwargs.values()) for x in xs ): # All arguments have been satisfied without needing to invoke # hypothesis test_runner(lambda: test(*arguments, **kwargs)) return def convert_to_specifier(v): if isinstance(v, HypothesisProvided): return strategy(v.value, settings) else: return sd.just(v) given_specifier = sd.tuples( sd.tuples(*map(convert_to_specifier, arguments)), sd.fixed_dictionaries(dict( (k, convert_to_specifier(v)) for (k, v) in kwargs.items())) ) def fail_health_check(message): message += ( '\nSee http://hypothesis.readthedocs.org/en/latest/health' 'checks.html for more information about this.' ) if settings.strict: raise FailedHealthCheck(message) else: warnings.warn(FailedHealthCheck(message)) search_strategy = strategy(given_specifier, settings) search_strategy.validate() if settings.database: storage = settings.database.storage( fully_qualified_name(test)) else: storage = None start = time.time() warned_random = [False] perform_health_check = settings.perform_health_check if Settings.default is not None: perform_health_check &= Settings.default.perform_health_check if perform_health_check: initial_state = getglobalrandomstate() health_check_random = Random(random.getrandbits(128)) count = 0 bad_draws = 0 filtered_draws = 0 errors = 0 while ( count < 10 and time.time() < start + 1 and filtered_draws < 50 and bad_draws < 50 ): try: with Settings(settings, verbosity=Verbosity.quiet): test_runner(reify_and_execute( search_strategy, search_strategy.draw_template( health_check_random, search_strategy.draw_parameter( health_check_random, )), lambda *args, **kwargs: None, )) count += 1 except BadTemplateDraw: bad_draws += 1 except UnsatisfiedAssumption: filtered_draws += 1 except Exception: if errors == 0: report(traceback.format_exc()) errors += 1 if test_runner is default_executor: fail_health_check( 'An exception occurred during data ' 'generation in initial health check. ' 'This indicates a bug in the strategy. ' 'This could either be a Hypothesis bug or ' "an error in a function you've passed to " 'it to construct your data.' ) else: fail_health_check( 'An exception occurred during data ' 'generation in initial health check. ' 'This indicates a bug in the strategy. ' 'This could either be a Hypothesis bug or ' 'an error in a function you\'ve passed to ' 'it to construct your data. Additionally, ' 'you have a custom executor, which means ' 'that this could be your executor failing ' 'to handle a function which returns None. ' ) if filtered_draws >= 50: fail_health_check(( 'It looks like your strategy is filtering out a lot ' 'of data. Health check found %d filtered examples but ' 'only %d good ones. This will make your tests much ' 'slower, and also will probably distort the data ' 'generation quite a lot. You should adapt your ' 'strategy to filter less.') % ( filtered_draws, count )) if bad_draws >= 50: fail_health_check( 'Hypothesis is struggling to generate examples. ' 'This is often a sign of a recursive strategy which ' 'fans out too broadly. If you\'re using recursive, ' 'try to reduce the size of the recursive step or ' 'increase the maximum permitted number of leaves.' ) runtime = time.time() - start if runtime > 1.0 or count < 10: fail_health_check(( 'Data generation is extremely slow: Only produced ' '%d valid examples in %.2f seconds. Try decreasing ' "size of the data you're generating (with e.g." 'average_size or max_leaves parameters).' ) % (count, runtime)) if getglobalrandomstate() != initial_state: warned_random[0] = True fail_health_check( 'Data generation depends on global random module. ' 'This makes results impossible to replay, which ' 'prevents Hypothesis from working correctly. ' 'If you want to use methods from random, use ' 'randoms() from hypothesis.strategies to get an ' 'instance of Random you can use. Alternatively, you ' 'can use the random_module() strategy to explicitly ' 'seed the random module.' ) last_exception = [None] repr_for_last_exception = [None] def is_template_example(xs): if perform_health_check and not warned_random[0]: initial_state = getglobalrandomstate() record_repr = [None] try: result = test_runner(reify_and_execute( search_strategy, xs, test, record_repr=record_repr, )) if result is not None: note_deprecation(( 'Tests run under @given should return None, but ' '%s returned %r instead.' 'In Hypothesis 2.0 this will become an error.' ) % (test.__name__, result), settings) return False except HypothesisDeprecationWarning: raise except UnsatisfiedAssumption as e: raise e except Exception as e: last_exception[0] = traceback.format_exc() repr_for_last_exception[0] = record_repr[0] verbose_report(last_exception[0]) return True finally: if ( not warned_random[0] and perform_health_check and getglobalrandomstate() != initial_state ): warned_random[0] = True fail_health_check( 'Your test used the global random module. ' 'This is unlikely to work correctly. You should ' 'consider using the randoms() strategy from ' 'hypothesis.strategies instead. Alternatively, ' 'you can use the random_module() strategy to ' 'explicitly seed the random module.' ) is_template_example.__name__ = test.__name__ is_template_example.__qualname__ = qualname(test) falsifying_template = None try: falsifying_template = best_satisfying_template( search_strategy, random, is_template_example, settings, storage, start_time=start, ) except NoSuchExample: return assert last_exception[0] is not None with settings: test_runner(reify_and_execute( search_strategy, falsifying_template, test, print_example=True, is_final=True )) report( u'Failed to reproduce exception. Expected: \n' + last_exception[0], ) test_runner(reify_and_execute( search_strategy, falsifying_template, test_is_flaky(test, repr_for_last_exception[0]), print_example=True, is_final=True ))
def test_can_give_list_of_examples(self): strategy(lists(strat)).example()
def wrapped_test(*arguments, **kwargs): import hypothesis.strategies as sd from hypothesis.internal.strategymethod import strategy selfy = None arguments, kwargs = convert_positional_arguments( wrapped_test, arguments, kwargs) # Anything in unused_kwargs hasn't been injected through # argspec.defaults, so we need to add them. for k in unused_kwargs: if k not in kwargs: kwargs[k] = unused_kwargs[k] # If the test function is a method of some kind, the bound object # will be the first named argument if there are any, otherwise the # first vararg (if any). if argspec.args: selfy = kwargs.get(argspec.args[0]) elif arguments: selfy = arguments[0] if isinstance(selfy, HypothesisProvided): selfy = None test_runner = executor(selfy) for example in getattr( wrapped_test, u'hypothesis_explicit_examples', () ): if example.args: example_kwargs = dict(zip( argspec.args[-len(example.args):], example.args )) else: example_kwargs = dict(example.kwargs) for k, v in kwargs.items(): if not isinstance(v, HypothesisProvided): example_kwargs[k] = v # Note: Test may mutate arguments and we can't rerun explicit # examples, so we have to calculate the failure message at this # point rather than than later. message_on_failure = u'Falsifying example: %s(%s)' % ( test.__name__, arg_string(test, arguments, example_kwargs) ) try: test_runner( lambda: test(*arguments, **example_kwargs) ) except BaseException: report(message_on_failure) raise if not any( isinstance(x, HypothesisProvided) for xs in (arguments, kwargs.values()) for x in xs ): # All arguments have been satisfied without needing to invoke # hypothesis test_runner(lambda: test(*arguments, **kwargs)) return def convert_to_specifier(v): if isinstance(v, HypothesisProvided): return strategy(v.value, settings) else: return sd.just(v) given_specifier = sd.tuples( sd.tuples(*map(convert_to_specifier, arguments)), sd.fixed_dictionaries(dict( (k, convert_to_specifier(v)) for (k, v) in kwargs.items())) ) search_strategy = strategy(given_specifier, settings) if settings.database: storage = settings.database.storage( fully_qualified_name(test)) else: storage = None last_exception = [None] repr_for_last_exception = [None] def is_template_example(xs): record_repr = [None] try: test_runner(reify_and_execute( search_strategy, xs, test, always_print=settings.max_shrinks <= 0, record_repr=record_repr, )) return False except UnsatisfiedAssumption as e: raise e except Exception as e: if settings.max_shrinks <= 0: raise e last_exception[0] = traceback.format_exc() repr_for_last_exception[0] = record_repr[0] verbose_report(last_exception[0]) return True is_template_example.__name__ = test.__name__ is_template_example.__qualname__ = qualname(test) falsifying_template = None try: falsifying_template = best_satisfying_template( search_strategy, random, is_template_example, settings, storage ) except NoSuchExample: return assert last_exception[0] is not None with settings: test_runner(reify_and_execute( search_strategy, falsifying_template, test, print_example=True, is_final=True )) report( u'Failed to reproduce exception. Expected: \n' + last_exception[0], ) test_runner(reify_and_execute( search_strategy, falsifying_template, test_is_flaky(test, repr_for_last_exception[0]), print_example=True, is_final=True ))
def templates_for_strategy(specifier, settings): return TemplatesStrategy(strategy(specifier.base, settings))
def wrapped_test(*arguments, **kwargs): if settings.derandomize: random = Random(function_digest(test)) else: random = provided_random or new_random() import hypothesis.strategies as sd from hypothesis.internal.strategymethod import strategy selfy = None arguments, kwargs = convert_positional_arguments( wrapped_test, arguments, kwargs) for arg in hypothesis_owned_arguments: try: value = kwargs[arg] except KeyError: continue if not isinstance(value, HypothesisProvided): note_deprecation( 'Passing in explicit values to override Hypothesis ' 'provided values is deprecated and will no longer ' 'work in Hypothesis 2.0. If you need to do this, ' 'extract a common function and call that from a ' 'Hypothesis based test.', settings) # Anything in unused_kwargs hasn't been injected through # argspec.defaults, so we need to add them. for k in unused_kwargs: if k not in kwargs: kwargs[k] = unused_kwargs[k] # If the test function is a method of some kind, the bound object # will be the first named argument if there are any, otherwise the # first vararg (if any). if argspec.args: selfy = kwargs.get(argspec.args[0]) elif arguments: selfy = arguments[0] if isinstance(selfy, HypothesisProvided): selfy = None test_runner = executor(selfy) for example in reversed( getattr(wrapped_test, u'hypothesis_explicit_examples', ())): if example.args: example_kwargs = dict( zip(argspec.args[-len(example.args):], example.args)) else: example_kwargs = dict(example.kwargs) for k, v in kwargs.items(): if not isinstance(v, HypothesisProvided): example_kwargs[k] = v # Note: Test may mutate arguments and we can't rerun explicit # examples, so we have to calculate the failure message at this # point rather than than later. message_on_failure = u'Falsifying example: %s(%s)' % ( test.__name__, arg_string(test, arguments, example_kwargs)) try: with BuildContext() as b: test_runner(lambda: test(*arguments, **example_kwargs)) except BaseException: report(message_on_failure) for n in b.notes: report(n) raise if not any( isinstance(x, HypothesisProvided) for xs in (arguments, kwargs.values()) for x in xs): # All arguments have been satisfied without needing to invoke # hypothesis test_runner(lambda: test(*arguments, **kwargs)) return def convert_to_specifier(v): if isinstance(v, HypothesisProvided): return strategy(v.value, settings) else: return sd.just(v) given_specifier = sd.tuples( sd.tuples(*map(convert_to_specifier, arguments)), sd.fixed_dictionaries( dict((k, convert_to_specifier(v)) for (k, v) in kwargs.items()))) def fail_health_check(message): message += ( '\nSee http://hypothesis.readthedocs.org/en/latest/health' 'checks.html for more information about this.') if settings.strict: raise FailedHealthCheck(message) else: warnings.warn(FailedHealthCheck(message)) search_strategy = strategy(given_specifier, settings) search_strategy.validate() if settings.database: storage = settings.database.storage(fully_qualified_name(test)) else: storage = None start = time.time() warned_random = [False] perform_health_check = settings.perform_health_check if Settings.default is not None: perform_health_check &= Settings.default.perform_health_check if perform_health_check: initial_state = getglobalrandomstate() health_check_random = Random(random.getrandbits(128)) count = 0 bad_draws = 0 filtered_draws = 0 errors = 0 while (count < 10 and time.time() < start + 1 and filtered_draws < 50 and bad_draws < 50): try: with Settings(settings, verbosity=Verbosity.quiet): test_runner( reify_and_execute( search_strategy, search_strategy.draw_template( health_check_random, search_strategy.draw_parameter( health_check_random, )), lambda *args, **kwargs: None, )) count += 1 except BadTemplateDraw: bad_draws += 1 except UnsatisfiedAssumption: filtered_draws += 1 except Exception: if errors == 0: report(traceback.format_exc()) errors += 1 if test_runner is default_executor: fail_health_check( 'An exception occurred during data ' 'generation in initial health check. ' 'This indicates a bug in the strategy. ' 'This could either be a Hypothesis bug or ' "an error in a function you've passed to " 'it to construct your data.') else: fail_health_check( 'An exception occurred during data ' 'generation in initial health check. ' 'This indicates a bug in the strategy. ' 'This could either be a Hypothesis bug or ' 'an error in a function you\'ve passed to ' 'it to construct your data. Additionally, ' 'you have a custom executor, which means ' 'that this could be your executor failing ' 'to handle a function which returns None. ') if filtered_draws >= 50: fail_health_check(( 'It looks like your strategy is filtering out a lot ' 'of data. Health check found %d filtered examples but ' 'only %d good ones. This will make your tests much ' 'slower, and also will probably distort the data ' 'generation quite a lot. You should adapt your ' 'strategy to filter less.') % (filtered_draws, count)) if bad_draws >= 50: fail_health_check( 'Hypothesis is struggling to generate examples. ' 'This is often a sign of a recursive strategy which ' 'fans out too broadly. If you\'re using recursive, ' 'try to reduce the size of the recursive step or ' 'increase the maximum permitted number of leaves.') runtime = time.time() - start if runtime > 1.0 or count < 10: fail_health_check( ('Data generation is extremely slow: Only produced ' '%d valid examples in %.2f seconds. Try decreasing ' "size of the data you're generating (with e.g." 'average_size or max_leaves parameters).') % (count, runtime)) if getglobalrandomstate() != initial_state: warned_random[0] = True fail_health_check( 'Data generation depends on global random module. ' 'This makes results impossible to replay, which ' 'prevents Hypothesis from working correctly. ' 'If you want to use methods from random, use ' 'randoms() from hypothesis.strategies to get an ' 'instance of Random you can use. Alternatively, you ' 'can use the random_module() strategy to explicitly ' 'seed the random module.') last_exception = [None] repr_for_last_exception = [None] def is_template_example(xs): if perform_health_check and not warned_random[0]: initial_state = getglobalrandomstate() record_repr = [None] try: result = test_runner( reify_and_execute( search_strategy, xs, test, record_repr=record_repr, )) if result is not None: note_deprecation( ('Tests run under @given should return None, but ' '%s returned %r instead.' 'In Hypothesis 2.0 this will become an error.') % (test.__name__, result), settings) return False except HypothesisDeprecationWarning: raise except UnsatisfiedAssumption as e: raise e except Exception as e: last_exception[0] = traceback.format_exc() repr_for_last_exception[0] = record_repr[0] verbose_report(last_exception[0]) return True finally: if (not warned_random[0] and perform_health_check and getglobalrandomstate() != initial_state): warned_random[0] = True fail_health_check( 'Your test used the global random module. ' 'This is unlikely to work correctly. You should ' 'consider using the randoms() strategy from ' 'hypothesis.strategies instead. Alternatively, ' 'you can use the random_module() strategy to ' 'explicitly seed the random module.') is_template_example.__name__ = test.__name__ is_template_example.__qualname__ = qualname(test) falsifying_template = None try: falsifying_template = best_satisfying_template( search_strategy, random, is_template_example, settings, storage, start_time=start, ) except NoSuchExample: return assert last_exception[0] is not None with settings: test_runner( reify_and_execute(search_strategy, falsifying_template, test, print_example=True, is_final=True)) report( u'Failed to reproduce exception. Expected: \n' + last_exception[0], ) test_runner( reify_and_execute(search_strategy, falsifying_template, test_is_flaky( test, repr_for_last_exception[0]), print_example=True, is_final=True))
def templates_for(strat): return TemplatesStrategy(strategy(strat))
def strategy_test_suite( specifier, max_examples=10, random=None, ): settings = Settings( database=None, max_examples=max_examples, max_iterations=max_examples * 2, min_satisfying_examples=2, average_list_length=2.0, ) random = random or Random() strat = strategy(specifier, settings) specifier_test = given(templates_for(specifier), randoms(), settings=settings) class ValidationSuite(TestCase): def __repr__(self): return u'strategy_test_suite(%s)' % (repr(specifier), ) @given(specifier, settings=settings) def test_does_not_error(self, value): pass def test_can_give_example(self): strat.example() def test_can_give_list_of_examples(self): strategy(lists(strat)).example() def test_will_give_unsatisfiable_if_all_rejected(self): @given(specifier, settings=settings) def nope(x): assume(False) self.assertRaises(Unsatisfiable, nope) def test_will_find_a_constant_failure(self): @given(specifier, settings=settings) def nope(x): raise Rejected() self.assertRaises(Rejected, nope) def test_will_find_a_failure_from_the_database(self): db = ExampleDatabase() @given(specifier, settings=Settings(settings, max_examples=10, database=db)) def nope(x): raise Rejected() try: for i in hrange(3): self.assertRaises(Rejected, nope) # pragma: no cover finally: db.close() @given(templates_for(specifier), templates_for(specifier), settings=settings) def test_simplicity_is_asymmetric(self, x, y): assert not (strat.strictly_simpler(x, y) and strat.strictly_simpler(y, x)) @given(integers(), settings=settings) def test_templates_generated_from_same_random_are_equal(self, i): try: t1 = strat.draw_and_produce(Random(i)) t2 = strat.draw_and_produce(Random(i)) except BadTemplateDraw: assume(False) if t1 is not t2: assert t1 == t2 assert hash(t1) == hash(t2) @given(integers(), settings=settings) def test_templates_generated_from_same_random_are_equal_after_reify( self, i): try: t1 = strat.draw_and_produce(Random(i)) t2 = strat.draw_and_produce(Random(i)) except BadTemplateDraw: assume(False) if t1 is not t2: with BuildContext(): strat.reify(t1) with BuildContext(): strat.reify(t2) assert t1 == t2 assert hash(t1) == hash(t2) @given(randoms(), settings=settings) def test_will_handle_a_really_weird_failure(self, rnd): db = ExampleDatabase() @given(specifier, settings=Settings( settings, database=db, max_examples=max_examples, min_satisfying_examples=2, average_list_length=2.0, ), random=rnd) def nope(x): s = hashlib.sha1(repr(x).encode(u'utf-8')).digest() assert Random(s).randint(0, 1) == Random(s).randint(0, 1) if Random(s).randint(0, 1): raise Rejected(u'%r with digest %r' % (x, s)) try: try: nope() except Rejected: pass try: nope() except Rejected: pass finally: db.close() @specifier_test def test_is_basic(self, value, rnd): def is_basic(v): if v is None or isinstance(v, text_type): return True if isinstance(v, integer_types): return not (abs(v) >> 64) if isinstance(v, list): return all(is_basic(w) for w in v) return False supposedly_basic = strat.to_basic(value) self.assertTrue(is_basic(supposedly_basic), repr(supposedly_basic)) @specifier_test def test_only_raises_bad_data_in_from_basic(self, value, rnd): basic = strat.to_basic(value) messed_basic = mutate_basic(basic, rnd) try: strat.from_basic(messed_basic) except BadData: pass @specifier_test def test_can_round_trip_through_the_database(self, template, rnd): empty_db = ExampleDatabase(backend=SQLiteBackend(u':memory:'), ) try: storage = empty_db.storage(u'round trip') storage.save(template, strat) values = list(storage.fetch(strat)) assert len(values) == 1 assert strat.to_basic(template) == strat.to_basic(values[0]) finally: empty_db.close() @specifier_test def test_template_is_hashable(self, template, rnd): hash(template) # It can be easy to forget to convert a list... hash(strat.from_basic(strat.to_basic(template))) @specifier_test def test_can_minimize_to_empty(self, template, rnd): simplest = template tracker = Tracker() while True: for t in strat.full_simplify(rnd, simplest): if tracker.track(t) == 1: simplest = t break else: break assert list(strat.full_simplify(rnd, simplest)) == [] @specifier_test def test_full_simplify_completes(self, template, rnd): # Cut off at 100 for the occasional case where we get # really very large templates which have too many simplifies. for x in islice(strat.full_simplify(rnd, template), 100): pass @specifier_test def test_does_not_increase_complexity(self, template, rnd): for s in islice(strat.full_simplify(rnd, template), 100): assert not strat.strictly_simpler(template, s) @given(randoms(), settings=Settings(settings, max_examples=100)) def test_can_create_templates(self, random): parameter = strat.draw_parameter(random) try: strat.draw_template(random, parameter) except BadTemplateDraw: assume(False) return ValidationSuite
def wrapped_test(*arguments, **kwargs): import hypothesis.strategies as sd from hypothesis.internal.strategymethod import strategy selfy = None arguments, kwargs = convert_positional_arguments( wrapped_test, arguments, kwargs) # Anything in unused_kwargs hasn't been injected through # argspec.defaults, so we need to add them. for k in unused_kwargs: if k not in kwargs: kwargs[k] = unused_kwargs[k] # If the test function is a method of some kind, the bound object # will be the first named argument if there are any, otherwise the # first vararg (if any). if argspec.args: selfy = kwargs.get(argspec.args[0]) elif arguments: selfy = arguments[0] if isinstance(selfy, HypothesisProvided): selfy = None test_runner = executor(selfy) for example in getattr(wrapped_test, u'hypothesis_explicit_examples', ()): if example.args: example_kwargs = dict( zip(argspec.args[-len(example.args):], example.args)) else: example_kwargs = dict(example.kwargs) for k, v in kwargs.items(): if not isinstance(v, HypothesisProvided): example_kwargs[k] = v # Note: Test may mutate arguments and we can't rerun explicit # examples, so we have to calculate the failure message at this # point rather than than later. message_on_failure = u'Falsifying example: %s(%s)' % ( test.__name__, arg_string(test, arguments, example_kwargs)) try: test_runner(lambda: test(*arguments, **example_kwargs)) except BaseException: report(message_on_failure) raise if not any( isinstance(x, HypothesisProvided) for xs in (arguments, kwargs.values()) for x in xs): # All arguments have been satisfied without needing to invoke # hypothesis test_runner(lambda: test(*arguments, **kwargs)) return def convert_to_specifier(v): if isinstance(v, HypothesisProvided): return strategy(v.value, settings) else: return sd.just(v) given_specifier = sd.tuples( sd.tuples(*map(convert_to_specifier, arguments)), sd.fixed_dictionaries( dict((k, convert_to_specifier(v)) for (k, v) in kwargs.items()))) search_strategy = strategy(given_specifier, settings) if settings.database: storage = settings.database.storage(fully_qualified_name(test)) else: storage = None last_exception = [None] repr_for_last_exception = [None] def is_template_example(xs): record_repr = [None] try: test_runner( reify_and_execute( search_strategy, xs, test, always_print=settings.max_shrinks <= 0, record_repr=record_repr, )) return False except UnsatisfiedAssumption as e: raise e except Exception as e: if settings.max_shrinks <= 0: raise e last_exception[0] = traceback.format_exc() repr_for_last_exception[0] = record_repr[0] verbose_report(last_exception[0]) return True is_template_example.__name__ = test.__name__ is_template_example.__qualname__ = qualname(test) falsifying_template = None try: falsifying_template = best_satisfying_template( search_strategy, random, is_template_example, settings, storage) except NoSuchExample: return assert last_exception[0] is not None with settings: test_runner( reify_and_execute(search_strategy, falsifying_template, test, print_example=True, is_final=True)) report( u'Failed to reproduce exception. Expected: \n' + last_exception[0], ) test_runner( reify_and_execute(search_strategy, falsifying_template, test_is_flaky( test, repr_for_last_exception[0]), print_example=True, is_final=True))
def convert_to_specifier(v): if isinstance(v, HypothesisProvided): return strategy(v.value, settings) else: return sd.just(v)
def find(specifier, condition, settings=None, random=None, storage=None): settings = settings or Settings( max_examples=2000, min_satisfying_examples=0, max_shrinks=2000, ) from hypothesis.internal.strategymethod import strategy search = strategy(specifier, settings) if storage is None and settings.database is not None: storage = settings.database.storage( u'find(%s)' % (binascii.hexlify(function_digest(condition)).decode(u'ascii'), )) random = random or Random() successful_examples = [0] def template_condition(template): with BuildContext(): result = search.reify(template) success = condition(result) if success: successful_examples[0] += 1 if not successful_examples[0]: verbose_report(lambda: u'Trying example %s' % (repr(result), )) elif success: if successful_examples[0] == 1: verbose_report(lambda: u'Found satisfying example %s' % (repr(result), )) else: verbose_report(lambda: u'Shrunk example to %s' % (repr(result), )) return success template_condition.__name__ = condition.__name__ tracker = Tracker() try: template = best_satisfying_template( search, random, template_condition, settings, tracker=tracker, max_parameter_tries=2, storage=storage, ) with BuildContext(is_final=True, close_on_capture=False): return search.reify(template) except Timeout: raise except NoSuchExample: if search.template_upper_bound <= len(tracker): raise DefinitelyNoSuchExample( get_pretty_function_description(condition), search.template_upper_bound, ) raise NoSuchExample(get_pretty_function_description(condition))
def strategy_test_suite( specifier, max_examples=10, random=None, ): settings = Settings( database=None, max_examples=max_examples, max_iterations=max_examples * 2, min_satisfying_examples=2, average_list_length=2.0, ) random = random or Random() strat = strategy(specifier, settings) specifier_test = given( templates_for(specifier), randoms(), settings=settings ) class ValidationSuite(TestCase): def __repr__(self): return u'strategy_test_suite(%s)' % ( repr(specifier), ) @given(specifier, settings=settings) def test_does_not_error(self, value): pass def test_can_give_example(self): strat.example() def test_can_give_list_of_examples(self): strategy(lists(strat)).example() def test_will_give_unsatisfiable_if_all_rejected(self): @given(specifier, settings=settings) def nope(x): assume(False) self.assertRaises(Unsatisfiable, nope) def test_will_find_a_constant_failure(self): @given(specifier, settings=settings) def nope(x): raise Rejected() self.assertRaises(Rejected, nope) def test_will_find_a_failure_from_the_database(self): db = ExampleDatabase() @given(specifier, settings=Settings( settings, max_examples=10, database=db)) def nope(x): raise Rejected() try: for i in hrange(3): self.assertRaises(Rejected, nope) # pragma: no cover finally: db.close() @given( templates_for(specifier), templates_for(specifier), settings=settings ) def test_simplicity_is_asymmetric(self, x, y): assert not ( strat.strictly_simpler(x, y) and strat.strictly_simpler(y, x) ) @given(integers(), settings=settings) def test_templates_generated_from_same_random_are_equal(self, i): try: t1 = strat.draw_and_produce(Random(i)) t2 = strat.draw_and_produce(Random(i)) except BadTemplateDraw: assume(False) if t1 is not t2: assert t1 == t2 assert hash(t1) == hash(t2) @given(integers(), settings=settings) def test_templates_generated_from_same_random_are_equal_after_reify( self, i ): try: t1 = strat.draw_and_produce(Random(i)) t2 = strat.draw_and_produce(Random(i)) except BadTemplateDraw: assume(False) if t1 is not t2: with BuildContext(): strat.reify(t1) with BuildContext(): strat.reify(t2) assert t1 == t2 assert hash(t1) == hash(t2) @given(randoms(), settings=settings) def test_will_handle_a_really_weird_failure(self, rnd): db = ExampleDatabase() @given( specifier, settings=Settings( settings, database=db, max_examples=max_examples, min_satisfying_examples=2, average_list_length=2.0, ), random=rnd ) def nope(x): s = hashlib.sha1(repr(x).encode(u'utf-8')).digest() assert Random(s).randint(0, 1) == Random(s).randint(0, 1) if Random(s).randint(0, 1): raise Rejected(u'%r with digest %r' % ( x, s )) try: try: nope() except Rejected: pass try: nope() except Rejected: pass finally: db.close() @specifier_test def test_is_basic(self, value, rnd): def is_basic(v): if v is None or isinstance(v, text_type): return True if isinstance(v, integer_types): return not (abs(v) >> 64) if isinstance(v, list): return all(is_basic(w) for w in v) return False supposedly_basic = strat.to_basic(value) self.assertTrue(is_basic(supposedly_basic), repr(supposedly_basic)) @specifier_test def test_only_raises_bad_data_in_from_basic(self, value, rnd): basic = strat.to_basic(value) messed_basic = mutate_basic(basic, rnd) try: strat.from_basic(messed_basic) except BadData: pass @specifier_test def test_can_round_trip_through_the_database(self, template, rnd): empty_db = ExampleDatabase( backend=SQLiteBackend(u':memory:'), ) try: storage = empty_db.storage(u'round trip') storage.save(template, strat) values = list(storage.fetch(strat)) assert len(values) == 1 assert strat.to_basic(template) == strat.to_basic(values[0]) finally: empty_db.close() @specifier_test def test_template_is_hashable(self, template, rnd): hash(template) # It can be easy to forget to convert a list... hash(strat.from_basic(strat.to_basic(template))) @specifier_test def test_can_minimize_to_empty(self, template, rnd): simplest = template tracker = Tracker() while True: for t in strat.full_simplify(rnd, simplest): if tracker.track(t) == 1: simplest = t break else: break assert list(strat.full_simplify(rnd, simplest)) == [] @specifier_test def test_full_simplify_completes(self, template, rnd): # Cut off at 100 for the occasional case where we get # really very large templates which have too many simplifies. for x in islice(strat.full_simplify(rnd, template), 100): pass @specifier_test def test_does_not_increase_complexity(self, template, rnd): for s in islice(strat.full_simplify(rnd, template), 100): assert not strat.strictly_simpler(template, s) @given(randoms(), settings=Settings(settings, max_examples=100)) def test_can_create_templates(self, random): parameter = strat.draw_parameter(random) try: strat.draw_template(random, parameter) except BadTemplateDraw: assume(False) return ValidationSuite