def test_strategy_for_integer_range_produces_only_integers_in_that_range(): just_one_integer = strategy(specifiers.IntegerRange(1, 1)) for _ in hrange(100): pv = just_one_integer.draw_parameter(random) x = just_one_integer.produce_template(BuildContext(random), pv) assert x == 1 some_integers = strategy(specifiers.IntegerRange(1, 10)) for _ in hrange(100): pv = some_integers.produce_parameter(random) x = some_integers.produce_template(BuildContext(random), pv) assert 1 <= x <= 10
def test_errors_if_you_mark_bad_before_fetching(): source = ParameterSource( context=BuildContext(random.Random()), strategy=strategy(int), ) with pytest.raises(ValueError): source.mark_bad()
def test_can_perform_all_basic_operations(self, random): parameter = strat.draw_parameter(random) template = strat.draw_template(BuildContext(random), parameter) minimal_template = list( strat.simplify_such_that(template, lambda x: True))[-1] strat.reify(minimal_template) assert (strat.to_basic(minimal_template) == strat.to_basic( strat.from_basic(strat.to_basic(minimal_template))))
def new_template(self, seed, parameter_seed): context = BuildContext(Random(seed)) parameter = self.source_strategy.draw_parameter(Random(parameter_seed)) def templates(): while True: yield self.source_strategy.draw_template(context, parameter) return StreamTemplate(seed, parameter_seed, templates())
def test_errors_if_you_mark_bad_twice(): source = ParameterSource( context=BuildContext(random.Random()), strategy=strategy(int), ) next(iter(source)) source.mark_bad() with pytest.raises(ValueError): source.mark_bad()
def test_can_perform_all_basic_operations(specifier, random): strat = strategy(specifier, settings) parameter = strat.draw_parameter(random) template = strat.produce_template(BuildContext(random), parameter) assert (strat.to_basic(template) == strat.to_basic( strat.from_basic(strat.to_basic(template)))) minimal_template = last(strat.simplify_such_that(template, lambda x: True)) strat.reify(minimal_template) assert (strat.to_basic(minimal_template) == strat.to_basic( strat.from_basic(strat.to_basic(minimal_template))))
def test_named_tuples_always_produce_named_tuples(): s = strategy(Litter(int, int)) for i in hrange(100): assert isinstance( s.produce_template(BuildContext(random), s.produce_parameter(random)), Litter) for x in s.simplify(Litter(100, 100)): assert isinstance(x, Litter)
def test_strategy_for_integer_range_can_produce_end_points(): some_integers = strategy(specifiers.IntegerRange(1, 10)) found = set() for _ in hrange(1000): # pragma: no branch pv = some_integers.produce_parameter(random) x = some_integers.produce_template(BuildContext(random), pv) found.add(x) if 1 in found and 10 in found: break else: assert False # pragma: no cover assert 1 in found assert 10 in found
def test_tries_each_parameter_at_least_min_index_times(): source = ParameterSource(context=BuildContext(random.Random()), strategy=strategy(int), min_tries=5) i = 0 for x in source.examples(): i += 1 if i > 500: break if i % 2: source.mark_bad() # The last index may not have been fully populated assert all(c >= 5 for c in source.counts[:-1])
def test_negative_is_not_too_far_off_mean(): source = ParameterSource( context=BuildContext(random.Random()), strategy=strategy(int), ) positive = 0 i = 0 for example in source.examples(): if example >= 0: positive += 1 i += 1 if i >= N_EXAMPLES: break assert 0.3 <= float(positive) / N_EXAMPLES <= 0.7
def test_marking_negative_avoids_similar_examples(): source = ParameterSource( context=BuildContext(random.Random()), strategy=strategy(int), ) positive = 0 i = 0 for example in source.examples(): if example >= 0: positive += 1 else: source.mark_bad() i += 1 if i >= N_EXAMPLES: break assert float(positive) / N_EXAMPLES >= 0.8
def test_can_grow_the_set_of_available_parameters_if_doing_badly(): runs = 10 number_grown = 0 for _ in hrange(runs): source = ParameterSource( context=BuildContext(random.Random()), strategy=strategy(int), min_parameters=1, ) i = 0 for example in source.examples(): if example < 0: source.mark_bad() i += 1 if i >= 100: break if len(source.parameters) > 1: number_grown += 1 assert len(source.parameters) < 100
def run_test(): if condition is None: _condition = lambda x: True condition_string = '' else: _condition = condition condition_string = strip_lambda( reflection.get_pretty_function_description(condition)) count = 0 successful_runs = 0 s = strategy(specifier) for _ in hrange(MAX_RUNS): pv = s.draw_parameter(random) x = s.reify(s.draw_template(BuildContext(random), pv)) if not _condition(x): continue successful_runs += 1 if predicate(x): count += 1 if successful_runs < MIN_RUNS: raise ConditionTooHard( ('Unable to find enough examples satisfying predicate %s ' 'only found %d but required at least %d for validity') % (condition_string, successful_runs, MIN_RUNS)) result = Result( count, successful_runs, q, predicate, condition_string, ) p = cumulative_binomial_probability(successful_runs, q, count) run_test.test_result = result # The test passes if we fail to reject the null hypothesis that # the probability is at least q if p < REQUIRED_P: result.failed = True raise HypothesisFalsified(result.description() + ' rejected')
def test_can_recover_from_bad_data_in_mapped_strategy(r): param = OrderedPairs.draw_parameter(r) template = OrderedPairs.draw_template(BuildContext(r), param) OrderedPairs.reify(template) for simplification in OrderedPairs.full_simplify(r, template): if isinstance(simplification, OrderedPairs.TemplateFromTemplate): break else: assume(False) assume(isinstance(simplification, OrderedPairs.TemplateFromTemplate)) basic = OrderedPairs.to_basic(simplification) assert len(basic) == 4 assert isinstance(basic, list) assert isinstance(basic[-1], list) basic[-1] = 1 new_template = OrderedPairs.from_basic(basic) assert isinstance(new_template, OrderedPairs.TemplateFromBasic) reified = OrderedPairs.reify(new_template) assert type(reified) == tuple x, y = reified assert x < y
def simplify(self, davt): random = RandomWithSeed(davt.random) for d in self.specifier_strategy.simplify(davt.specifier): new_template = self.strategy( self.specifier_strategy.reify(d)).draw_and_produce( BuildContext(random)) yield DescriptorWithValue( specifier=d, template=new_template, value=None, random=davt.random, ) strat = self.strategy(self.specifier_strategy.reify(davt.specifier)) for v in strat.simplify(davt.template): yield DescriptorWithValue( specifier=davt.specifier, template=v, value=None, random=davt.random, )
def basic_simplify(self, random, davt): assert isinstance(random, Random) for d in self.specifier_strategy.full_simplify(random, davt.specifier): new_template = self.strategy( self.specifier_strategy.reify(d)).draw_and_produce( BuildContext(random)) yield DescriptorWithValue( specifier=d, template=new_template, value=None, random=davt.random, ) strat = self.strategy( self.specifier_strategy.reify(davt.specifier)) for v in strat.full_simplify(random, davt.template): yield DescriptorWithValue( specifier=davt.specifier, template=v, value=None, random=davt.random, )
def test_random_repr_has_seed(): strat = strategy(random.Random) rnd = strat.reify( strat.produce_template(BuildContext(random.Random()), None)) seed = rnd.seed assert text_type(seed) in repr(rnd)
def test_can_create_templates(self, random): parameter = strat.draw_parameter(random) strat.draw_template(BuildContext(random), parameter)
def test_float_strategy_does_not_overflow(): s = strategy(float) for _ in hrange(100): s.produce_template(BuildContext(random), s.produce_parameter(random))
def test_random_only_produces_special_random(): st = strategy(random.Random) assert isinstance( st.reify( st.produce_template(BuildContext(random), st.draw_parameter(random))), RandomWithSeed)
def test_template_is_hashable(specifier, random): strat = strategy(specifier, settings) parameter = strat.draw_parameter(random) template = strat.produce_template(BuildContext(random), parameter) hash(template)
def test_copies_all_its_values_correctly(desc, random): strat = strategy(desc, settings) value = strat.produce_template( BuildContext(random), strat.draw_parameter(random)) assert show(strat.reify(value)) == show(strat.reify(value))
def falsify(self, hypothesis, *argument_types, **kwargs): # pylint: disable=too-many-locals,too-many-branches """ Attempt to construct an example tuple x matching argument_types such that hypothesis(*x) returns a falsey value """ teardown_example = kwargs.get('teardown_example') or (lambda x: None) setup_example = kwargs.get('setup_example') or (lambda: None) random = self.random if random is None: random = Random(function_digest(hypothesis)) build_context = BuildContext(random) search_strategy = strategy(argument_types, self.settings) storage = None if self.database is not None: storage = self.database.storage_for(argument_types) def falsifies(args): # pylint: disable=missing-docstring example = None try: try: setup_example() example = search_strategy.reify(args) return not hypothesis(*example) except UnsatisfiedAssumption: return False finally: teardown_example(example) track_seen = Tracker() falsifying_examples = [] if storage: for example in storage.fetch(): track_seen.track(example) if falsifies(example): falsifying_examples = [example] break satisfying_examples = 0 timed_out = False max_examples = self.max_examples min_satisfying_examples = self.min_satisfying_examples parameter_source = ParameterSource(context=build_context, strategy=search_strategy, min_parameters=max( 2, int(float(max_examples) / 10))) start_time = time.time() def time_to_call_it_a_day(): """Have we exceeded our timeout?""" if self.timeout <= 0: return False return time.time() >= start_time + self.timeout for parameter in islice(parameter_source, max_examples - len(track_seen)): if len(track_seen) >= search_strategy.size_upper_bound: break if falsifying_examples: break if time_to_call_it_a_day(): break args = search_strategy.produce_template(build_context, parameter) if track_seen.track(args) > 1: parameter_source.mark_bad() continue try: setup_example() a = None try: a = search_strategy.reify(args) is_falsifying_example = not hypothesis(*a) finally: teardown_example(a) except UnsatisfiedAssumption: parameter_source.mark_bad() continue satisfying_examples += 1 if is_falsifying_example: falsifying_examples.append(args) run_time = time.time() - start_time timed_out = self.timeout >= 0 and run_time >= self.timeout if not falsifying_examples: if (satisfying_examples and len(track_seen) >= search_strategy.size_lower_bound): raise Exhausted(hypothesis, satisfying_examples) elif satisfying_examples < min_satisfying_examples: if timed_out: raise Timeout(hypothesis, satisfying_examples, run_time) else: raise Unsatisfiable(hypothesis, satisfying_examples, run_time) else: raise Unfalsifiable(hypothesis) for example in falsifying_examples: if not falsifies(example): raise Flaky(hypothesis, example) best_example = falsifying_examples[0] for simpler in search_strategy.simplify_such_that( random, best_example, falsifies, tracker=track_seen, ): best_example = simpler if time_to_call_it_a_day(): # We no cover in here because it's a bit sensitive to timing # and tends to make tests flaky. There are tests that mean # this is definitely covered most of the time. break # pragma: no cover if storage is not None: storage.save(best_example) setup_example() return search_strategy.reify(best_example)
def find_satisfying_template(search_strategy, random, condition, tracker, settings, storage=None): """Attempt to find a template for search_strategy such that condition is truthy. Exceptions other than UnsatisfiedAssumption will be immediately propagated. UnsatisfiedAssumption will indicate that similar examples should be avoided in future. Returns such a template as soon as it is found, otherwise stops after settings.max_examples examples have been considered or settings.timeout seconds have passed (if settings.timeout > 0). May raise a variety of exceptions depending on exact circumstances, but these will all subclass either Unsatisfiable (to indicate not enough examples were found which did not raise UnsatisfiedAssumption to consider this a valid test) or NoSuchExample (to indicate that this probably means that condition is true with very high probability). """ satisfying_examples = 0 timed_out = False max_examples = settings.max_examples min_satisfying_examples = settings.min_satisfying_examples start_time = time.time() if storage: for example in storage.fetch(): if time_to_call_it_a_day(settings, start_time): break tracker.track(example) try: if condition(example): return example satisfying_examples += 1 except UnsatisfiedAssumption: pass build_context = BuildContext(random) parameter_source = ParameterSource(context=build_context, strategy=search_strategy, min_parameters=max( 2, int(float(max_examples) / 10))) for parameter in islice(parameter_source, max_examples - len(tracker)): if len(tracker) >= search_strategy.size_upper_bound: break if time_to_call_it_a_day(settings, start_time): break example = search_strategy.produce_template(build_context, parameter) if tracker.track(example) > 1: parameter_source.mark_bad() continue try: if condition(example): return example except UnsatisfiedAssumption: parameter_source.mark_bad() continue satisfying_examples += 1 run_time = time.time() - start_time timed_out = settings.timeout >= 0 and run_time >= settings.timeout if (satisfying_examples and len(tracker) >= search_strategy.size_lower_bound): raise DefinitelyNoSuchExample( get_pretty_function_description(condition), satisfying_examples, ) elif satisfying_examples < min_satisfying_examples: if timed_out: raise Timeout(condition, satisfying_examples, run_time) else: raise Unsatisfiable(condition, satisfying_examples, run_time) else: raise NoSuchExample(get_pretty_function_description(condition))
def some_minimal_element(s): strat = strategy(s) template = strat.draw_and_produce(BuildContext(random)) for t in strat.simplify_such_that(template, lambda _: True): template = t return strat.reify(template)
def some_template(spec): return strategy(spec).draw_and_produce(BuildContext(Random()))
def test_string_strategy_produces_strings(): strings = strategy(text_type) result = strings.produce_template(BuildContext(random), strings.produce_parameter(random)) assert result is not None