def test_errors_if_you_mark_bad_before_fetching(): source = ParameterSource( random=random.Random(), strategy=integers(), ) with pytest.raises(ValueError): source.mark_bad()
def test_can_use_none_max_tries(): source = ParameterSource( random=random.Random(), strategy=integers(), max_tries=None, ) source.pick_a_parameter()
def test_errors_if_you_mark_bad_before_fetching(): source = ParameterSource( context=BuildContext(random.Random()), strategy=strategy(int), ) with pytest.raises(ValueError): source.mark_bad()
def test_errors_if_you_mark_bad_twice(): source = ParameterSource( random=random.Random(), strategy=integers(), ) next(iter(source)) source.mark_bad() with pytest.raises(ValueError): source.mark_bad()
def test_errors_if_you_mark_bad_twice(): source = ParameterSource( context=BuildContext(random.Random()), strategy=strategy(int), ) next(iter(source)) source.mark_bad() with pytest.raises(ValueError): source.mark_bad()
def test_tries_each_parameter_at_least_min_index_times(): source = ParameterSource(context=BuildContext(random.Random()), strategy=strategy(int), min_tries=5) i = 0 for x in source.examples(): i += 1 if i > 500: break if i % 2: source.mark_bad() # The last index may not have been fully populated assert all(c >= 5 for c in source.counts[:-1])
def test_negative_is_not_too_far_off_mean(): source = ParameterSource( random=random.Random(), strategy=integers(), ) positive = 0 i = 0 for example in source.examples(): if example >= 0: positive += 1 i += 1 if i >= N_EXAMPLES: break assert 0.3 <= float(positive) / N_EXAMPLES <= 0.7
def test_marking_negative_avoids_similar_examples(): positive = 0 k = 10 for _ in hrange(k): source = ParameterSource( random=random.Random(), strategy=integers(), ) n = N_EXAMPLES // k for example in islice(source.examples(), n): if example >= 0: positive += 1 else: source.mark_bad() assert float(positive) / N_EXAMPLES >= 0.7
def test_marking_negative_avoids_similar_examples(): source = ParameterSource( context=BuildContext(random.Random()), strategy=strategy(int), ) positive = 0 i = 0 for example in source.examples(): if example >= 0: positive += 1 else: source.mark_bad() i += 1 if i >= N_EXAMPLES: break assert float(positive) / N_EXAMPLES >= 0.8
def test_can_grow_the_set_of_available_parameters_if_doing_badly(): runs = 10 number_grown = 0 for _ in hrange(runs): source = ParameterSource( context=BuildContext(random.Random()), strategy=strategy(int), min_parameters=1, ) i = 0 for example in source.examples(): if example < 0: source.mark_bad() i += 1 if i >= 100: break if len(source.parameters) > 1: number_grown += 1 assert len(source.parameters) < 100
def find_satisfying_template(search_strategy, random, condition, tracker, settings, storage=None): """Attempt to find a template for search_strategy such that condition is truthy. Exceptions other than UnsatisfiedAssumption will be immediately propagated. UnsatisfiedAssumption will indicate that similar examples should be avoided in future. Returns such a template as soon as it is found, otherwise stops after settings.max_examples examples have been considered or settings.timeout seconds have passed (if settings.timeout > 0). May raise a variety of exceptions depending on exact circumstances, but these will all subclass either Unsatisfiable (to indicate not enough examples were found which did not raise UnsatisfiedAssumption to consider this a valid test) or NoSuchExample (to indicate that this probably means that condition is true with very high probability). """ satisfying_examples = 0 timed_out = False max_examples = settings.max_examples min_satisfying_examples = settings.min_satisfying_examples start_time = time.time() if storage: for example in storage.fetch(): if time_to_call_it_a_day(settings, start_time): break tracker.track(example) try: if condition(example): return example satisfying_examples += 1 except UnsatisfiedAssumption: pass build_context = BuildContext(random) parameter_source = ParameterSource(context=build_context, strategy=search_strategy, min_parameters=max( 2, int(float(max_examples) / 10))) for parameter in islice(parameter_source, max_examples - len(tracker)): if len(tracker) >= search_strategy.size_upper_bound: break if time_to_call_it_a_day(settings, start_time): break example = search_strategy.produce_template(build_context, parameter) if tracker.track(example) > 1: parameter_source.mark_bad() continue try: if condition(example): return example except UnsatisfiedAssumption: parameter_source.mark_bad() continue satisfying_examples += 1 run_time = time.time() - start_time timed_out = settings.timeout >= 0 and run_time >= settings.timeout if (satisfying_examples and len(tracker) >= search_strategy.size_lower_bound): raise DefinitelyNoSuchExample( get_pretty_function_description(condition), satisfying_examples, ) elif satisfying_examples < min_satisfying_examples: if timed_out: raise Timeout(condition, satisfying_examples, run_time) else: raise Unsatisfiable(condition, satisfying_examples, run_time) else: raise NoSuchExample(get_pretty_function_description(condition))
def find_satisfying_template( search_strategy, random, condition, tracker, settings, storage=None, max_parameter_tries=None, ): """Attempt to find a template for search_strategy such that condition is truthy. Exceptions other than UnsatisfiedAssumption will be immediately propagated. UnsatisfiedAssumption will indicate that similar examples should be avoided in future. Returns such a template as soon as it is found, otherwise stops after settings.max_examples examples have been considered or settings.timeout seconds have passed (if settings.timeout > 0). May raise a variety of exceptions depending on exact circumstances, but these will all subclass either Unsatisfiable (to indicate not enough examples were found which did not raise UnsatisfiedAssumption to consider this a valid test) or NoSuchExample (to indicate that this probably means that condition is true with very high probability). """ satisfying_examples = 0 examples_considered = 0 timed_out = False max_iterations = max(settings.max_iterations, settings.max_examples) max_examples = min(max_iterations, settings.max_examples) min_satisfying_examples = min( settings.min_satisfying_examples, max_examples, ) start_time = time.time() if storage: for example in storage.fetch(search_strategy): if examples_considered >= max_iterations: break examples_considered += 1 if time_to_call_it_a_day(settings, start_time): break tracker.track(example) try: if condition(example): return example satisfying_examples += 1 except UnsatisfiedAssumption: pass if satisfying_examples >= max_examples: break parameter_source = ParameterSource( random=random, strategy=search_strategy, max_tries=max_parameter_tries, ) assert search_strategy.template_upper_bound >= 0 if isinstance(search_strategy.template_upper_bound, float): assert math.isinf(search_strategy.template_upper_bound) else: assert isinstance(search_strategy.template_upper_bound, int) for parameter in parameter_source: # pragma: no branch if len(tracker) >= search_strategy.template_upper_bound: break if examples_considered >= max_iterations: break if satisfying_examples >= max_examples: break if time_to_call_it_a_day(settings, start_time): break examples_considered += 1 try: example = search_strategy.draw_template(random, parameter) except BadTemplateDraw: debug_report('Failed attempt to draw a template') parameter_source.mark_bad() continue if tracker.track(example) > 1: debug_report('Skipping duplicate example') parameter_source.mark_bad() continue try: if condition(example): return example except UnsatisfiedAssumption: parameter_source.mark_bad() continue satisfying_examples += 1 run_time = time.time() - start_time timed_out = settings.timeout >= 0 and run_time >= settings.timeout if (satisfying_examples and len(tracker) >= search_strategy.template_upper_bound): raise DefinitelyNoSuchExample( get_pretty_function_description(condition), satisfying_examples, ) elif satisfying_examples < min_satisfying_examples: if timed_out: raise Timeout( ('Ran out of time before finding a satisfying example for ' '%s. Only found %d examples (%d satisfying assumptions) in ' + '%.2fs.') % (get_pretty_function_description(condition), len(tracker), satisfying_examples, run_time)) else: raise Unsatisfiable( ('Unable to satisfy assumptions of hypothesis %s. ' + 'Only %d out of %d examples considered satisfied assumptions') % (get_pretty_function_description(condition), satisfying_examples, len(tracker))) else: raise NoSuchExample(get_pretty_function_description(condition))
def falsify(self, hypothesis, *argument_types, **kwargs): # pylint: disable=too-many-locals,too-many-branches """ Attempt to construct an example tuple x matching argument_types such that hypothesis(*x) returns a falsey value """ teardown_example = kwargs.get('teardown_example') or (lambda x: None) setup_example = kwargs.get('setup_example') or (lambda: None) random = self.random if random is None: random = Random(function_digest(hypothesis)) build_context = BuildContext(random) search_strategy = strategy(argument_types, self.settings) storage = None if self.database is not None: storage = self.database.storage_for(argument_types) def falsifies(args): # pylint: disable=missing-docstring example = None try: try: setup_example() example = search_strategy.reify(args) return not hypothesis(*example) except UnsatisfiedAssumption: return False finally: teardown_example(example) track_seen = Tracker() falsifying_examples = [] if storage: for example in storage.fetch(): track_seen.track(example) if falsifies(example): falsifying_examples = [example] break satisfying_examples = 0 timed_out = False max_examples = self.max_examples min_satisfying_examples = self.min_satisfying_examples parameter_source = ParameterSource(context=build_context, strategy=search_strategy, min_parameters=max( 2, int(float(max_examples) / 10))) start_time = time.time() def time_to_call_it_a_day(): """Have we exceeded our timeout?""" if self.timeout <= 0: return False return time.time() >= start_time + self.timeout for parameter in islice(parameter_source, max_examples - len(track_seen)): if len(track_seen) >= search_strategy.size_upper_bound: break if falsifying_examples: break if time_to_call_it_a_day(): break args = search_strategy.produce_template(build_context, parameter) if track_seen.track(args) > 1: parameter_source.mark_bad() continue try: setup_example() a = None try: a = search_strategy.reify(args) is_falsifying_example = not hypothesis(*a) finally: teardown_example(a) except UnsatisfiedAssumption: parameter_source.mark_bad() continue satisfying_examples += 1 if is_falsifying_example: falsifying_examples.append(args) run_time = time.time() - start_time timed_out = self.timeout >= 0 and run_time >= self.timeout if not falsifying_examples: if (satisfying_examples and len(track_seen) >= search_strategy.size_lower_bound): raise Exhausted(hypothesis, satisfying_examples) elif satisfying_examples < min_satisfying_examples: if timed_out: raise Timeout(hypothesis, satisfying_examples, run_time) else: raise Unsatisfiable(hypothesis, satisfying_examples, run_time) else: raise Unfalsifiable(hypothesis) for example in falsifying_examples: if not falsifies(example): raise Flaky(hypothesis, example) best_example = falsifying_examples[0] for simpler in search_strategy.simplify_such_that( random, best_example, falsifies, tracker=track_seen, ): best_example = simpler if time_to_call_it_a_day(): # We no cover in here because it's a bit sensitive to timing # and tends to make tests flaky. There are tests that mean # this is definitely covered most of the time. break # pragma: no cover if storage is not None: storage.save(best_example) setup_example() return search_strategy.reify(best_example)