def test_track_complex_with_nan(): t = Tracker() nan = float(u'nan') assert t.track(complex(nan, 2)) == 1 assert t.track(complex(nan, 2)) == 2 assert t.track(complex(0, nan)) == 1 assert t.track(complex(0, nan)) == 2 assert t.track(complex(nan, nan)) == 1 assert t.track(complex(nan, nan)) == 2
def test_track_complex_with_nan(): t = Tracker() nan = float('nan') assert t.track(complex(nan, 2)) == 1 assert t.track(complex(nan, 2)) == 2 assert t.track(complex(0, nan)) == 1 assert t.track(complex(0, nan)) == 2 assert t.track(complex(nan, nan)) == 1 assert t.track(complex(nan, nan)) == 2
def test_can_track_morphers(): t = Tracker() assert t.track(Morpher(0, 0)) == 1 assert t.track(Morpher(0, 0)) == 2 m1 = Morpher(0, 1) m2 = Morpher(0, 1) m1.become(s.lists(s.integers())) m2.become(s.lists(s.integers())) assert t.track(m1) == 1 assert t.track(m2) == 2
def test_can_track_morphers(): t = Tracker() assert t.track(Morpher(0, 0)) == 1 assert t.track(Morpher(0, 0)) == 2 m1 = Morpher(0, 1) m2 = Morpher(0, 1) m1.become(s.lists(s.integers())) m2.become(s.lists(s.integers())) assert t.track(m1) == 1 assert t.track(m2) == 2
def simplify_such_that(self, t, f): """Perform a greedy search to produce a "simplest" version of a template that satisfies some predicate. Care is taken to avoid cycles in simplify. f should produce the same result deterministically. This function may raise an error given f such that f(t) returns False sometimes and True some other times. """ if not f(t): raise ValueError( '%r does not satisfy predicate %s' % (t, f)) tracker = Tracker() yield t while True: simpler = self.simplify(t) for s in simpler: if tracker.track(s) > 1: continue if f(s): yield s t = s break else: break
def simplify_such_that(self, random, t, f, tracker=None): """Perform a greedy search to produce a "simplest" version of a template that satisfies some predicate. Care is taken to avoid cycles in simplify. f should produce the same result deterministically. This function may raise an error given f such that f(t) returns False sometimes and True some other times. """ assert isinstance(random, Random) if tracker is None: tracker = Tracker() yield t changed = True while changed: changed = False for simplify in self.simplifiers(t): while True: simpler = simplify(random, t) for s in simpler: if tracker.track(s) > 1: continue if f(s): changed = True yield s t = s break else: break
def simplify(self, x): t = Tracker() for cs in self.element_strategies: if cs.could_have_produced(x): for y in cs.simplify(x): if t.track(y) == 1: yield y
def simplify_such_that(self, random, t, f, tracker=None): """Perform a greedy search to produce a "simplest" version of a template that satisfies some predicate. Care is taken to avoid cycles in simplify. f should produce the same result deterministically. This function may raise an error given f such that f(t) returns False sometimes and True some other times. """ assert isinstance(random, Random) if tracker is None: tracker = Tracker() yield t changed = True while changed: changed = False for simplify in self.simplifiers(t): while True: simpler = simplify(random, t) for s in simpler: if tracker.track(s) > 1: continue if f(s): changed = True yield s t = s break else: break
def simplify_such_that(self, t, f): """Perform a greedy search to produce a "simplest" version of t that satisfies the predicate s. As each simpler version is found, yield it in turn. Stops when it has a value such that no value in simplify on the last value found satisfies f. Care is taken to avoid cycles in simplify. f should produce the same result deterministically. This function may raise an error given f such that f(t) returns False sometimes and True some other times. """ assert self.could_have_produced(t) if not f(t): raise ValueError('%r does not satisfy predicate %s' % (t, f)) tracker = Tracker() yield t while True: simpler = self.simplify(t) for s in simpler: assert self.could_have_produced(s) if tracker.track(s) > 1: continue if f(s): yield s t = s break else: break
def simplify_such_that(self, t, f): """Perform a greedy search to produce a "simplest" version of a template that satisfies some predicate. Care is taken to avoid cycles in simplify. f should produce the same result deterministically. This function may raise an error given f such that f(t) returns False sometimes and True some other times. """ if not f(t): raise ValueError('%r does not satisfy predicate %s' % (t, f)) tracker = Tracker() yield t while True: simpler = self.simplify(t) for s in simpler: if tracker.track(s) > 1: continue if f(s): yield s t = s break else: break
def test_can_minimize_to_empty(self, template, rnd): simplest = template tracker = Tracker() while True: for t in strat.full_simplify(rnd, simplest): if tracker.track(t) == 1: simplest = t break else: break assert list(strat.full_simplify(rnd, simplest)) == []
def test_can_minimize_to_empty(self, template, rnd): simplest = template tracker = Tracker() while True: for t in strat.full_simplify(rnd, simplest): if tracker.track(t) == 1: simplest = t break else: break assert list(strat.full_simplify(rnd, simplest)) == []
def minimal_element(strategy, random): tracker = Tracker() element = some_template(strategy, random) while True: for new_element in strategy.full_simplify(random, element): if tracker.track(new_element) > 1: continue try: strategy.reify(new_element) element = new_element break except UnsatisfiedAssumption: pass else: break return element
def minimal_element(strategy, random): tracker = Tracker() element = some_template(strategy, random) while True: for new_element in strategy.full_simplify(random, element): if tracker.track(new_element) > 1: continue try: strategy.reify(new_element) element = new_element break except UnsatisfiedAssumption: pass else: break return element
def test_track_ints(): t = Tracker() assert t.track(1) == 1 assert t.track(1) == 2
def test_tracking_classes_of_custom(): t = Tracker() assert t.track(Foo) == 1 assert t.track(Foo) == 2
def test_tracking_custom(): t = Tracker() assert t.track(Foo()) == 1 assert t.track(Foo()) == 2
def test_track_nan(): t = Tracker() assert t.track(float('nan')) == 1 assert t.track(float('nan')) == 2
def assert_no_duplicates_in_simplify(s, x): s = strategy(s) t = Tracker() t.track(x) for y in s.simplify(x): assert t.track(y) == 1
def test_track_nan(): t = Tracker() assert t.track(float(u'nan')) == 1 assert t.track(float(u'nan')) == 2
def test_track_dict(): t = Tracker() assert t.track({1: 2}) == 1 assert t.track({1: 3}) == 1
def test_track_iterables(): t = Tracker() assert t.track(iter([1])) == 1 assert t.track(iter([1])) == 2
def falsify(self, hypothesis, *argument_types, **kwargs): # pylint: disable=too-many-locals,too-many-branches """ Attempt to construct an example tuple x matching argument_types such that hypothesis(*x) returns a falsey value """ teardown_example = kwargs.get('teardown_example') or (lambda x: None) setup_example = kwargs.get('setup_example') or (lambda: None) random = self.random if random is None: random = Random(function_digest(hypothesis)) build_context = BuildContext(random) search_strategy = strategy(argument_types, self.settings) storage = None if self.database is not None: storage = self.database.storage_for(argument_types) def falsifies(args): # pylint: disable=missing-docstring example = None try: try: setup_example() example = search_strategy.reify(args) return not hypothesis(*example) except UnsatisfiedAssumption: return False finally: teardown_example(example) track_seen = Tracker() falsifying_examples = [] if storage: for example in storage.fetch(): track_seen.track(example) if falsifies(example): falsifying_examples = [example] break satisfying_examples = 0 timed_out = False max_examples = self.max_examples min_satisfying_examples = self.min_satisfying_examples parameter_source = ParameterSource(context=build_context, strategy=search_strategy, min_parameters=max( 2, int(float(max_examples) / 10))) start_time = time.time() def time_to_call_it_a_day(): """Have we exceeded our timeout?""" if self.timeout <= 0: return False return time.time() >= start_time + self.timeout for parameter in islice(parameter_source, max_examples - len(track_seen)): if len(track_seen) >= search_strategy.size_upper_bound: break if falsifying_examples: break if time_to_call_it_a_day(): break args = search_strategy.produce_template(build_context, parameter) if track_seen.track(args) > 1: parameter_source.mark_bad() continue try: setup_example() a = None try: a = search_strategy.reify(args) is_falsifying_example = not hypothesis(*a) finally: teardown_example(a) except UnsatisfiedAssumption: parameter_source.mark_bad() continue satisfying_examples += 1 if is_falsifying_example: falsifying_examples.append(args) run_time = time.time() - start_time timed_out = self.timeout >= 0 and run_time >= self.timeout if not falsifying_examples: if (satisfying_examples and len(track_seen) >= search_strategy.size_lower_bound): raise Exhausted(hypothesis, satisfying_examples) elif satisfying_examples < min_satisfying_examples: if timed_out: raise Timeout(hypothesis, satisfying_examples, run_time) else: raise Unsatisfiable(hypothesis, satisfying_examples, run_time) else: raise Unfalsifiable(hypothesis) for example in falsifying_examples: if not falsifies(example): raise Flaky(hypothesis, example) best_example = falsifying_examples[0] for simpler in search_strategy.simplify_such_that( random, best_example, falsifies, tracker=track_seen, ): best_example = simpler if time_to_call_it_a_day(): # We no cover in here because it's a bit sensitive to timing # and tends to make tests flaky. There are tests that mean # this is definitely covered most of the time. break # pragma: no cover if storage is not None: storage.save(best_example) setup_example() return search_strategy.reify(best_example)
def assert_no_duplicates_in_simplify(s, x): s = strategy(s) t = Tracker() t.track(x) for y in s.simplify(x): assert t.track(y) == 1
def test_track_dict(): t = Tracker() assert t.track({1: 2}) == 1 assert t.track({1: 3}) == 1
def test_tracking_custom(): t = Tracker() assert t.track(Foo()) == 1 assert t.track(Foo()) == 2
def test_track_lists(): t = Tracker() assert t.track([1]) == 1 assert t.track([1]) == 2
def falsify( self, hypothesis, *argument_types, **kwargs ): # pylint: disable=too-many-locals,too-many-branches """ Attempt to construct an example tuple x matching argument_types such that hypothesis(*x) returns a falsey value """ teardown_example = kwargs.get('teardown_example') or (lambda x: None) setup_example = kwargs.get('setup_example') or (lambda: None) random = self.random if random is None: random = Random( function_digest(hypothesis) ) build_context = BuildContext(random) search_strategy = strategy(argument_types, self.settings) storage = None if self.database is not None: storage = self.database.storage_for(argument_types) def falsifies(args): # pylint: disable=missing-docstring example = None try: try: setup_example() example = search_strategy.reify(args) return not hypothesis(*example) except UnsatisfiedAssumption: return False finally: teardown_example(example) track_seen = Tracker() falsifying_examples = [] if storage: for example in storage.fetch(): track_seen.track(example) if falsifies(example): falsifying_examples = [example] break satisfying_examples = 0 timed_out = False max_examples = self.max_examples min_satisfying_examples = self.min_satisfying_examples parameter_source = ParameterSource( context=build_context, strategy=search_strategy, min_parameters=max(2, int(float(max_examples) / 10)) ) start_time = time.time() def time_to_call_it_a_day(): """Have we exceeded our timeout?""" if self.timeout <= 0: return False return time.time() >= start_time + self.timeout for parameter in islice( parameter_source, max_examples - len(track_seen) ): if len(track_seen) >= search_strategy.size_upper_bound: break if falsifying_examples: break if time_to_call_it_a_day(): break args = search_strategy.produce_template( build_context, parameter ) if track_seen.track(args) > 1: parameter_source.mark_bad() continue try: setup_example() a = None try: a = search_strategy.reify(args) is_falsifying_example = not hypothesis(*a) finally: teardown_example(a) except UnsatisfiedAssumption: parameter_source.mark_bad() continue satisfying_examples += 1 if is_falsifying_example: falsifying_examples.append(args) run_time = time.time() - start_time timed_out = self.timeout >= 0 and run_time >= self.timeout if not falsifying_examples: if ( satisfying_examples and len(track_seen) >= search_strategy.size_lower_bound ): raise Exhausted( hypothesis, satisfying_examples) elif satisfying_examples < min_satisfying_examples: if timed_out: raise Timeout(hypothesis, satisfying_examples, run_time) else: raise Unsatisfiable( hypothesis, satisfying_examples, run_time) else: raise Unfalsifiable(hypothesis) for example in falsifying_examples: if not falsifies(example): raise Flaky(hypothesis, example) best_example = falsifying_examples[0] for simpler in search_strategy.simplify_such_that( random, best_example, falsifies, tracker=track_seen, ): best_example = simpler if time_to_call_it_a_day(): # We no cover in here because it's a bit sensitive to timing # and tends to make tests flaky. There are tests that mean # this is definitely covered most of the time. break # pragma: no cover if storage is not None: storage.save(best_example) setup_example() return search_strategy.reify(best_example)
def test_track_iterables(): t = Tracker() assert t.track(iter([1])) == 1 assert t.track(iter([1])) == 2
def test_track_ints(): t = Tracker() assert t.track(1) == 1 assert t.track(1) == 2
def test_nested_unhashables(): t = Tracker() x = {u'foo': [1, 2, set((3, 4, 5, 6))], u'bar': 10} assert t.track(x) == 1 assert t.track(x) == 2
def test_nested_unhashables(): t = Tracker() x = {'foo': [1, 2, {3, 4, 5, 6}], 'bar': 10} assert t.track(x) == 1 assert t.track(x) == 2
def falsify(self, hypothesis, *argument_types): # pylint: disable=too-many-locals,too-many-branches """ Attempt to construct an example tuple x matching argument_types such that hypothesis(*x) returns a falsey value or throws an AssertionError """ random = self.random if random is None: random = Random(function_digest(hypothesis)) search_strategy = ( self.strategy_table.specification_for(argument_types)) storage = None if self.database is not None: try: storage = self.database.storage_for(argument_types) except NotSerializeable: pass def falsifies(args): # pylint: disable=missing-docstring try: return not hypothesis(*search_strategy.copy(args)) except AssertionError: return True except UnsatisfiedAssumption: return False track_seen = Tracker() falsifying_examples = [] examples_found = 0 satisfying_examples = 0 timed_out = False if argument_types: max_examples = self.max_examples min_satisfying_examples = self.min_satisfying_examples else: max_examples = 1 min_satisfying_examples = 1 example_source = ExampleSource(random=random, strategy=search_strategy, storage=storage, min_parameters=max( 2, int(float(max_examples) / 10))) start_time = time.time() def time_to_call_it_a_day(): """Have we exceeded our timeout?""" return time.time() >= start_time + self.timeout skipped_examples = 0 examples_seen = 0 # At present this loop will never exit normally . This needs proper # testing when "database only" mode becomes available but right now # it's not. for args in example_source: # pragma: no branch assert search_strategy.could_have_produced(args) if falsifying_examples: break if examples_seen >= max_examples: break if time_to_call_it_a_day(): break examples_seen += 1 if track_seen.track(args) > 1: example_source.mark_bad() skipped_examples += 1 if skipped_examples >= self.max_skipped_examples: raise Exhausted(hypothesis, examples_found) else: # This really is covered. I suspect a bug in coverage that # I have not yet narrowed down. It is impossible to execute # the other branch without first executing this one and # there is a test that cannot pass without executing the # other branch. continue # pragma: no cover else: skipped_examples = 0 examples_found += 1 try: is_falsifying_example = not hypothesis( *search_strategy.copy(args)) except AssertionError: is_falsifying_example = True except UnsatisfiedAssumption: example_source.mark_bad() continue satisfying_examples += 1 if is_falsifying_example: falsifying_examples.append(args) run_time = time.time() - start_time timed_out = run_time >= self.timeout if not falsifying_examples: if satisfying_examples < min_satisfying_examples: raise Unsatisfiable(hypothesis, satisfying_examples, run_time) elif timed_out: raise Timeout(hypothesis, satisfying_examples, run_time) else: raise Unfalsifiable(hypothesis) for example in falsifying_examples: if not falsifies(example): raise Flaky(hypothesis, example) best_example = falsifying_examples[0] for simpler in search_strategy.simplify_such_that( best_example, falsifies): best_example = simpler if time_to_call_it_a_day(): break if storage is not None: storage.save(best_example) return best_example
def test_track_lists(): t = Tracker() assert t.track([1]) == 1 assert t.track([1]) == 2
def test_tracking_classes_of_custom(): t = Tracker() assert t.track(Foo) == 1 assert t.track(Foo) == 2
def test_nested_unhashables(): t = Tracker() x = {'foo': [1, 2, set((3, 4, 5, 6))], 'bar': 10} assert t.track(x) == 1 assert t.track(x) == 2