def test_database_uses_values_from_secondary_key(): key = b"key" database = InMemoryExampleDatabase() def f(data): if data.draw_bits(8) >= 5: data.mark_interesting() else: data.mark_invalid() runner = ConjectureRunner( f, settings=settings( max_examples=1, buffer_size=1024, database=database, suppress_health_check=HealthCheck.all(), ), database_key=key, ) for i in range(10): database.save(runner.secondary_key, hbytes([i])) runner.test_function(ConjectureData.for_buffer(hbytes([10]))) assert runner.interesting_examples assert len(set(database.fetch(key))) == 1 assert len(set(database.fetch(runner.secondary_key))) == 10 runner.clear_secondary_key() assert len(set(database.fetch(key))) == 1 assert set(map(int_from_bytes, database.fetch(runner.secondary_key))) == set(range(6, 11)) v, = runner.interesting_examples.values() assert list(v.buffer) == [5]
def test_does_not_duplicate_elements(): def test(data): data.target_observations["m"] = data.draw_bits(8) runner = ConjectureRunner( test, settings=settings(TEST_SETTINGS, database=InMemoryExampleDatabase()), database_key=b"stuff", ) d1 = runner.cached_test_function([1]).as_result() assert len(runner.pareto_front) == 1 # This can happen in practice if we e.g. reexecute a test because it has # expired from the cache. It's easier just to test it directly though # rather than simulate the failure mode. is_pareto = runner.pareto_front.add(d1) assert is_pareto assert len(runner.pareto_front) == 1
def test_will_save_covering_examples(): tags = {} def tagged(data): b = hbytes(data.draw_bytes(4)) try: tag = tags[b] except KeyError: if len(tags) < 10: tag = len(tags) tags[b] = tag else: tag = None if tag is not None: data.add_tag(tag) db = InMemoryExampleDatabase() runner = ConjectureRunner(tagged, settings=settings( max_examples=100, max_shrinks=0, buffer_size=1024, database=db, ), database_key=b'stuff') runner.run() assert len(all_values(db)) == len(tags)
def test_fully_exhaust_base(monkeypatch): """In this test we generate all possible values for the first byte but never get to the point where we exhaust the root of the tree.""" seed_random(0) seen = set() def f(data): key = data.draw_bytes(2) assert key not in seen seen.add(key) runner = ConjectureRunner(f, settings=settings( max_examples=10000, max_shrinks=0, buffer_size=1024, database=None, )) for c in hrange(256): runner.cached_test_function([0, c]) assert 1 in runner.dead runner.run()
def test_optimises_the_pareto_front(): def test(data): count = 0 while data.draw_bits(8): count += 1 data.target_observations[""] = min(count, 5) runner = ConjectureRunner( test, settings=settings(max_examples=10000, database=InMemoryExampleDatabase()), database_key=b"stuff", ) runner.cached_test_function([255] * 20 + [0]) runner.pareto_optimise() assert len(runner.pareto_front) == 6 for i, data in enumerate(runner.pareto_front): assert list(data.buffer) == [1] * i + [0]
def test_database_contains_only_pareto_front(): with deterministic_PRNG(): def test(data): data.target_observations["1"] = data.draw_bits(4) data.draw_bits(64) data.target_observations["2"] = data.draw_bits(8) db = InMemoryExampleDatabase() runner = ConjectureRunner( test, settings=settings( max_examples=500, database=db, suppress_health_check=HealthCheck.all() ), database_key=b"stuff", ) runner.run() assert len(runner.pareto_front) <= 500 for v in runner.pareto_front: assert v.status >= Status.VALID assert len(db.data) == 1 (values,) = db.data.values() values = set(values) assert len(values) == len(runner.pareto_front) for data in runner.pareto_front: assert data.buffer in values assert data in runner.pareto_front for k in values: assert runner.cached_test_function(k) in runner.pareto_front
def test_stops_optimising_once_interesting(): hi = 2**16 - 1 def test(data): n = data.draw_bits(16) data.target_observations[""] = n if n < hi: data.mark_interesting() runner = ConjectureRunner( test, settings=settings( max_examples=10000, database=InMemoryExampleDatabase(), ), database_key=b"stuff", ) data = runner.cached_test_function([255] * 2) assert data.status == Status.VALID runner.pareto_optimise() assert runner.call_count <= 20 assert runner.interesting_examples
def test_does_not_optimise_the_pareto_front_if_interesting(): def test(data): n = data.draw_bits(8) data.target_observations[""] = n if n == 255: data.mark_interesting() runner = ConjectureRunner( test, settings=settings( max_examples=10000, database=InMemoryExampleDatabase(), ), database_key=b"stuff", ) runner.cached_test_function([0]) runner.pareto_optimise = None runner.optimise_targets() assert runner.interesting_examples
def test_replaces_all_dominated(): def test(data): data.target_observations["m"] = 3 - data.draw_bits(2) data.target_observations["n"] = 3 - data.draw_bits(2) runner = ConjectureRunner( test, settings=settings(TEST_SETTINGS, database=InMemoryExampleDatabase()), database_key=b"stuff", ) d1 = runner.cached_test_function([0, 1]).as_result() d2 = runner.cached_test_function([1, 0]).as_result() assert len(runner.pareto_front) == 2 assert runner.pareto_front[0] == d1 assert runner.pareto_front[1] == d2 d3 = runner.cached_test_function([0, 0]).as_result() assert len(runner.pareto_front) == 1 assert runner.pareto_front[0] == d3
def test_terminates_shrinks(n, monkeypatch): db = InMemoryExampleDatabase() def generate_new_examples(self): self.cached_test_function([255] * 1000) monkeypatch.setattr( ConjectureRunner, "generate_new_examples", generate_new_examples ) monkeypatch.setattr(engine_module, "MAX_SHRINKS", n) runner = ConjectureRunner( slow_shrinker(), settings=settings(max_examples=5000, database=db), random=Random(0), database_key=b"key", ) runner.run() (last_data,) = runner.interesting_examples.values() assert last_data.status == Status.INTERESTING assert runner.shrinks == n in_db = set(db.data[runner.secondary_key]) assert len(in_db) == n
def test_will_shrink_covering_examples(): best = [None] replaced = [] def tagged(data): b = hbytes(data.draw_bytes(4)) if any(b): data.add_tag('nonzero') if best[0] is None: best[0] = b elif b < best[0]: replaced.append(best[0]) best[0] = b db = InMemoryExampleDatabase() runner = ConjectureRunner(tagged, settings=settings( max_examples=100, phases=no_shrink, buffer_size=1024, database=db, ), database_key=b'stuff') runner.run() saved = set(all_values(db)) assert best[0] in saved for r in replaced: assert r not in saved
def test_terminates_shrinks(n, monkeypatch): db = InMemoryExampleDatabase() def generate_new_examples(self): def draw_bytes(data, n): return hbytes([255] * n) self.test_function(ConjectureData( draw_bytes=draw_bytes, max_length=self.settings.buffer_size)) monkeypatch.setattr( ConjectureRunner, 'generate_new_examples', generate_new_examples) runner = ConjectureRunner(slow_shrinker(), settings=settings( max_examples=5000, max_shrinks=n, database=db, timeout=unlimited, ), random=Random(0), database_key=b'key') runner.run() last_data, = runner.interesting_examples.values() assert last_data.status == Status.INTERESTING assert runner.shrinks == n in_db = set(db.data[runner.secondary_key]) assert len(in_db) == n
def test_debug_data(capsys): buf = [0, 1, 2] def f(data): for x in hbytes(buf): if data.draw_bits(8) != x: data.mark_invalid() data.start_example(1) data.stop_example() data.mark_interesting() runner = ConjectureRunner(f, settings=settings( max_examples=5000, buffer_size=1024, database=None, suppress_health_check=HealthCheck.all(), verbosity=Verbosity.debug )) runner.test_function(ConjectureData.for_buffer(buf)) runner.run() out, _ = capsys.readouterr() assert re.match(u'\\d+ bytes \\[.*\\] -> ', out) assert 'INTERESTING' in out assert '[]' not in out
def test_will_not_reset_the_tree_after_interesting_example(monkeypatch): monkeypatch.setattr(engine_module, 'CACHE_RESET_FREQUENCY', 3) def f(data): if data.draw_bits(8) == 7: data.mark_interesting() with deterministic_PRNG(): runner = ConjectureRunner(f, settings=settings( database=None, suppress_health_check=HealthCheck.all(), )) def step(n): runner.test_function(ConjectureData.for_buffer([n])) step(0) step(1) assert len(runner.tree) > 1 step(7) assert len(runner.tree) > 1 t = len(runner.tree) runner.shrink_interesting_examples() assert len(runner.tree) > t
def test_stops_after_max_examples_when_generating_more_bugs(examples): seen = [] bad = [False, False] def f(data): seen.append(data.draw_bits(32)) # Rare, potentially multi-error conditions if seen[-1] > 2**31: bad[0] = True raise ValueError bad[1] = True raise Exception runner = ConjectureRunner(f, settings=settings(max_examples=examples, phases=[Phase.generate])) try: runner.run() except Exception: pass # No matter what, whether examples is larger or smalller than MAX_TEST_CALLS, # we stop looking at max_examples. (and re-run each failure for the traceback) assert len(seen) <= examples + sum(bad)
def test_saves_data_while_shrinking(): key = b'hi there' n = 5 db = ExampleDatabase(':memory:') assert list(db.fetch(key)) == [] seen = set() def f(data): x = data.draw_bytes(512) if sum(x) >= 5000 and len(seen) < n: seen.add(hbytes(x)) if hbytes(x) in seen: data.mark_interesting() runner = ConjectureRunner(f, settings=settings(database=db), database_key=key) runner.run() assert runner.last_data.status == Status.INTERESTING assert len(seen) == n in_db = set(v for vs in db.data.values() for v in vs) assert in_db.issubset(seen) assert in_db == seen
def test_stops_after_max_iterations_when_reading(): key = b'key' max_iterations = 1 db = ExampleDatabase(':memory:') for i in range(10): db.save(key, hbytes([i])) seen = [] def f(data): seen.append(data.draw_bytes(1)) data.mark_invalid() runner = ConjectureRunner(f, settings=settings( max_examples=1, max_iterations=max_iterations, database=db, ), database_key=key) runner.run() assert len(seen) == max_iterations
def test_saves_data_while_shrinking(): key = b'hi there' n = 5 db = InMemoryExampleDatabase() assert list(db.fetch(key)) == [] seen = set() def f(data): x = data.draw_bytes(512) if sum(x) >= 5000 and len(seen) < n: seen.add(hbytes(x)) if hbytes(x) in seen: data.mark_interesting() runner = ConjectureRunner(f, settings=settings(database=db), database_key=key) runner.run() assert runner.interesting_examples assert len(seen) == n in_db = non_covering_examples(db) assert in_db.issubset(seen) assert in_db == seen
def test_optimises_multiple_targets(): with deterministic_PRNG(): def test(data): n = data.draw_bits(8) m = data.draw_bits(8) if n + m > 256: data.mark_invalid() data.target_observations["m"] = m data.target_observations["n"] = n data.target_observations["m + n"] = m + n runner = ConjectureRunner(test, settings=TEST_SETTINGS) runner.cached_test_function([200, 0]) runner.cached_test_function([0, 200]) try: runner.optimise_targets() except RunIsComplete: pass assert runner.best_observed_targets["m"] == 255 assert runner.best_observed_targets["n"] == 255 assert runner.best_observed_targets["m + n"] == 256
def test_branch_ending_in_write(): seen = set() def tf(data): count = 0 while data.draw_bits(1): count += 1 if count > 1: data.draw_bits(1, forced=0) b = hbytes(data.buffer) assert b not in seen seen.add(b) with deterministic_PRNG(): runner = ConjectureRunner(tf, settings=TEST_SETTINGS) for _ in hrange(100): prefix = runner.generate_novel_prefix() attempt = prefix + hbytes(2) data = runner.cached_test_function(attempt) assert data.status == Status.VALID assert attempt.startswith(data.buffer)
def test_stops_after_max_iterations_when_generating(): key = b'key' value = b'rubber baby buggy bumpers' max_iterations = 100 db = ExampleDatabase(':memory:') db.save(key, value) seen = [] def f(data): seen.append(data.draw_bytes(len(value))) data.mark_invalid() runner = ConjectureRunner(f, settings=settings( max_examples=1, max_iterations=max_iterations, database=db, ), database_key=key) runner.run() assert len(seen) == max_iterations assert value in seen
def test_can_find_endpoints_of_a_range(lower, upper, score_up): with deterministic_PRNG(): def test(data): n = data.draw_bits(16) if n < lower or n > upper: data.mark_invalid() if not score_up: n = -n data.target_observations["n"] = n runner = ConjectureRunner( test, settings=settings(TEST_SETTINGS, max_examples=1000) ) runner.cached_test_function(int_to_bytes((lower + upper) // 2, 2)) try: runner.optimise_targets() except RunIsComplete: pass if score_up: assert runner.best_observed_targets["n"] == upper else: assert runner.best_observed_targets["n"] == -lower
def test_can_patch_up_examples(): with deterministic_PRNG(): def test(data): data.start_example(42) m = data.draw_bits(6) data.target_observations["m"] = m for _ in range(m): data.draw_bits(1) data.stop_example() for i in range(4): if i != data.draw_bits(8): data.mark_invalid() runner = ConjectureRunner(test, settings=TEST_SETTINGS) d = runner.cached_test_function([0, 0, 1, 2, 3, 4]) assert d.status == Status.VALID try: runner.optimise_targets() except RunIsComplete: pass assert runner.best_observed_targets["m"] == 63
def test_can_reduce_poison_from_any_subtree(size, seed): """This test validates that we can minimize to any leaf node of a binary tree, regardless of where in the tree the leaf is.""" random = Random(seed) # Initially we create the minimal tree of size n, regardless of whether it # is poisoned (which it won't be - the poison event essentially never # happens when drawing uniformly at random). # Choose p so that the expected size of the tree is equal to the desired # size. p = 1.0 / (2.0 - 1.0 / size) strat = PoisonedTree(p) def test_function(data): v = data.draw(strat) if len(v) >= size: data.mark_interesting() runner = ConjectureRunner(test_function, random=random, settings=settings(TEST_SETTINGS, buffer_size=LOTS)) while not runner.interesting_examples: runner.test_function( runner.new_conjecture_data(lambda data, n: uniform(random, n))) runner.shrink_interesting_examples() data, = runner.interesting_examples.values() assert len(ConjectureData.for_buffer(data.buffer).draw(strat)) == size starts = [b.start for b in data.blocks if b.length == 2] assert len(starts) % 2 == 0 for i in hrange(0, len(starts), 2): # Now for each leaf position in the tree we try inserting a poison # value artificially. Additionally, we add a marker to the end that # must be preserved. The marker means that we are not allow to rely on # discarding the end of the buffer to get the desired shrink. u = starts[i] marker = hbytes([1, 2, 3, 4]) def test_function_with_poison(data): v = data.draw(strat) m = data.draw_bytes(len(marker)) if POISON in v and m == marker: data.mark_interesting() runner = ConjectureRunner(test_function_with_poison, random=random, settings=TEST_SETTINGS) runner.cached_test_function(data.buffer[:u] + hbytes([255]) * 4 + data.buffer[u + 4:] + marker) assert runner.interesting_examples runner.shrink_interesting_examples() shrunk, = runner.interesting_examples.values() assert ConjectureData.for_buffer( shrunk.buffer).draw(strat) == (POISON, )
def run_engine(self): """Run the test function many times, on database input and generated input, using the Conjecture engine. """ # Tell pytest to omit the body of this function from tracebacks __tracebackhide__ = True try: database_key = self.wrapped_test._hypothesis_internal_database_key except AttributeError: if global_force_seed is None: database_key = function_digest(self.test) else: database_key = None runner = ConjectureRunner( self._execute_once_for_engine, settings=self.settings, random=self.random, database_key=database_key, ) # Use the Conjecture engine to run the test function many times # on different inputs. runner.run() note_engine_for_statistics(runner) if runner.call_count == 0: return if runner.interesting_examples: self.falsifying_examples = sorted( runner.interesting_examples.values(), key=lambda d: sort_key(d.buffer), reverse=True, ) else: if runner.valid_examples == 0: raise Unsatisfiable( "Unable to satisfy assumptions of hypothesis %s." % (get_pretty_function_description(self.test), )) if not self.falsifying_examples: return elif not self.settings.report_multiple_bugs: # Pretend that we only found one failure, by discarding the others. del self.falsifying_examples[:-1] # The engine found one or more failures, so we need to reproduce and # report them. self.failed_normally = True flaky = 0 for falsifying_example in self.falsifying_examples: info = falsifying_example.extra_information ran_example = ConjectureData.for_buffer(falsifying_example.buffer) self.__was_flaky = False assert info.__expected_exception is not None try: self.execute_once( ran_example, print_example=not self.is_find, is_final=True, expected_failure=( info.__expected_exception, info.__expected_traceback, ), ) except (UnsatisfiedAssumption, StopTest): report(traceback.format_exc()) self.__flaky( "Unreliable assumption: An example which satisfied " "assumptions on the first run now fails it.") except BaseException as e: if len(self.falsifying_examples) <= 1: # There is only one failure, so we can report it by raising # it directly. raise # We are reporting multiple failures, so we need to manually # print each exception's stack trace and information. tb = get_trimmed_traceback() report("".join(traceback.format_exception(type(e), e, tb))) finally: # pragma: no cover # Mostly useful for ``find`` and ensuring that objects that # hold on to a reference to ``data`` know that it's now been # finished and they shouldn't attempt to draw more data from # it. ran_example.freeze() # This section is in fact entirely covered by the tests in # test_reproduce_failure, but it seems to trigger a lovely set # of coverage bugs: The branches show up as uncovered (despite # definitely being covered - you can add an assert False else # branch to verify this and see it fail - and additionally the # second branch still complains about lack of coverage even if # you add a pragma: no cover to it! # See https://bitbucket.org/ned/coveragepy/issues/623/ if self.settings.print_blob: report(("\nYou can reproduce this example by temporarily " "adding @reproduce_failure(%r, %r) as a decorator " "on your test case") % (__version__, encode_failure(falsifying_example.buffer))) if self.__was_flaky: flaky += 1 # If we only have one example then we should have raised an error or # flaky prior to this point. assert len(self.falsifying_examples) > 1 if flaky > 0: raise Flaky( ("Hypothesis found %d distinct failures, but %d of them " "exhibited some sort of flaky behaviour.") % (len(self.falsifying_examples), flaky)) else: raise MultipleFailures(("Hypothesis found %d distinct failures.") % (len(self.falsifying_examples)))
def find( specifier, # type: SearchStrategy condition, # type: Callable[[Any], bool] settings=None, # type: Settings random=None, # type: Any database_key=None, # type: bytes ): # type: (...) -> Any """Returns the minimal example from the given strategy ``specifier`` that matches the predicate function ``condition``.""" if settings is None: settings = Settings(max_examples=2000) settings = Settings(settings, suppress_health_check=HealthCheck.all()) if database_key is None and settings.database is not None: database_key = function_digest(condition) if not isinstance(specifier, SearchStrategy): raise InvalidArgument("Expected SearchStrategy but got %r of type %s" % (specifier, type(specifier).__name__)) specifier.validate() search = specifier random = random or new_random() successful_examples = [0] last_data = [None] last_repr = [None] def template_condition(data): with BuildContext(data): try: data.is_find = True with deterministic_PRNG(): result = data.draw(search) data.note(result) success = condition(result) except UnsatisfiedAssumption: data.mark_invalid() if success: successful_examples[0] += 1 if settings.verbosity >= Verbosity.verbose: if not successful_examples[0]: report(u"Tried non-satisfying example %s" % (nicerepr(result), )) elif success: if successful_examples[0] == 1: last_repr[0] = nicerepr(result) report(u"Found satisfying example %s" % (last_repr[0], )) last_data[0] = data elif (sort_key(hbytes(data.buffer)) < sort_key( last_data[0].buffer) ) and nicerepr(result) != last_repr[0]: last_repr[0] = nicerepr(result) report(u"Shrunk example to %s" % (last_repr[0], )) last_data[0] = data if success and not data.frozen: data.mark_interesting() runner = ConjectureRunner(template_condition, settings=settings, random=random, database_key=database_key) runner.run() note_engine_for_statistics(runner) if runner.interesting_examples: data = ConjectureData.for_buffer( list(runner.interesting_examples.values())[0].buffer) with BuildContext(data): with deterministic_PRNG(): return data.draw(search) if runner.valid_examples == 0 and (runner.exit_reason != ExitReason.finished): raise Unsatisfiable("Unable to satisfy assumptions of %s." % (get_pretty_function_description(condition), )) raise NoSuchExample(get_pretty_function_description(condition))
def run(self): # Tell pytest to omit the body of this function from tracebacks __tracebackhide__ = True if global_force_seed is None: database_key = function_digest(self.test) else: database_key = None runner = ConjectureRunner( self.evaluate_test_data, settings=self.settings, random=self.random, database_key=database_key, ) try: runner.run() finally: self.used_examples_from_database = runner.used_examples_from_database note_engine_for_statistics(runner) self.used_examples_from_database = runner.used_examples_from_database if runner.call_count == 0: return if runner.interesting_examples: self.falsifying_examples = sorted( [d for d in runner.interesting_examples.values()], key=lambda d: sort_key(d.buffer), reverse=True, ) else: if runner.valid_examples == 0: raise Unsatisfiable( "Unable to satisfy assumptions of hypothesis %s." % (get_pretty_function_description(self.test), )) if not self.falsifying_examples: return self.failed_normally = True flaky = 0 for falsifying_example in self.falsifying_examples: ran_example = ConjectureData.for_buffer(falsifying_example.buffer) self.__was_flaky = False assert falsifying_example.__expected_exception is not None try: self.execute( ran_example, print_example=True, is_final=True, expected_failure=( falsifying_example.__expected_exception, falsifying_example.__expected_traceback, ), ) except (UnsatisfiedAssumption, StopTest): report(traceback.format_exc()) self.__flaky( "Unreliable assumption: An example which satisfied " "assumptions on the first run now fails it.") except BaseException as e: if len(self.falsifying_examples) <= 1: raise tb = get_trimmed_traceback() report("".join(traceback.format_exception(type(e), e, tb))) finally: # pragma: no cover # This section is in fact entirely covered by the tests in # test_reproduce_failure, but it seems to trigger a lovely set # of coverage bugs: The branches show up as uncovered (despite # definitely being covered - you can add an assert False else # branch to verify this and see it fail - and additionally the # second branch still complains about lack of coverage even if # you add a pragma: no cover to it! # See https://bitbucket.org/ned/coveragepy/issues/623/ if self.settings.print_blob is not PrintSettings.NEVER: failure_blob = encode_failure(falsifying_example.buffer) # Have to use the example we actually ran, not the original # falsifying example! Otherwise we won't catch problems # where the repr of the generated example doesn't parse. can_use_repr = ran_example.can_reproduce_example_from_repr if self.settings.print_blob is PrintSettings.ALWAYS or ( self.settings.print_blob is PrintSettings.INFER and self.settings.verbosity >= Verbosity.normal and not can_use_repr and len(failure_blob) < 200): report(( "\nYou can reproduce this example by temporarily " "adding @reproduce_failure(%r, %r) as a decorator " "on your test case") % (__version__, failure_blob)) if self.__was_flaky: flaky += 1 # If we only have one example then we should have raised an error or # flaky prior to this point. assert len(self.falsifying_examples) > 1 if flaky > 0: raise Flaky( ("Hypothesis found %d distinct failures, but %d of them " "exhibited some sort of flaky behaviour.") % (len(self.falsifying_examples), flaky)) else: raise MultipleFailures(("Hypothesis found %d distinct failures.") % (len(self.falsifying_examples)))
def normalize( base_name, test_function, *, required_successes=100, allowed_to_update=False, max_dfas=10, ): """Attempt to ensure that this test function successfully normalizes - i.e. whenever it declares a test case to be interesting, we are able to shrink that to the same interesting test case (which logically should be the shortlex minimal interesting test case, though we may not be able to detect if it is). Will run until we have seen ``required_successes`` many interesting test cases in a row normalize to the same value. If ``allowed_to_update`` is True, whenever we fail to normalize we will learn a new DFA-based shrink pass that allows us to make progress. Any learned DFAs will be written back into the learned DFA file at the end of this function. If ``allowed_to_update`` is False, this will raise an error as soon as it encounters a failure to normalize. Additionally, if more than ``max_dfas` DFAs are required to normalize this test function, this function will raise an error - it's essentially designed for small patches that other shrink passes don't cover, and if it's learning too many patches then you need a better shrink pass than this can provide. """ # Need import inside the function to avoid circular imports from hypothesis.internal.conjecture.engine import BUFFER_SIZE, ConjectureRunner runner = ConjectureRunner( test_function, settings=settings(database=None, suppress_health_check=HealthCheck.all()), ignore_limits=True, ) seen = set() dfas_added = 0 found_interesting = False consecutive_successes = 0 failures_to_find_interesting = 0 while consecutive_successes < required_successes: attempt = runner.cached_test_function(b"", extend=BUFFER_SIZE) if attempt.status < Status.INTERESTING: failures_to_find_interesting += 1 assert (found_interesting or failures_to_find_interesting <= 1000 ), "Test function seems to have no interesting test cases" continue found_interesting = True target = attempt.interesting_origin def shrinking_predicate(d): return d.status == Status.INTERESTING and d.interesting_origin == target if target not in seen: seen.add(target) runner.shrink(attempt, shrinking_predicate) continue previous = fully_shrink(runner, runner.interesting_examples[target], shrinking_predicate) current = fully_shrink(runner, attempt, shrinking_predicate) if current.buffer == previous.buffer: consecutive_successes += 1 continue consecutive_successes = 0 if not allowed_to_update: raise FailedToNormalise( "Shrinker failed to normalize %r to %r and we are not allowed to learn new DFAs." % (previous.buffer, current.buffer)) if dfas_added >= max_dfas: raise FailedToNormalise( "Test function is too hard to learn: Added %d DFAs and still not done." % (dfas_added, )) dfas_added += 1 new_dfa = learn_a_new_dfa(runner, previous.buffer, current.buffer, shrinking_predicate) name = (base_name + "-" + hashlib.sha256(repr(new_dfa).encode("utf-8")).hexdigest()[:10]) # If there is a name collision this DFA should already be being # used for shrinking, so we should have already been able to shrink # v further. assert name not in SHRINKING_DFAS SHRINKING_DFAS[name] = new_dfa if dfas_added > 0: # We've learned one or more DFAs in the course of normalising, so now # we update the file to record those for posterity. update_learned_dfas()
def learner_for(strategy): """Returns an LStar learner that predicts whether a buffer corresponds to a discard free choice sequence leading to a valid value for this strategy.""" try: return LEARNERS[strategy] except KeyError: pass def test_function(data): data.draw(strategy) data.mark_interesting() runner = ConjectureRunner( test_function, settings=settings( database=None, verbosity=Verbosity.quiet, suppress_health_check=HealthCheck.all(), ), random=Random(0), ignore_limits=True, ) def predicate(s): result = runner.cached_test_function(s) if result.status < Status.VALID: return False if result.has_discards: return False return result.buffer == s learner = LStar(predicate) runner.run() (v,) = runner.interesting_examples.values() # We make sure the learner has properly learned small examples. # This is all fairly ad hoc but is mostly designed to get it # to understand what the smallest example is and avoid any # loops at the beginning of the DFA that don't really exist. learner.learn(v.buffer) for n in [1, 2, 3]: for _ in range(5): learner.learn(uniform(runner.random, n) + v.buffer) prev = -1 while learner.generation != prev: prev = learner.generation for _ in range(10): s = uniform(runner.random, len(v.buffer)) + bytes(BUFFER_SIZE) learner.learn(s) data = runner.cached_test_function(s) if data.status >= Status.VALID: learner.learn(data.buffer) LEARNERS[strategy] = learner return learner
def test_overruns_if_not_enough_bytes_for_block(): runner = ConjectureRunner(lambda data: data.draw_bytes(2), settings=TEST_SETTINGS, random=Random(0)) runner.cached_test_function(b"\0\0") assert runner.tree.rewrite(b"\0")[1] == Status.OVERRUN