def run_benchmark_for_sizes(benchmark, n_runs): click.echo('Calculating data for %s' % (benchmark.name,)) total_sizes = [] with click.progressbar(range(n_runs)) as runs: for _ in runs: sizes = [] valid_seed = random.getrandbits(64).to_bytes(8, 'big') interesting_seed = random.getrandbits(64).to_bytes(8, 'big') def test_function(data): try: try: value = data.draw(benchmark.strategy) except UnsatisfiedAssumption: data.mark_invalid() if not data.frozen: if not benchmark.valid(valid_seed, data, value): data.mark_invalid() if benchmark.interesting( interesting_seed, data, value ): data.mark_interesting() finally: sizes.append(len(data.buffer)) engine = ConjectureRunner( test_function, settings=BENCHMARK_SETTINGS, random=random ) engine.run() assert len(sizes) > 0 total_sizes.append(sum(sizes)) return total_sizes
def test_will_shrink_covering_examples(): best = [None] replaced = [] def tagged(data): b = hbytes(data.draw_bytes(4)) if any(b): data.add_tag('nonzero') if best[0] is None: best[0] = b elif b < best[0]: replaced.append(best[0]) best[0] = b db = InMemoryExampleDatabase() runner = ConjectureRunner(tagged, settings=settings( max_examples=100, max_iterations=10000, max_shrinks=0, buffer_size=1024, database=db, ), database_key=b'stuff') runner.run() saved = set(all_values(db)) assert best[0] in saved for r in replaced: assert r not in saved
def test_saves_data_while_shrinking(): key = b'hi there' n = 5 db = ExampleDatabase(':memory:') assert list(db.fetch(key)) == [] seen = set() def f(data): x = data.draw_bytes(512) if sum(x) >= 5000 and len(seen) < n: seen.add(hbytes(x)) if hbytes(x) in seen: data.mark_interesting() runner = ConjectureRunner( f, settings=settings(database=db), database_key=key) runner.run() assert runner.last_data.status == Status.INTERESTING assert len(seen) == n in_db = set( v for vs in db.data.values() for v in vs ) assert in_db.issubset(seen) assert in_db == seen
def test_clears_out_its_database_on_shrinking( initial_attempt, skip_target, monkeypatch ): def generate_new_examples(self): self.test_function( ConjectureData.for_buffer(hbytes([initial_attempt]))) monkeypatch.setattr( ConjectureRunner, 'generate_new_examples', generate_new_examples) key = b'key' db = InMemoryExampleDatabase() def f(data): if data.draw_bits(8) >= 127: data.mark_interesting() runner = ConjectureRunner( f, settings=settings(database=db, max_examples=256), database_key=key, random=Random(0), ) for n in hrange(256): if n != 127 or not skip_target: db.save(runner.secondary_key, hbytes([n])) runner.run() assert len(runner.interesting_examples) == 1 for b in db.fetch(runner.secondary_key): assert b[0] >= 127 assert len(list(db.fetch(runner.database_key))) == 1
def test_can_delete_intervals(monkeypatch): def generate_new_examples(self): self.test_function( ConjectureData.for_buffer(hbytes([255] * 10 + [1, 3]))) monkeypatch.setattr( ConjectureRunner, 'generate_new_examples', generate_new_examples) monkeypatch.setattr( Shrinker, 'shrink', fixate(Shrinker.adaptive_example_deletion) ) def f(data): while True: n = data.draw_bits(8) if n == 255: continue elif n == 1: break else: data.mark_invalid() if data.draw_bits(8) == 3: data.mark_interesting() runner = ConjectureRunner(f, settings=settings(database=None)) runner.run() x, = runner.interesting_examples.values() assert x.buffer == hbytes([1, 3])
def test_can_cover_without_a_database_key(): def tagged(data): data.add_tag(0) runner = ConjectureRunner(tagged, settings=settings(), database_key=None) runner.run() assert len(runner.covering_examples) == 1
def test_terminates_shrinks(n, monkeypatch): from hypothesis.internal.conjecture import engine db = InMemoryExampleDatabase() def generate_new_examples(self): def draw_bytes(data, n): return hbytes([255] * n) self.test_function(self.new_conjecture_data(draw_bytes)) monkeypatch.setattr( ConjectureRunner, "generate_new_examples", generate_new_examples ) monkeypatch.setattr(engine, "MAX_SHRINKS", n) runner = ConjectureRunner( slow_shrinker(), settings=settings(max_examples=5000, database=db), random=Random(0), database_key=b"key", ) runner.run() last_data, = runner.interesting_examples.values() assert last_data.status == Status.INTERESTING assert runner.shrinks == n in_db = set(db.data[runner.secondary_key]) assert len(in_db) == n
def test_run_nothing(): def f(data): assert False runner = ConjectureRunner(f, settings=settings(phases=())) runner.run() assert runner.call_count == 0
def test_garbage_collects_the_database(): key = b'hi there' n = 200 db = ExampleDatabase(':memory:') local_settings = settings( database=db, max_shrinks=n, timeout=unlimited) runner = ConjectureRunner( slow_shrinker(), settings=local_settings, database_key=key) runner.run() assert runner.last_data.status == Status.INTERESTING def in_db(): return set( v for vs in db.data.values() for v in vs ) assert len(in_db()) == n + 1 runner = ConjectureRunner( lambda data: data.draw_bytes(4), settings=local_settings, database_key=key) runner.run() assert 0 < len(in_db()) < n
def test_garbage_collects_the_database(): key = b'hi there' n = 200 db = ExampleDatabase(':memory:') assert list(db.fetch(key)) == [] seen = set() go = True def f(data): x = hbytes(data.draw_bytes(512)) if not go: return if sum(x) >= 5000 and len(seen) < n: seen.add(x) if x in seen: data.mark_interesting() runner = ConjectureRunner( f, settings=settings(database=db, max_shrinks=2 * n), database_key=key) runner.run() assert runner.last_data.status == Status.INTERESTING assert len(seen) == n assert set(db.fetch(key)) == seen go = False runner = ConjectureRunner( f, settings=settings(database=db, max_shrinks=2 * n), database_key=key) runner.run() assert 0 < len(set(db.fetch(key))) < n
def test_saves_data_while_shrinking(monkeypatch): key = b'hi there' n = 5 db = InMemoryExampleDatabase() assert list(db.fetch(key)) == [] seen = set() monkeypatch.setattr( ConjectureRunner, 'generate_new_examples', lambda runner: runner.test_function( ConjectureData.for_buffer([255] * 10))) def f(data): x = data.draw_bytes(10) if sum(x) >= 2000 and len(seen) < n: seen.add(hbytes(x)) if hbytes(x) in seen: data.mark_interesting() runner = ConjectureRunner( f, settings=settings(database=db), database_key=key) runner.run() assert runner.interesting_examples assert len(seen) == n in_db = non_covering_examples(db) assert in_db.issubset(seen) assert in_db == seen
def run_to_data(f): with deterministic_PRNG(): runner = ConjectureRunner(f, settings=TEST_SETTINGS) runner.run() assert runner.interesting_examples last_data, = runner.interesting_examples.values() return last_data
def test_debug_data(capsys): buf = [0, 1, 2] def f(data): for x in hbytes(buf): if data.draw_bits(8) != x: data.mark_invalid() data.start_example(1) data.stop_example() data.mark_interesting() runner = ConjectureRunner( f, settings=settings( max_examples=5000, buffer_size=1024, database=None, suppress_health_check=HealthCheck.all(), verbosity=Verbosity.debug, ), ) runner.cached_test_function(buf) runner.run() out, _ = capsys.readouterr() assert re.match(u"\\d+ bytes \\[.*\\] -> ", out) assert "INTERESTING" in out
def test_exhaust_space(): with deterministic_PRNG(): runner = ConjectureRunner( lambda data: data.draw_bits(1), settings=TEST_SETTINGS ) runner.run() assert runner.tree.is_exhausted assert runner.valid_examples == 2
def test_note_events(event): def f(data): data.note_event(event) data.draw_bytes(1) runner = ConjectureRunner(f) runner.run() assert runner.event_call_counts[str(event)] == runner.call_count > 0
def run_to_buffer(f): runner = ConjectureRunner(f, settings=settings( max_examples=5000, max_iterations=10000, max_shrinks=MAX_SHRINKS, buffer_size=1024, database=None, )) runner.run() assert runner.last_data.status == Status.INTERESTING return hbytes(runner.last_data.buffer)
def test_stops_after_max_examples_when_generating(): seen = [] def f(data): seen.append(data.draw_bytes(1)) runner = ConjectureRunner(f, settings=settings(max_examples=1, database=None)) runner.run() assert len(seen) == 1
def run_to_buffer(f): with deterministic_PRNG(): runner = ConjectureRunner(f, settings=settings( max_examples=5000, buffer_size=1024, database=None, suppress_health_check=HealthCheck.all(), )) runner.run() assert runner.interesting_examples last_data, = runner.interesting_examples.values() return hbytes(last_data.buffer)
def test_can_navigate_to_a_valid_example(): def f(data): i = int_from_bytes(data.draw_bytes(2)) data.draw_bytes(i) data.mark_interesting() runner = ConjectureRunner(f, settings=settings( max_examples=5000, buffer_size=2, database=None, )) runner.run() assert runner.interesting_examples
def test_run_with_timeout_while_boring(): def f(data): time.sleep(0.1) runner = ConjectureRunner( f, settings=settings(database=None, timeout=0.2)) start = time.time() runner.run() assert time.time() <= start + 1 assert runner.valid_examples > 0
def test_run_with_timeout_while_boring(): def f(data): time.sleep(0.1) runner = ConjectureRunner( f, settings=settings(database=None, timeout=0.2)) start = time.time() runner.run() assert time.time() <= start + 1 assert runner.last_data.status == Status.VALID
def accept(f): runner = ConjectureRunner(f, settings=settings( max_examples=100, phases=no_shrink, buffer_size=1024, database=None, **kwargs )) with pytest.raises(FailedHealthCheck) as e: runner.run() assert e.value.health_check == label assert not runner.interesting_examples
def test_max_shrinks_can_disable_shrinking(): seen = set() def f(data): seen.add(hbytes(data.draw_bytes(32))) data.mark_interesting() runner = ConjectureRunner( f, settings=settings(database=None, max_shrinks=0,)) runner.run() assert len(seen) == 1
def x(data): rnd = Random(data.draw_bytes(1)) def g(d2): d2.draw_bytes(1) data.mark_interesting() runner = ConjectureRunner(g, random=rnd) children.append(runner) runner.run() if runner.interesting_examples: data.mark_interesting()
def test_phases_can_disable_shrinking(): seen = set() def f(data): seen.add(hbytes(data.draw_bytes(32))) data.mark_interesting() runner = ConjectureRunner(f, settings=settings( database=None, phases=(Phase.reuse, Phase.generate), )) runner.run() assert len(seen) == 1
def test_saves_on_interrupt(): def interrupts(data): raise KeyboardInterrupt() db = InMemoryExampleDatabase() runner = ConjectureRunner( interrupts, settings=settings(database=db), database_key=b'key') with pytest.raises(KeyboardInterrupt): runner.run() assert db.data
def test_uniqueness_is_preserved_when_writing_at_beginning(): seen = set() def f(data): data.write(hbytes(1)) n = data.draw_bits(3) assert n not in seen seen.add(n) runner = ConjectureRunner(f, settings=settings(max_examples=50)) runner.run() assert runner.valid_examples == len(seen)
def test_can_navigate_to_a_valid_example(): def f(data): i = int_from_bytes(data.draw_bytes(2)) data.draw_bytes(i) data.mark_interesting() runner = ConjectureRunner(f, settings=settings( max_examples=5000, max_iterations=10000, buffer_size=2, database=None, )) runner.run() assert runner.last_data.status == Status.INTERESTING return hbytes(runner.last_data.buffer)
def test_detects_flakiness(): failed_once = [False] count = [0] def tf(data): data.draw_bytes(1) count[0] += 1 if not failed_once[0]: failed_once[0] = True data.mark_interesting() runner = ConjectureRunner(tf) runner.run() assert count == [2]
def test_run_with_timeout_while_shrinking(): def f(data): time.sleep(0.1) x = data.draw_bytes(32) if any(x): data.mark_interesting() runner = ConjectureRunner( f, settings=settings(database=None, timeout=0.2)) start = time.time() runner.run() assert time.time() <= start + 1 assert runner.interesting_examples
def test_stops_after_max_iterations_when_reading(): key = b'key' max_iterations = 1 db = ExampleDatabase(':memory:') for i in range(10): db.save(key, hbytes([i])) seen = [] def f(data): seen.append(data.draw_bytes(1)) data.mark_invalid() runner = ConjectureRunner(f, settings=settings( max_examples=1, max_iterations=max_iterations, database=db, ), database_key=key) runner.run() assert len(seen) == max_iterations
def test_can_delete_intervals(monkeypatch): def generate_new_examples(self): self.test_function( ConjectureData.for_buffer(hbytes([255] * 10 + [0]))) monkeypatch.setattr( ConjectureRunner, 'generate_new_examples', generate_new_examples) monkeypatch.setattr( Shrinker, 'shrink', Shrinker.adaptive_example_deletion ) def f(data): if data.draw_bits(1): while data.draw_bits(8): pass data.mark_interesting() runner = ConjectureRunner(f, settings=settings(database=None)) runner.run() x, = runner.interesting_examples.values() assert x.buffer == hbytes([1, 0])
def run_test(): if condition is None: def _condition(x): return True condition_string = u'' else: _condition = condition condition_string = strip_lambda( reflection.get_pretty_function_description(condition)) def test_function(data): try: value = data.draw(specifier) except UnsatisfiedAssumption: data.mark_invalid() if not _condition(value): data.mark_invalid() if predicate(value): data.mark_interesting() successes = 0 for _ in range(RUNS): runner = ConConjectureRunner(test_function, settings=Settings(max_examples=200, max_iterations=1000, max_shrinks=0)) runner.run() if runner.last_data.status == Status.INTERESTING: successes += 1 if successes >= REQUIRED_RUNS: return event = condition_string if condition is not None: event += '|' event += condition_string description = (u'P(%s) ~ %d / %d = %.2f < %.2f') % ( event, successes, RUNS, successes / RUNS, (REQUIRED_RUNS / RUNS)) raise HypothesisFalsified(description + u' rejected')
def run_language_test_for(root, data, seed): random.seed(seed) def test(local_data): node = root while not isinstance(node, Terminal): if isinstance(node, Write): local_data.write(hbytes(node.value)) node = node.child else: assert isinstance(node, Branch) c = local_data.draw_bits(node.bits) try: node = node.children[c] except KeyError: if data is None: return node = node.children.setdefault(c, data.draw(nodes)) assert isinstance(node, Terminal) if node.status == Status.INTERESTING: local_data.mark_interesting(node.payload) elif node.status == Status.INVALID: local_data.mark_invalid() runner = ConjectureRunner( test, settings=settings( max_examples=1, buffer_size=512, database=None, suppress_health_check=HealthCheck.all(), verbosity=Verbosity.quiet, phases=list(Phase), ), ) try: runner.run() finally: if data is not None: note(root) assume(runner.interesting_examples)
def test_saves_data_while_shrinking(): key = b'hi there' n = 5 db = ExampleDatabase(':memory:') assert list(db.fetch(key)) == [] seen = set() def f(data): x = data.draw_bytes(512) if sum(x) >= 5000 and len(seen) < n: seen.add(hbytes(x)) if hbytes(x) in seen: data.mark_interesting() runner = ConjectureRunner( f, settings=settings(database=db), database_key=key) runner.run() assert runner.last_data.status == Status.INTERESTING assert len(seen) == n in_db = set(db.fetch(key)) assert in_db.issubset(seen) assert in_db == seen
def test_fully_exhaust_base(): """In this test we generate all possible values for the first byte but never get to the point where we exhaust the root of the tree.""" seed_random(0) seen = set() def f(data): seen.add(data.draw_bytes(2)) runner = ConjectureRunner(f, settings=settings( max_examples=10000, max_iterations=10000, max_shrinks=MAX_SHRINKS, buffer_size=1024, database=None, )) runner.run() assert len(seen) > 256 assert len({x[0] for x in seen}) == 256
def test_discards_kill_branches(): starts = set() with deterministic_PRNG(): def test(data): assert runner.call_count <= 256 while True: data.start_example(1) b = data.draw_bits(8) data.stop_example(b != 0) if len(data.buffer) == 1: s = bytes(data.buffer) assert s not in starts starts.add(s) if b == 0: break runner = ConjectureRunner(test, settings=SMALL_COUNT_SETTINGS) runner.run() assert runner.call_count == 256
def test_uses_tags_in_calculating_pareto_front(): with deterministic_PRNG(): def test(data): if data.draw_bits(1): data.start_example(11) data.draw_bits(8) data.stop_example() runner = ConjectureRunner( test, settings=settings( max_examples=10, database=InMemoryExampleDatabase(), ), database_key=b"stuff", ) runner.run() assert len(runner.pareto_front) == 2
def test_stops_after_max_iterations_when_generating(): key = b'key' value = b'rubber baby buggy bumpers' max_iterations = 100 db = ExampleDatabase(':memory:') db.save(key, value) seen = [] def f(data): seen.append(data.draw_bytes(len(value))) data.mark_invalid() runner = ConjectureRunner(f, settings=settings( max_examples=1, max_iterations=max_iterations, database=db, ), database_key=key) runner.run() assert len(seen) == max_iterations assert value in seen
def test_terminates_shrinks(): shrinks = [-1] def tf(data): x = hbytes(data.draw_bytes(100)) if sum(x) >= 500: shrinks[0] += 1 data.mark_interesting() runner = ConjectureRunner(tf, settings=settings( max_examples=5000, max_iterations=10000, max_shrinks=10, database=None, )) runner.run() assert runner.last_data.status == Status.INTERESTING # There's an extra non-shrinking check step to abort in the presence of # flakiness assert shrinks[0] == 11
def test_database_contains_only_pareto_front(): with deterministic_PRNG(): def test(data): data.target_observations["1"] = data.draw_bits(4) data.draw_bits(64) data.target_observations["2"] = data.draw_bits(8) db = InMemoryExampleDatabase() runner = ConjectureRunner( test, settings=settings( max_examples=500, database=db, suppress_health_check=HealthCheck.all(), ), database_key=b"stuff", ) runner.run() assert len(runner.pareto_front) <= 500 for v in runner.pareto_front: assert v.status >= Status.VALID assert len(db.data) == 1 (values, ) = db.data.values() values = set(values) assert len(values) == len(runner.pareto_front) for data in runner.pareto_front: assert data.buffer in values assert data in runner.pareto_front for k in values: assert runner.cached_test_function(k) in runner.pareto_front
def test_garbage_collects_the_database(): key = b'hi there' n = 200 db = ExampleDatabase(':memory:') assert list(db.fetch(key)) == [] seen = set() go = True counter = [0] def f(data): """This function is designed to shrink very badly. So we only occasionally mark things as interesting, and require a certain amount of complexity to do so. """ x = hbytes(data.draw_bytes(512)) if not go: return if counter[0] % 10 == 0 and len(seen) < n and sum(x) > 1000: seen.add(x) counter[0] += 1 if x in seen: data.mark_interesting() local_settings = settings(database=db, max_shrinks=2 * n, timeout=unlimited) runner = ConjectureRunner(f, settings=local_settings, database_key=key) runner.run() assert runner.last_data.status == Status.INTERESTING assert len(seen) == n assert set(db.fetch(key)) == seen go = False runner = ConjectureRunner(f, settings=local_settings, database_key=key) runner.run() assert 0 < len(set(db.fetch(key))) < n
def test_fully_exhaust_base(monkeypatch): """In this test we generate all possible values for the first byte but never get to the point where we exhaust the root of the tree.""" seed_random(0) seen = set() def f(data): key = (data.draw_bits(2), data.draw_bits(2)) assert key not in seen seen.add(key) runner = ConjectureRunner(f, settings=settings( max_examples=10000, phases=no_shrink, buffer_size=1024, database=None, )) for c in hrange(4): runner.cached_test_function([0, c]) assert 1 in runner.dead runner.run()
def test_garbage_collects_the_database(): key = b'hi there' n = 200 db = ExampleDatabase(':memory:') local_settings = settings(database=db, max_shrinks=n, timeout=unlimited) runner = ConjectureRunner(slow_shrinker(), settings=local_settings, database_key=key) runner.run() assert runner.last_data.status == Status.INTERESTING def in_db(): return set(v for vs in db.data.values() for v in vs) assert len(in_db()) == n + 1 runner = ConjectureRunner(lambda data: None, settings=local_settings, database_key=key) runner.run() assert 0 < len(in_db()) < n
def test_garbage_collects_the_database(): key = b'hi there' n = 200 db = InMemoryExampleDatabase() local_settings = settings(database=db, max_shrinks=n, timeout=unlimited) runner = ConjectureRunner(slow_shrinker(), settings=local_settings, database_key=key) runner.run() assert runner.interesting_examples def in_db(): return set(v for vs in db.data.values() for v in vs) assert len(in_db()) == n + 1 runner = ConjectureRunner(lambda data: data.draw_bytes(4), settings=local_settings, database_key=key) runner.run() assert 0 < len(in_db()) < n
def test_terminates_shrinks(n, monkeypatch): db = InMemoryExampleDatabase() def generate_new_examples(self): self.cached_test_function([255] * 1000) monkeypatch.setattr(ConjectureRunner, "generate_new_examples", generate_new_examples) monkeypatch.setattr(engine_module, "MAX_SHRINKS", n) runner = ConjectureRunner( slow_shrinker(), settings=settings(max_examples=5000, database=db), random=Random(0), database_key=b"key", ) runner.run() (last_data, ) = runner.interesting_examples.values() assert last_data.status == Status.INTERESTING assert runner.shrinks == n in_db = set(db.data[runner.secondary_key]) assert len(in_db) == n
def test_will_save_covering_examples(): tags = {} def tagged(data): b = hbytes(data.draw_bytes(4)) try: tag = tags[b] except KeyError: if len(tags) < 10: tag = len(tags) tags[b] = tag else: tag = None if tag is not None: data.add_tag(tag) db = InMemoryExampleDatabase() runner = ConjectureRunner(tagged, settings=settings( max_examples=100, phases=no_shrink, buffer_size=1024, database=db, ), database_key=b'stuff') runner.run() assert len(all_values(db)) == len(tags)
def test_debug_data(capsys): buf = [0, 1, 2] def f(data): for x in hbytes(buf): if data.draw_bits(8) != x: data.mark_invalid() data.start_example(1) data.stop_example() data.mark_interesting() runner = ConjectureRunner(f, settings=settings( max_examples=5000, buffer_size=1024, database=None, suppress_health_check=HealthCheck.all(), verbosity=Verbosity.debug )) runner.test_function(ConjectureData.for_buffer(buf)) runner.run() out, _ = capsys.readouterr() assert re.match(u'\\d+ bytes \\[.*\\] -> ', out) assert 'INTERESTING' in out assert '[]' not in out
def test_terminates_shrinks(n, monkeypatch): db = InMemoryExampleDatabase() def generate_new_examples(self): def draw_bytes(data, n): return hbytes([255] * n) self.test_function(ConjectureData( draw_bytes=draw_bytes, max_length=self.settings.buffer_size)) monkeypatch.setattr( ConjectureRunner, 'generate_new_examples', generate_new_examples) runner = ConjectureRunner(slow_shrinker(), settings=settings( max_examples=5000, max_shrinks=n, database=db, timeout=unlimited, ), random=Random(0), database_key=b'key') runner.run() last_data, = runner.interesting_examples.values() assert last_data.status == Status.INTERESTING assert runner.shrinks == n in_db = set(db.data[runner.secondary_key]) assert len(in_db) == n
def test_will_shrink_covering_examples(): best = [None] replaced = [] def tagged(data): b = hbytes(data.draw_bytes(4)) if any(b): data.add_tag('nonzero') if best[0] is None: best[0] = b elif b < best[0]: replaced.append(best[0]) best[0] = b db = InMemoryExampleDatabase() runner = ConjectureRunner(tagged, settings=settings( max_examples=100, phases=no_shrink, buffer_size=1024, database=db, ), database_key=b'stuff') runner.run() saved = set(all_values(db)) assert best[0] in saved for r in replaced: assert r not in saved
def test_saves_data_while_shrinking(): key = b'hi there' n = 5 db = InMemoryExampleDatabase() assert list(db.fetch(key)) == [] seen = set() def f(data): x = data.draw_bytes(512) if sum(x) >= 5000 and len(seen) < n: seen.add(hbytes(x)) if hbytes(x) in seen: data.mark_interesting() runner = ConjectureRunner(f, settings=settings(database=db), database_key=key) runner.run() assert runner.interesting_examples assert len(seen) == n in_db = non_covering_examples(db) assert in_db.issubset(seen) assert in_db == seen
def test_stops_after_max_examples_when_generating_more_bugs(examples): seen = [] bad = [False, False] def f(data): seen.append(data.draw_bits(32)) # Rare, potentially multi-error conditions if seen[-1] > 2**31: bad[0] = True raise ValueError bad[1] = True raise Exception runner = ConjectureRunner(f, settings=settings(max_examples=examples, phases=[Phase.generate])) try: runner.run() except Exception: pass # No matter what, whether examples is larger or smalller than MAX_TEST_CALLS, # we stop looking at max_examples. (and re-run each failure for the traceback) assert len(seen) <= examples + sum(bad)
def run_engine(self): """Run the test function many times, on database input and generated input, using the Conjecture engine. """ # Tell pytest to omit the body of this function from tracebacks __tracebackhide__ = True try: database_key = self.wrapped_test._hypothesis_internal_database_key except AttributeError: if global_force_seed is None: database_key = function_digest(self.test) else: database_key = None runner = ConjectureRunner( self._execute_once_for_engine, settings=self.settings, random=self.random, database_key=database_key, ) # Use the Conjecture engine to run the test function many times # on different inputs. runner.run() note_engine_for_statistics(runner) if runner.call_count == 0: return if runner.interesting_examples: self.falsifying_examples = sorted( runner.interesting_examples.values(), key=lambda d: sort_key(d.buffer), reverse=True, ) else: if runner.valid_examples == 0: raise Unsatisfiable( "Unable to satisfy assumptions of hypothesis %s." % (get_pretty_function_description(self.test), )) if not self.falsifying_examples: return elif not self.settings.report_multiple_bugs: # Pretend that we only found one failure, by discarding the others. del self.falsifying_examples[:-1] # The engine found one or more failures, so we need to reproduce and # report them. self.failed_normally = True flaky = 0 for falsifying_example in self.falsifying_examples: info = falsifying_example.extra_information ran_example = ConjectureData.for_buffer(falsifying_example.buffer) self.__was_flaky = False assert info.__expected_exception is not None try: self.execute_once( ran_example, print_example=not self.is_find, is_final=True, expected_failure=( info.__expected_exception, info.__expected_traceback, ), ) except (UnsatisfiedAssumption, StopTest): report(traceback.format_exc()) self.__flaky( "Unreliable assumption: An example which satisfied " "assumptions on the first run now fails it.") except BaseException as e: if len(self.falsifying_examples) <= 1: # There is only one failure, so we can report it by raising # it directly. raise # We are reporting multiple failures, so we need to manually # print each exception's stack trace and information. tb = get_trimmed_traceback() report("".join(traceback.format_exception(type(e), e, tb))) finally: # pragma: no cover # Mostly useful for ``find`` and ensuring that objects that # hold on to a reference to ``data`` know that it's now been # finished and they shouldn't attempt to draw more data from # it. ran_example.freeze() # This section is in fact entirely covered by the tests in # test_reproduce_failure, but it seems to trigger a lovely set # of coverage bugs: The branches show up as uncovered (despite # definitely being covered - you can add an assert False else # branch to verify this and see it fail - and additionally the # second branch still complains about lack of coverage even if # you add a pragma: no cover to it! # See https://bitbucket.org/ned/coveragepy/issues/623/ if self.settings.print_blob: report(("\nYou can reproduce this example by temporarily " "adding @reproduce_failure(%r, %r) as a decorator " "on your test case") % (__version__, encode_failure(falsifying_example.buffer))) if self.__was_flaky: flaky += 1 # If we only have one example then we should have raised an error or # flaky prior to this point. assert len(self.falsifying_examples) > 1 if flaky > 0: raise Flaky( ("Hypothesis found %d distinct failures, but %d of them " "exhibited some sort of flaky behaviour.") % (len(self.falsifying_examples), flaky)) else: raise MultipleFailures(("Hypothesis found %d distinct failures.") % (len(self.falsifying_examples)))
def run(self): # Tell pytest to omit the body of this function from tracebacks __tracebackhide__ = True database_key = str_to_bytes(fully_qualified_name(self.test)) self.start_time = time.time() runner = ConjectureRunner( self.evaluate_test_data, settings=self.settings, random=self.random, database_key=database_key, ) runner.run() note_engine_for_statistics(runner) run_time = time.time() - self.start_time timed_out = runner.exit_reason == ExitReason.timeout if runner.last_data is None: return if runner.interesting_examples: self.falsifying_examples = sorted( [d for d in runner.interesting_examples.values()], key=lambda d: sort_key(d.buffer), reverse=True) else: if timed_out: note_deprecation(( 'Your tests are hitting the settings timeout (%.2fs). ' 'This functionality will go away in a future release ' 'and you should not rely on it. Instead, try setting ' 'max_examples to be some value lower than %d (the number ' 'of examples your test successfully ran here). Or, if you ' 'would prefer your tests to run to completion, regardless ' 'of how long they take, you can set the timeout value to ' 'hypothesis.unlimited.') % (self.settings.timeout, runner.valid_examples), self.settings) if runner.valid_examples < min( self.settings.min_satisfying_examples, self.settings.max_examples, ) and not (runner.exit_reason == ExitReason.finished and self.at_least_one_success): if timed_out: raise Timeout( ('Ran out of time before finding a satisfying ' 'example for ' '%s. Only found %d examples in ' + '%.2fs.') % (get_pretty_function_description( self.test), runner.valid_examples, run_time)) else: raise Unsatisfiable( ('Unable to satisfy assumptions of hypothesis ' '%s. Only %d examples considered ' 'satisfied assumptions') % ( get_pretty_function_description(self.test), runner.valid_examples, )) if not self.falsifying_examples: return flaky = 0 for falsifying_example in self.falsifying_examples: self.__was_flaky = False raised_exception = False try: with self.settings: self.test_runner( ConjectureData.for_buffer(falsifying_example.buffer), reify_and_execute(self.search_strategy, self.test, print_example=True, is_final=True)) except (UnsatisfiedAssumption, StopTest): report(traceback.format_exc()) self.__flaky( 'Unreliable assumption: An example which satisfied ' 'assumptions on the first run now fails it.') except: if len(self.falsifying_examples) <= 1: raise raised_exception = True report(traceback.format_exc()) if not raised_exception: report( 'Failed to reproduce exception. Expected: \n' + falsifying_example.__expected_exception, ) filter_message = ( 'Unreliable test data: Failed to reproduce a failure ' 'and then when it came to recreating the example in ' 'order to print the test data with a flaky result ' 'the example was filtered out (by e.g. a ' 'call to filter in your strategy) when we didn\'t ' 'expect it to be.') try: self.test_runner( ConjectureData.for_buffer(falsifying_example.buffer), reify_and_execute(self.search_strategy, test_is_flaky( self.test, self.repr_for_last_exception), print_example=True, is_final=True)) except (UnsatisfiedAssumption, StopTest): self.__flaky(filter_message) except Flaky as e: if len(self.falsifying_examples) > 1: self.__flaky(e.args[0]) else: raise if self.__was_flaky: flaky += 1 # If we only have one example then we should have raised an error or # flaky prior to this point. assert len(self.falsifying_examples) > 1 if flaky > 0: raise Flaky( ('Hypothesis found %d distinct failures, but %d of them ' 'exhibited some sort of flaky behaviour.') % (len(self.falsifying_examples), flaky)) else: raise MultipleFailures(('Hypothesis found %d distinct failures.') % (len(self.falsifying_examples, )))
def run(self): database_key = str_to_bytes(fully_qualified_name(self.test)) start_time = time.time() runner = ConjectureRunner( self.evaluate_test_data, settings=self.settings, random=self.random, database_key=database_key, ) runner.run() note_engine_for_statistics(runner) run_time = time.time() - start_time timed_out = (self.settings.timeout > 0 and run_time >= self.settings.timeout) if runner.last_data is None: return if runner.last_data.status == Status.INTERESTING: self.falsifying_example = runner.last_data.buffer if self.settings.database is not None: self.settings.database.save(database_key, self.falsifying_example) else: if runner.valid_examples < min( self.settings.min_satisfying_examples, self.settings.max_examples, ) and not (runner.exit_reason == ExitReason.finished and self.at_least_one_success): if timed_out: raise Timeout( ('Ran out of time before finding a satisfying ' 'example for ' '%s. Only found %d examples in ' + '%.2fs.') % (get_pretty_function_description( self.test), runner.valid_examples, run_time)) else: raise Unsatisfiable( ('Unable to satisfy assumptions of hypothesis ' '%s. Only %d examples considered ' 'satisfied assumptions') % ( get_pretty_function_description(self.test), runner.valid_examples, )) if self.falsifying_example is None: return assert self.last_exception is not None try: with self.settings: self.test_runner( ConjectureData.for_buffer(self.falsifying_example), reify_and_execute(self.search_strategy, self.test, print_example=True, is_final=True)) except (UnsatisfiedAssumption, StopTest): report(traceback.format_exc()) raise Flaky('Unreliable assumption: An example which satisfied ' 'assumptions on the first run now fails it.') report( 'Failed to reproduce exception. Expected: \n' + self.last_exception, ) filter_message = ( 'Unreliable test data: Failed to reproduce a failure ' 'and then when it came to recreating the example in ' 'order to print the test data with a flaky result ' 'the example was filtered out (by e.g. a ' 'call to filter in your strategy) when we didn\'t ' 'expect it to be.') try: self.test_runner( ConjectureData.for_buffer(self.falsifying_example), reify_and_execute(self.search_strategy, test_is_flaky(self.test, self.repr_for_last_exception), print_example=True, is_final=True)) except (UnsatisfiedAssumption, StopTest): raise Flaky(filter_message)
def find(specifier, condition, settings=None, random=None, database_key=None): """Returns the minimal example from the given strategy ``specifier`` that matches the predicate function ``condition``.""" settings = settings or Settings( max_examples=2000, min_satisfying_examples=0, max_shrinks=2000, ) settings = Settings(settings, perform_health_check=False) if database_key is None and settings.database is not None: database_key = function_digest(condition) if not isinstance(specifier, SearchStrategy): raise InvalidArgument( 'Expected SearchStrategy but got %r of type %s' % ( specifier, type(specifier).__name__ )) specifier.validate() search = specifier random = random or new_random() successful_examples = [0] last_data = [None] def template_condition(data): with BuildContext(data): try: data.is_find = True result = data.draw(search) data.note(result) success = condition(result) except UnsatisfiedAssumption: data.mark_invalid() if success: successful_examples[0] += 1 if settings.verbosity == Verbosity.verbose: if not successful_examples[0]: report(lambda: u'Trying example %s' % ( nicerepr(result), )) elif success: if successful_examples[0] == 1: report(lambda: u'Found satisfying example %s' % ( nicerepr(result), )) last_data[0] = data elif ( sort_key(hbytes(data.buffer)) < sort_key(last_data[0].buffer) ): report(lambda: u'Shrunk example to %s' % ( nicerepr(result), )) last_data[0] = data if success and not data.frozen: data.mark_interesting() start = time.time() runner = ConjectureRunner( template_condition, settings=settings, random=random, database_key=database_key, ) runner.run() note_engine_for_statistics(runner) run_time = time.time() - start if runner.interesting_examples: data = ConjectureData.for_buffer( list(runner.interesting_examples.values())[0].buffer) with BuildContext(data): return data.draw(search) if ( runner.valid_examples <= settings.min_satisfying_examples and runner.exit_reason != ExitReason.finished ): if settings.timeout > 0 and run_time > settings.timeout: raise Timeout(( 'Ran out of time before finding enough valid examples for ' '%s. Only %d valid examples found in %.2f seconds.' ) % ( get_pretty_function_description(condition), runner.valid_examples, run_time)) else: raise Unsatisfiable(( 'Unable to satisfy assumptions of ' '%s. Only %d examples considered satisfied assumptions' ) % ( get_pretty_function_description(condition), runner.valid_examples,)) raise NoSuchExample(get_pretty_function_description(condition))
def run(self): # Tell pytest to omit the body of this function from tracebacks __tracebackhide__ = True if global_force_seed is None: database_key = str_to_bytes(fully_qualified_name(self.test)) else: database_key = None self.start_time = time.time() global in_given runner = ConjectureRunner( self.evaluate_test_data, settings=self.settings, random=self.random, database_key=database_key, ) if in_given or self.collector is None: runner.run() else: # pragma: no cover in_given = True original_trace = sys.gettrace() try: sys.settrace(None) runner.run() finally: in_given = False sys.settrace(original_trace) self.used_examples_from_database = \ runner.used_examples_from_database note_engine_for_statistics(runner) run_time = time.time() - self.start_time self.used_examples_from_database = runner.used_examples_from_database if runner.used_examples_from_database: if self.settings.derandomize: note_deprecation( 'In future derandomize will imply database=None, but your ' 'test is currently using examples from the database. To ' 'get the future behaviour, update your settings to ' 'include database=None.') if self.__had_seed: note_deprecation( 'In future use of @seed will imply database=None in your ' 'settings, but your test is currently using examples from ' 'the database. To get the future behaviour, update your ' 'settings for this test to include database=None.') timed_out = runner.exit_reason == ExitReason.timeout if runner.call_count == 0: return if runner.interesting_examples: self.falsifying_examples = sorted( [d for d in runner.interesting_examples.values()], key=lambda d: sort_key(d.buffer), reverse=True) else: if timed_out: note_deprecation(( 'Your tests are hitting the settings timeout (%.2fs). ' 'This functionality will go away in a future release ' 'and you should not rely on it. Instead, try setting ' 'max_examples to be some value lower than %d (the number ' 'of examples your test successfully ran here). Or, if you ' 'would prefer your tests to run to completion, regardless ' 'of how long they take, you can set the timeout value to ' 'hypothesis.unlimited.') % (self.settings.timeout, runner.valid_examples), self.settings) if runner.valid_examples == 0: if timed_out: raise Timeout( ('Ran out of time before finding a satisfying ' 'example for %s. Only found %d examples in %.2fs.') % (get_pretty_function_description( self.test), runner.valid_examples, run_time)) else: raise Unsatisfiable( 'Unable to satisfy assumptions of hypothesis %s.' % (get_pretty_function_description(self.test), )) if not self.falsifying_examples: return self.failed_normally = True flaky = 0 for falsifying_example in self.falsifying_examples: ran_example = ConjectureData.for_buffer(falsifying_example.buffer) self.__was_flaky = False assert falsifying_example.__expected_exception is not None try: self.execute(ran_example, print_example=True, is_final=True, expected_failure=( falsifying_example.__expected_exception, falsifying_example.__expected_traceback, )) except (UnsatisfiedAssumption, StopTest): report(traceback.format_exc()) self.__flaky( 'Unreliable assumption: An example which satisfied ' 'assumptions on the first run now fails it.') except BaseException: if len(self.falsifying_examples) <= 1: raise report(traceback.format_exc()) finally: # pragma: no cover # This section is in fact entirely covered by the tests in # test_reproduce_failure, but it seems to trigger a lovely set # of coverage bugs: The branches show up as uncovered (despite # definitely being covered - you can add an assert False else # branch to verify this and see it fail - and additionally the # second branch still complains about lack of coverage even if # you add a pragma: no cover to it! # See https://bitbucket.org/ned/coveragepy/issues/623/ if self.settings.print_blob is not PrintSettings.NEVER: failure_blob = encode_failure(falsifying_example.buffer) # Have to use the example we actually ran, not the original # falsifying example! Otherwise we won't catch problems # where the repr of the generated example doesn't parse. can_use_repr = ran_example.can_reproduce_example_from_repr if (self.settings.print_blob is PrintSettings.ALWAYS or (self.settings.print_blob is PrintSettings.INFER and not can_use_repr and len(failure_blob) < 200)): report(( '\n' 'You can reproduce this example by temporarily ' 'adding @reproduce_failure(%r, %r) as a decorator ' 'on your test case') % ( __version__, failure_blob, )) if self.__was_flaky: flaky += 1 # If we only have one example then we should have raised an error or # flaky prior to this point. assert len(self.falsifying_examples) > 1 if flaky > 0: raise Flaky( ('Hypothesis found %d distinct failures, but %d of them ' 'exhibited some sort of flaky behaviour.') % (len(self.falsifying_examples), flaky)) else: raise MultipleFailures(('Hypothesis found %d distinct failures.') % (len(self.falsifying_examples, )))
def run(self): # Tell pytest to omit the body of this function from tracebacks __tracebackhide__ = True if global_force_seed is None: database_key = function_digest(self.test) else: database_key = None runner = ConjectureRunner( self.evaluate_test_data, settings=self.settings, random=self.random, database_key=database_key, ) try: runner.run() finally: self.used_examples_from_database = runner.used_examples_from_database note_engine_for_statistics(runner) self.used_examples_from_database = runner.used_examples_from_database if runner.call_count == 0: return if runner.interesting_examples: self.falsifying_examples = sorted( [d for d in runner.interesting_examples.values()], key=lambda d: sort_key(d.buffer), reverse=True, ) else: if runner.valid_examples == 0: raise Unsatisfiable( "Unable to satisfy assumptions of hypothesis %s." % (get_pretty_function_description(self.test), )) if not self.falsifying_examples: return self.failed_normally = True flaky = 0 for falsifying_example in self.falsifying_examples: ran_example = ConjectureData.for_buffer(falsifying_example.buffer) self.__was_flaky = False assert falsifying_example.__expected_exception is not None try: self.execute( ran_example, print_example=True, is_final=True, expected_failure=( falsifying_example.__expected_exception, falsifying_example.__expected_traceback, ), ) except (UnsatisfiedAssumption, StopTest): report(traceback.format_exc()) self.__flaky( "Unreliable assumption: An example which satisfied " "assumptions on the first run now fails it.") except BaseException as e: if len(self.falsifying_examples) <= 1: raise tb = get_trimmed_traceback() report("".join(traceback.format_exception(type(e), e, tb))) finally: # pragma: no cover # This section is in fact entirely covered by the tests in # test_reproduce_failure, but it seems to trigger a lovely set # of coverage bugs: The branches show up as uncovered (despite # definitely being covered - you can add an assert False else # branch to verify this and see it fail - and additionally the # second branch still complains about lack of coverage even if # you add a pragma: no cover to it! # See https://bitbucket.org/ned/coveragepy/issues/623/ if self.settings.print_blob is not PrintSettings.NEVER: failure_blob = encode_failure(falsifying_example.buffer) # Have to use the example we actually ran, not the original # falsifying example! Otherwise we won't catch problems # where the repr of the generated example doesn't parse. can_use_repr = ran_example.can_reproduce_example_from_repr if self.settings.print_blob is PrintSettings.ALWAYS or ( self.settings.print_blob is PrintSettings.INFER and self.settings.verbosity >= Verbosity.normal and not can_use_repr and len(failure_blob) < 200): report(( "\nYou can reproduce this example by temporarily " "adding @reproduce_failure(%r, %r) as a decorator " "on your test case") % (__version__, failure_blob)) if self.__was_flaky: flaky += 1 # If we only have one example then we should have raised an error or # flaky prior to this point. assert len(self.falsifying_examples) > 1 if flaky > 0: raise Flaky( ("Hypothesis found %d distinct failures, but %d of them " "exhibited some sort of flaky behaviour.") % (len(self.falsifying_examples), flaky)) else: raise MultipleFailures(("Hypothesis found %d distinct failures.") % (len(self.falsifying_examples)))
def learner_for(strategy): """Returns an LStar learner that predicts whether a buffer corresponds to a discard free choice sequence leading to a valid value for this strategy.""" try: return LEARNERS[strategy] except KeyError: pass def test_function(data): data.draw(strategy) data.mark_interesting() runner = ConjectureRunner( test_function, settings=settings( database=None, verbosity=Verbosity.quiet, suppress_health_check=HealthCheck.all(), ), random=Random(0), ignore_limits=True, ) def predicate(s): result = runner.cached_test_function(s) if result.status < Status.VALID: return False if result.has_discards: return False return result.buffer == s learner = LStar(predicate) runner.run() (v,) = runner.interesting_examples.values() # We make sure the learner has properly learned small examples. # This is all fairly ad hoc but is mostly designed to get it # to understand what the smallest example is and avoid any # loops at the beginning of the DFA that don't really exist. learner.learn(v.buffer) for n in [1, 2, 3]: for _ in range(5): learner.learn(uniform(runner.random, n) + v.buffer) prev = -1 while learner.generation != prev: prev = learner.generation for _ in range(10): s = uniform(runner.random, len(v.buffer)) + bytes(BUFFER_SIZE) learner.learn(s) data = runner.cached_test_function(s) if data.status >= Status.VALID: learner.learn(data.buffer) LEARNERS[strategy] = learner return learner
def find( specifier, # type: SearchStrategy condition, # type: Callable[[Any], bool] settings=None, # type: Settings random=None, # type: Any database_key=None, # type: bytes ): # type: (...) -> Any """Returns the minimal example from the given strategy ``specifier`` that matches the predicate function ``condition``.""" if settings is None: settings = Settings(max_examples=2000) settings = Settings(settings, suppress_health_check=HealthCheck.all()) if database_key is None and settings.database is not None: database_key = function_digest(condition) if not isinstance(specifier, SearchStrategy): raise InvalidArgument("Expected SearchStrategy but got %r of type %s" % (specifier, type(specifier).__name__)) specifier.validate() search = specifier random = random or new_random() successful_examples = [0] last_data = [None] last_repr = [None] def template_condition(data): with BuildContext(data): try: data.is_find = True with deterministic_PRNG(): result = data.draw(search) data.note(result) success = condition(result) except UnsatisfiedAssumption: data.mark_invalid() if success: successful_examples[0] += 1 if settings.verbosity >= Verbosity.verbose: if not successful_examples[0]: report(u"Tried non-satisfying example %s" % (nicerepr(result), )) elif success: if successful_examples[0] == 1: last_repr[0] = nicerepr(result) report(u"Found satisfying example %s" % (last_repr[0], )) last_data[0] = data elif (sort_key(hbytes(data.buffer)) < sort_key( last_data[0].buffer) ) and nicerepr(result) != last_repr[0]: last_repr[0] = nicerepr(result) report(u"Shrunk example to %s" % (last_repr[0], )) last_data[0] = data if success and not data.frozen: data.mark_interesting() runner = ConjectureRunner(template_condition, settings=settings, random=random, database_key=database_key) runner.run() note_engine_for_statistics(runner) if runner.interesting_examples: data = ConjectureData.for_buffer( list(runner.interesting_examples.values())[0].buffer) with BuildContext(data): with deterministic_PRNG(): return data.draw(search) if runner.valid_examples == 0 and (runner.exit_reason != ExitReason.finished): raise Unsatisfiable("Unable to satisfy assumptions of %s." % (get_pretty_function_description(condition), )) raise NoSuchExample(get_pretty_function_description(condition))