def test_clears_out_database_as_things_get_boring(): key = b"a database key" database = InMemoryExampleDatabase() do_we_care = True def stuff(): try: find( st.binary(min_size=50), lambda x: do_we_care and has_a_non_zero_byte(x), settings=settings(database=database, max_examples=10), database_key=key ) except NoSuchExample: pass stuff() assert len(all_values(database)) > 1 do_we_care = False stuff() initial = len(all_values(database)) assert initial > 0 for _ in range(initial): stuff() keys = len(all_values(database)) if not keys: break else: assert False
def test_clears_out_database_as_things_get_boring(): key = b'a database key' database = InMemoryExampleDatabase() do_we_care = True def stuff(): try: find(st.binary(min_size=50), lambda x: do_we_care and has_a_non_zero_byte(x), settings=settings(database=database, max_examples=10), database_key=key) except NoSuchExample: pass stuff() assert len(all_values(database)) > 1 do_we_care = False stuff() initial = len(all_values(database)) assert initial > 0 for _ in range(initial): stuff() keys = len(all_values(database)) if not keys: break else: assert False
def test_trashes_invalid_examples(): key = b"a database key" database = InMemoryExampleDatabase() invalid = set() def stuff(): try: def condition(x): assume(x not in invalid) return not invalid and has_a_non_zero_byte(x) return find( st.binary(min_size=5), condition, settings=settings(database=database), database_key=key, ) except (Unsatisfiable, NoSuchExample): pass with deterministic_PRNG(): value = stuff() original = len(all_values(database)) assert original > 1 invalid.add(value) with deterministic_PRNG(): stuff() assert len(all_values(database)) < original
def test_will_shrink_covering_examples(): best = [None] replaced = [] def tagged(data): b = hbytes(data.draw_bytes(4)) if any(b): data.add_tag('nonzero') if best[0] is None: best[0] = b elif b < best[0]: replaced.append(best[0]) best[0] = b db = InMemoryExampleDatabase() runner = ConjectureRunner(tagged, settings=settings( max_examples=100, phases=no_shrink, buffer_size=1024, database=db, ), database_key=b'stuff') runner.run() saved = set(all_values(db)) assert best[0] in saved for r in replaced: assert r not in saved
def test_will_save_covering_examples(): tags = {} def tagged(data): b = hbytes(data.draw_bytes(4)) try: tag = tags[b] except KeyError: if len(tags) < 10: tag = len(tags) tags[b] = tag else: tag = None if tag is not None: data.add_tag(tag) db = InMemoryExampleDatabase() runner = ConjectureRunner(tagged, settings=settings( max_examples=100, phases=no_shrink, buffer_size=1024, database=db, ), database_key=b'stuff') runner.run() assert len(all_values(db)) == len(tags)
def test_respects_max_examples_in_database_usage(): key = b"a database key" database = InMemoryExampleDatabase() do_we_care = True counter = [0] def check(x): counter[0] += 1 return do_we_care and has_a_non_zero_byte(x) def stuff(): try: find( st.binary(min_size=100), check, settings=settings(database=database, max_examples=10), database_key=key, ) except NoSuchExample: pass with deterministic_PRNG(): stuff() assert len(all_values(database)) > 10 do_we_care = False counter[0] = 0 stuff() assert counter == [10]
def test_will_shrink_covering_examples(): best = [None] replaced = [] def tagged(data): b = hbytes(data.draw_bytes(4)) if any(b): data.add_tag('nonzero') if best[0] is None: best[0] = b elif b < best[0]: replaced.append(best[0]) best[0] = b db = InMemoryExampleDatabase() runner = ConjectureRunner(tagged, settings=settings( max_examples=100, max_iterations=10000, max_shrinks=0, buffer_size=1024, database=db, ), database_key=b'stuff') runner.run() saved = set(all_values(db)) assert best[0] in saved for r in replaced: assert r not in saved
def test_does_print_on_reuse_from_database(): passes_healthcheck = False database = InMemoryExampleDatabase() @settings(database=database) @given(st.integers()) def test(i): assume(passes_healthcheck) raise ValueError() with capture_out() as o: with pytest.raises(FailedHealthCheck): test() assert "@seed" in o.getvalue() passes_healthcheck = True with capture_out() as o: with pytest.raises(ValueError): test() assert all_values(database) assert "@seed" not in o.getvalue() passes_healthcheck = False with capture_out() as o: with pytest.raises(FailedHealthCheck): test() assert "@seed" in o.getvalue()
def test_respects_max_examples_in_database_usage(): key = b"a database key" database = InMemoryExampleDatabase() do_we_care = True counter = [0] def check(x): counter[0] += 1 return do_we_care and has_a_non_zero_byte(x) def stuff(): try: find( st.binary(min_size=100), check, settings=settings(database=database, max_examples=10), database_key=key ) except NoSuchExample: pass stuff() assert len(all_values(database)) > 10 do_we_care = False counter[0] = 0 stuff() assert counter == [10]
def test_does_print_on_reuse_from_database(): passes_healthcheck = False database = InMemoryExampleDatabase() @settings(database=database) @given(st.integers()) def test(i): assume(passes_healthcheck) raise ValueError() with capture_out() as o: with pytest.raises(FailedHealthCheck): test() assert '@seed' in o.getvalue() passes_healthcheck = True with capture_out() as o: with pytest.raises(ValueError): test() assert all_values(database) assert '@seed' not in o.getvalue() passes_healthcheck = False with capture_out() as o: with pytest.raises(FailedHealthCheck): test() assert '@seed' in o.getvalue()
def test_clears_out_everything_smaller_than_the_interesting_example(): in_clearing = False target = [None] for _ in range(5): # We retry the test run a few times to get a large enough initial # set of examples that we're not going to explore them all in the # initial run. cache = {} seen = set() database = InMemoryExampleDatabase() @settings(database=database, verbosity=Verbosity.quiet, max_examples=100) @given(st.binary(min_size=10, max_size=10)) def test(i): if not in_clearing: if len([b for b in i if b > 1]) >= 8: assert cache.setdefault(i, len(cache) % 10 != 9) elif len(seen) <= 20: seen.add(i) else: if target[0] is None: remainder = sorted([s for s in saved if s not in seen]) target[0] = remainder[len(remainder) // 2] assert i in seen or i < target[0] with pytest.raises(AssertionError): test() saved = all_values(database) if len(saved) > 30: break else: assert False, 'Never generated enough examples while shrinking' in_clearing = True with pytest.raises(AssertionError): test() saved = all_values(database) for s in saved: assert s >= target[0]
def test_saves_incremental_steps_in_database(): key = b"a database key" database = InMemoryExampleDatabase() find( st.binary(min_size=10), lambda x: has_a_non_zero_byte(x), settings=settings(database=database), database_key=key ) assert len(all_values(database)) > 1
def test_saves_incremental_steps_in_database(): key = b'a database key' database = InMemoryExampleDatabase() find(st.binary(min_size=10), lambda x: has_a_non_zero_byte(x), settings=settings(database=database), database_key=key) assert len(all_values(database)) > 1
def test_trashes_invalid_examples(): key = b'a database key' database = InMemoryExampleDatabase() finicky = False def stuff(): try: find(st.binary(min_size=100), lambda x: assume(not finicky) and has_a_non_zero_byte(x), settings=settings(database=database, max_shrinks=10), database_key=key) except Unsatisfiable: pass stuff() original = len(all_values(database)) assert original > 1 finicky = True stuff() assert len(all_values(database)) < original
def test_trashes_invalid_examples(): key = b"a database key" database = InMemoryExampleDatabase() finicky = False def stuff(): try: find( st.binary(min_size=100), lambda x: assume(not finicky) and has_a_non_zero_byte(x), settings=settings(database=database, max_shrinks=10), database_key=key ) except Unsatisfiable: pass stuff() original = len(all_values(database)) assert original > 1 finicky = True stuff() assert len(all_values(database)) < original
def test_will_shrink_covering_examples(): seen = [hbytes([255] * 4)] def tagged(data): seen[0] = min(seen[0], hbytes(data.draw_bytes(4))) data.add_tag(0) db = InMemoryExampleDatabase() runner = ConjectureRunner(tagged, settings=settings( max_examples=100, max_iterations=10000, max_shrinks=0, buffer_size=1024, database=db, ), database_key=b'stuff') runner.run() assert all_values(db) == set(seen)
def test_tracks_and_saves_coverage(): db = InMemoryExampleDatabase() def do_nothing(): """Use in place of pass for empty branches, which don't show up under coverge.""" pass @settings(database=db) @given(st.integers()) def test_branching(i): if i < -1000: do_nothing() elif i > 1000: do_nothing() else: do_nothing() test_branching() assert len(all_values(db)) == 3
def test_will_save_covering_examples(): tags = {} def tagged(data): b = hbytes(data.draw_bytes(4)) try: tag = tags[b] except KeyError: if len(tags) < 10: tag = len(tags) tags[b] = tag else: tag = None if tag is not None: data.add_tag(tag) db = InMemoryExampleDatabase() runner = ConjectureRunner(tagged, settings=settings( max_examples=100, max_iterations=10000, max_shrinks=0, buffer_size=1024, database=db, ), database_key=b'stuff') runner.run() assert len(all_values(db)) == len(tags)