def test_clears_out_database_as_things_get_boring():
key = b"a database key"
database = InMemoryExampleDatabase()
do_we_care = True
def stuff():
try:
find(
st.binary(min_size=50),
lambda x: do_we_care and has_a_non_zero_byte(x),
settings=settings(database=database, max_examples=10),
database_key=key
)
except NoSuchExample:
pass
stuff()
assert len(all_values(database)) > 1
do_we_care = False
stuff()
initial = len(all_values(database))
assert initial > 0
for _ in range(initial):
stuff()
keys = len(all_values(database))
if not keys:
break
else:
assert False
def test_clears_out_database_as_things_get_boring():
key = b'a database key'
database = InMemoryExampleDatabase()
do_we_care = True
def stuff():
try:
find(st.binary(min_size=50),
lambda x: do_we_care and has_a_non_zero_byte(x),
settings=settings(database=database, max_examples=10),
database_key=key)
except NoSuchExample:
pass
stuff()
assert len(all_values(database)) > 1
do_we_care = False
stuff()
initial = len(all_values(database))
assert initial > 0
for _ in range(initial):
stuff()
keys = len(all_values(database))
if not keys:
break
else:
assert False
def test_trashes_invalid_examples():
key = b"a database key"
database = InMemoryExampleDatabase()
invalid = set()
def stuff():
try:
def condition(x):
assume(x not in invalid)
return not invalid and has_a_non_zero_byte(x)
return find(
st.binary(min_size=5),
condition,
settings=settings(database=database),
database_key=key,
)
except (Unsatisfiable, NoSuchExample):
pass
with deterministic_PRNG():
value = stuff()
original = len(all_values(database))
assert original > 1
invalid.add(value)
with deterministic_PRNG():
stuff()
assert len(all_values(database)) < original
def test_will_shrink_covering_examples():
best = [None]
replaced = []
def tagged(data):
b = hbytes(data.draw_bytes(4))
if any(b):
data.add_tag('nonzero')
if best[0] is None:
best[0] = b
elif b < best[0]:
replaced.append(best[0])
best[0] = b
db = InMemoryExampleDatabase()
runner = ConjectureRunner(tagged,
settings=settings(
max_examples=100,
phases=no_shrink,
buffer_size=1024,
database=db,
),
database_key=b'stuff')
runner.run()
saved = set(all_values(db))
assert best[0] in saved
for r in replaced:
assert r not in saved
def test_will_save_covering_examples():
tags = {}
def tagged(data):
b = hbytes(data.draw_bytes(4))
try:
tag = tags[b]
except KeyError:
if len(tags) < 10:
tag = len(tags)
tags[b] = tag
else:
tag = None
if tag is not None:
data.add_tag(tag)
db = InMemoryExampleDatabase()
runner = ConjectureRunner(tagged,
settings=settings(
max_examples=100,
phases=no_shrink,
buffer_size=1024,
database=db,
),
database_key=b'stuff')
runner.run()
assert len(all_values(db)) == len(tags)
def test_respects_max_examples_in_database_usage():
key = b"a database key"
database = InMemoryExampleDatabase()
do_we_care = True
counter = [0]
def check(x):
counter[0] += 1
return do_we_care and has_a_non_zero_byte(x)
def stuff():
try:
find(
st.binary(min_size=100),
check,
settings=settings(database=database, max_examples=10),
database_key=key,
)
except NoSuchExample:
pass
with deterministic_PRNG():
stuff()
assert len(all_values(database)) > 10
do_we_care = False
counter[0] = 0
stuff()
assert counter == [10]
def test_will_shrink_covering_examples():
best = [None]
replaced = []
def tagged(data):
b = hbytes(data.draw_bytes(4))
if any(b):
data.add_tag('nonzero')
if best[0] is None:
best[0] = b
elif b < best[0]:
replaced.append(best[0])
best[0] = b
db = InMemoryExampleDatabase()
runner = ConjectureRunner(tagged, settings=settings(
max_examples=100, max_iterations=10000, max_shrinks=0,
buffer_size=1024,
database=db,
), database_key=b'stuff')
runner.run()
saved = set(all_values(db))
assert best[0] in saved
for r in replaced:
assert r not in saved
def test_does_print_on_reuse_from_database():
passes_healthcheck = False
database = InMemoryExampleDatabase()
@settings(database=database)
@given(st.integers())
def test(i):
assume(passes_healthcheck)
raise ValueError()
with capture_out() as o:
with pytest.raises(FailedHealthCheck):
test()
assert "@seed" in o.getvalue()
passes_healthcheck = True
with capture_out() as o:
with pytest.raises(ValueError):
test()
assert all_values(database)
assert "@seed" not in o.getvalue()
passes_healthcheck = False
with capture_out() as o:
with pytest.raises(FailedHealthCheck):
test()
assert "@seed" in o.getvalue()
def test_respects_max_examples_in_database_usage():
key = b"a database key"
database = InMemoryExampleDatabase()
do_we_care = True
counter = [0]
def check(x):
counter[0] += 1
return do_we_care and has_a_non_zero_byte(x)
def stuff():
try:
find(
st.binary(min_size=100), check,
settings=settings(database=database, max_examples=10),
database_key=key
)
except NoSuchExample:
pass
stuff()
assert len(all_values(database)) > 10
do_we_care = False
counter[0] = 0
stuff()
assert counter == [10]
def test_does_print_on_reuse_from_database():
passes_healthcheck = False
database = InMemoryExampleDatabase()
@settings(database=database)
@given(st.integers())
def test(i):
assume(passes_healthcheck)
raise ValueError()
with capture_out() as o:
with pytest.raises(FailedHealthCheck):
test()
assert '@seed' in o.getvalue()
passes_healthcheck = True
with capture_out() as o:
with pytest.raises(ValueError):
test()
assert all_values(database)
assert '@seed' not in o.getvalue()
passes_healthcheck = False
with capture_out() as o:
with pytest.raises(FailedHealthCheck):
test()
assert '@seed' in o.getvalue()
def test_clears_out_everything_smaller_than_the_interesting_example():
in_clearing = False
target = [None]
for _ in range(5):
# We retry the test run a few times to get a large enough initial
# set of examples that we're not going to explore them all in the
# initial run.
cache = {}
seen = set()
database = InMemoryExampleDatabase()
@settings(database=database,
verbosity=Verbosity.quiet,
max_examples=100)
@given(st.binary(min_size=10, max_size=10))
def test(i):
if not in_clearing:
if len([b for b in i if b > 1]) >= 8:
assert cache.setdefault(i, len(cache) % 10 != 9)
elif len(seen) <= 20:
seen.add(i)
else:
if target[0] is None:
remainder = sorted([s for s in saved if s not in seen])
target[0] = remainder[len(remainder) // 2]
assert i in seen or i < target[0]
with pytest.raises(AssertionError):
test()
saved = all_values(database)
if len(saved) > 30:
break
else:
assert False, 'Never generated enough examples while shrinking'
in_clearing = True
with pytest.raises(AssertionError):
test()
saved = all_values(database)
for s in saved:
assert s >= target[0]
def test_saves_incremental_steps_in_database():
key = b"a database key"
database = InMemoryExampleDatabase()
find(
st.binary(min_size=10), lambda x: has_a_non_zero_byte(x),
settings=settings(database=database), database_key=key
)
assert len(all_values(database)) > 1
def test_saves_incremental_steps_in_database():
key = b'a database key'
database = InMemoryExampleDatabase()
find(st.binary(min_size=10),
lambda x: has_a_non_zero_byte(x),
settings=settings(database=database),
database_key=key)
assert len(all_values(database)) > 1
def test_trashes_invalid_examples():
key = b'a database key'
database = InMemoryExampleDatabase()
finicky = False
def stuff():
try:
find(st.binary(min_size=100),
lambda x: assume(not finicky) and has_a_non_zero_byte(x),
settings=settings(database=database, max_shrinks=10),
database_key=key)
except Unsatisfiable:
pass
stuff()
original = len(all_values(database))
assert original > 1
finicky = True
stuff()
assert len(all_values(database)) < original
def test_trashes_invalid_examples():
key = b"a database key"
database = InMemoryExampleDatabase()
finicky = False
def stuff():
try:
find(
st.binary(min_size=100),
lambda x: assume(not finicky) and has_a_non_zero_byte(x),
settings=settings(database=database, max_shrinks=10),
database_key=key
)
except Unsatisfiable:
pass
stuff()
original = len(all_values(database))
assert original > 1
finicky = True
stuff()
assert len(all_values(database)) < original
def test_will_shrink_covering_examples():
seen = [hbytes([255] * 4)]
def tagged(data):
seen[0] = min(seen[0], hbytes(data.draw_bytes(4)))
data.add_tag(0)
db = InMemoryExampleDatabase()
runner = ConjectureRunner(tagged,
settings=settings(
max_examples=100,
max_iterations=10000,
max_shrinks=0,
buffer_size=1024,
database=db,
),
database_key=b'stuff')
runner.run()
assert all_values(db) == set(seen)
def test_tracks_and_saves_coverage():
db = InMemoryExampleDatabase()
def do_nothing():
"""Use in place of pass for empty branches, which don't show up under
coverge."""
pass
@settings(database=db)
@given(st.integers())
def test_branching(i):
if i < -1000:
do_nothing()
elif i > 1000:
do_nothing()
else:
do_nothing()
test_branching()
assert len(all_values(db)) == 3
def test_tracks_and_saves_coverage():
db = InMemoryExampleDatabase()
def do_nothing():
"""Use in place of pass for empty branches, which don't show up under
coverge."""
pass
@settings(database=db)
@given(st.integers())
def test_branching(i):
if i < -1000:
do_nothing()
elif i > 1000:
do_nothing()
else:
do_nothing()
test_branching()
assert len(all_values(db)) == 3
def test_will_save_covering_examples():
tags = {}
def tagged(data):
b = hbytes(data.draw_bytes(4))
try:
tag = tags[b]
except KeyError:
if len(tags) < 10:
tag = len(tags)
tags[b] = tag
else:
tag = None
if tag is not None:
data.add_tag(tag)
db = InMemoryExampleDatabase()
runner = ConjectureRunner(tagged, settings=settings(
max_examples=100, max_iterations=10000, max_shrinks=0,
buffer_size=1024,
database=db,
), database_key=b'stuff')
runner.run()
assert len(all_values(db)) == len(tags)