def run_benchmark_for_sizes(benchmark, n_runs):
click.echo('Calculating data for %s' % (benchmark.name,))
total_sizes = []
with click.progressbar(range(n_runs)) as runs:
for _ in runs:
sizes = []
valid_seed = random.getrandbits(64).to_bytes(8, 'big')
interesting_seed = random.getrandbits(64).to_bytes(8, 'big')
def test_function(data):
try:
try:
value = data.draw(benchmark.strategy)
except UnsatisfiedAssumption:
data.mark_invalid()
if not data.frozen:
if not benchmark.valid(valid_seed, data, value):
data.mark_invalid()
if benchmark.interesting(
interesting_seed, data, value
):
data.mark_interesting()
finally:
sizes.append(len(data.buffer))
engine = ConjectureRunner(
test_function, settings=BENCHMARK_SETTINGS, random=random
)
engine.run()
assert len(sizes) > 0
total_sizes.append(sum(sizes))
return total_sizes
def test_will_shrink_covering_examples():
best = [None]
replaced = []
def tagged(data):
b = hbytes(data.draw_bytes(4))
if any(b):
data.add_tag('nonzero')
if best[0] is None:
best[0] = b
elif b < best[0]:
replaced.append(best[0])
best[0] = b
db = InMemoryExampleDatabase()
runner = ConjectureRunner(tagged, settings=settings(
max_examples=100, max_iterations=10000, max_shrinks=0,
buffer_size=1024,
database=db,
), database_key=b'stuff')
runner.run()
saved = set(all_values(db))
assert best[0] in saved
for r in replaced:
assert r not in saved
def test_saves_data_while_shrinking():
key = b'hi there'
n = 5
db = ExampleDatabase(':memory:')
assert list(db.fetch(key)) == []
seen = set()
def f(data):
x = data.draw_bytes(512)
if sum(x) >= 5000 and len(seen) < n:
seen.add(hbytes(x))
if hbytes(x) in seen:
data.mark_interesting()
runner = ConjectureRunner(
f, settings=settings(database=db), database_key=key)
runner.run()
assert runner.last_data.status == Status.INTERESTING
assert len(seen) == n
in_db = set(
v
for vs in db.data.values()
for v in vs
)
assert in_db.issubset(seen)
assert in_db == seen
def test_clears_out_its_database_on_shrinking(
initial_attempt, skip_target, monkeypatch
):
def generate_new_examples(self):
self.test_function(
ConjectureData.for_buffer(hbytes([initial_attempt])))
monkeypatch.setattr(
ConjectureRunner, 'generate_new_examples', generate_new_examples)
key = b'key'
db = InMemoryExampleDatabase()
def f(data):
if data.draw_bits(8) >= 127:
data.mark_interesting()
runner = ConjectureRunner(
f, settings=settings(database=db, max_examples=256), database_key=key,
random=Random(0),
)
for n in hrange(256):
if n != 127 or not skip_target:
db.save(runner.secondary_key, hbytes([n]))
runner.run()
assert len(runner.interesting_examples) == 1
for b in db.fetch(runner.secondary_key):
assert b[0] >= 127
assert len(list(db.fetch(runner.database_key))) == 1
def test_can_delete_intervals(monkeypatch):
def generate_new_examples(self):
self.test_function(
ConjectureData.for_buffer(hbytes([255] * 10 + [1, 3])))
monkeypatch.setattr(
ConjectureRunner, 'generate_new_examples', generate_new_examples)
monkeypatch.setattr(
Shrinker, 'shrink', fixate(Shrinker.adaptive_example_deletion)
)
def f(data):
while True:
n = data.draw_bits(8)
if n == 255:
continue
elif n == 1:
break
else:
data.mark_invalid()
if data.draw_bits(8) == 3:
data.mark_interesting()
runner = ConjectureRunner(f, settings=settings(database=None))
runner.run()
x, = runner.interesting_examples.values()
assert x.buffer == hbytes([1, 3])
def test_can_cover_without_a_database_key():
def tagged(data):
data.add_tag(0)
runner = ConjectureRunner(tagged, settings=settings(), database_key=None)
runner.run()
assert len(runner.covering_examples) == 1
def test_terminates_shrinks(n, monkeypatch):
from hypothesis.internal.conjecture import engine
db = InMemoryExampleDatabase()
def generate_new_examples(self):
def draw_bytes(data, n):
return hbytes([255] * n)
self.test_function(self.new_conjecture_data(draw_bytes))
monkeypatch.setattr(
ConjectureRunner, "generate_new_examples", generate_new_examples
)
monkeypatch.setattr(engine, "MAX_SHRINKS", n)
runner = ConjectureRunner(
slow_shrinker(),
settings=settings(max_examples=5000, database=db),
random=Random(0),
database_key=b"key",
)
runner.run()
last_data, = runner.interesting_examples.values()
assert last_data.status == Status.INTERESTING
assert runner.shrinks == n
in_db = set(db.data[runner.secondary_key])
assert len(in_db) == n
def test_run_nothing():
def f(data):
assert False
runner = ConjectureRunner(f, settings=settings(phases=()))
runner.run()
assert runner.call_count == 0
def test_garbage_collects_the_database():
key = b'hi there'
n = 200
db = ExampleDatabase(':memory:')
local_settings = settings(
database=db, max_shrinks=n, timeout=unlimited)
runner = ConjectureRunner(
slow_shrinker(), settings=local_settings, database_key=key)
runner.run()
assert runner.last_data.status == Status.INTERESTING
def in_db():
return set(
v
for vs in db.data.values()
for v in vs
)
assert len(in_db()) == n + 1
runner = ConjectureRunner(
lambda data: data.draw_bytes(4),
settings=local_settings, database_key=key)
runner.run()
assert 0 < len(in_db()) < n
def test_garbage_collects_the_database():
key = b'hi there'
n = 200
db = ExampleDatabase(':memory:')
assert list(db.fetch(key)) == []
seen = set()
go = True
def f(data):
x = hbytes(data.draw_bytes(512))
if not go:
return
if sum(x) >= 5000 and len(seen) < n:
seen.add(x)
if x in seen:
data.mark_interesting()
runner = ConjectureRunner(
f, settings=settings(database=db, max_shrinks=2 * n), database_key=key)
runner.run()
assert runner.last_data.status == Status.INTERESTING
assert len(seen) == n
assert set(db.fetch(key)) == seen
go = False
runner = ConjectureRunner(
f, settings=settings(database=db, max_shrinks=2 * n), database_key=key)
runner.run()
assert 0 < len(set(db.fetch(key))) < n
def test_saves_data_while_shrinking(monkeypatch):
key = b'hi there'
n = 5
db = InMemoryExampleDatabase()
assert list(db.fetch(key)) == []
seen = set()
monkeypatch.setattr(
ConjectureRunner, 'generate_new_examples',
lambda runner: runner.test_function(
ConjectureData.for_buffer([255] * 10)))
def f(data):
x = data.draw_bytes(10)
if sum(x) >= 2000 and len(seen) < n:
seen.add(hbytes(x))
if hbytes(x) in seen:
data.mark_interesting()
runner = ConjectureRunner(
f, settings=settings(database=db), database_key=key)
runner.run()
assert runner.interesting_examples
assert len(seen) == n
in_db = non_covering_examples(db)
assert in_db.issubset(seen)
assert in_db == seen
def run_to_data(f):
with deterministic_PRNG():
runner = ConjectureRunner(f, settings=TEST_SETTINGS)
runner.run()
assert runner.interesting_examples
last_data, = runner.interesting_examples.values()
return last_data
def test_debug_data(capsys):
buf = [0, 1, 2]
def f(data):
for x in hbytes(buf):
if data.draw_bits(8) != x:
data.mark_invalid()
data.start_example(1)
data.stop_example()
data.mark_interesting()
runner = ConjectureRunner(
f,
settings=settings(
max_examples=5000,
buffer_size=1024,
database=None,
suppress_health_check=HealthCheck.all(),
verbosity=Verbosity.debug,
),
)
runner.cached_test_function(buf)
runner.run()
out, _ = capsys.readouterr()
assert re.match(u"\\d+ bytes \\[.*\\] -> ", out)
assert "INTERESTING" in out
def test_exhaust_space():
with deterministic_PRNG():
runner = ConjectureRunner(
lambda data: data.draw_bits(1), settings=TEST_SETTINGS
)
runner.run()
assert runner.tree.is_exhausted
assert runner.valid_examples == 2
def test_note_events(event):
def f(data):
data.note_event(event)
data.draw_bytes(1)
runner = ConjectureRunner(f)
runner.run()
assert runner.event_call_counts[str(event)] == runner.call_count > 0
def run_to_buffer(f):
runner = ConjectureRunner(f, settings=settings(
max_examples=5000, max_iterations=10000, max_shrinks=MAX_SHRINKS,
buffer_size=1024,
database=None,
))
runner.run()
assert runner.last_data.status == Status.INTERESTING
return hbytes(runner.last_data.buffer)
def test_stops_after_max_examples_when_generating():
seen = []
def f(data):
seen.append(data.draw_bytes(1))
runner = ConjectureRunner(f, settings=settings(max_examples=1, database=None))
runner.run()
assert len(seen) == 1
def run_to_buffer(f):
with deterministic_PRNG():
runner = ConjectureRunner(f, settings=settings(
max_examples=5000, buffer_size=1024,
database=None, suppress_health_check=HealthCheck.all(),
))
runner.run()
assert runner.interesting_examples
last_data, = runner.interesting_examples.values()
return hbytes(last_data.buffer)
def test_can_navigate_to_a_valid_example():
def f(data):
i = int_from_bytes(data.draw_bytes(2))
data.draw_bytes(i)
data.mark_interesting()
runner = ConjectureRunner(f, settings=settings(
max_examples=5000, buffer_size=2, database=None,
))
runner.run()
assert runner.interesting_examples
def test_run_with_timeout_while_boring():
def f(data):
time.sleep(0.1)
runner = ConjectureRunner(
f, settings=settings(database=None, timeout=0.2))
start = time.time()
runner.run()
assert time.time() <= start + 1
assert runner.valid_examples > 0
def test_run_with_timeout_while_boring():
def f(data):
time.sleep(0.1)
runner = ConjectureRunner(
f, settings=settings(database=None, timeout=0.2))
start = time.time()
runner.run()
assert time.time() <= start + 1
assert runner.last_data.status == Status.VALID
def accept(f):
runner = ConjectureRunner(f, settings=settings(
max_examples=100, phases=no_shrink, buffer_size=1024,
database=None, **kwargs
))
with pytest.raises(FailedHealthCheck) as e:
runner.run()
assert e.value.health_check == label
assert not runner.interesting_examples
def test_max_shrinks_can_disable_shrinking():
seen = set()
def f(data):
seen.add(hbytes(data.draw_bytes(32)))
data.mark_interesting()
runner = ConjectureRunner(
f, settings=settings(database=None, max_shrinks=0,))
runner.run()
assert len(seen) == 1
def x(data):
rnd = Random(data.draw_bytes(1))
def g(d2):
d2.draw_bytes(1)
data.mark_interesting()
runner = ConjectureRunner(g, random=rnd)
children.append(runner)
runner.run()
if runner.interesting_examples:
data.mark_interesting()
def test_phases_can_disable_shrinking():
seen = set()
def f(data):
seen.add(hbytes(data.draw_bytes(32)))
data.mark_interesting()
runner = ConjectureRunner(f, settings=settings(
database=None, phases=(Phase.reuse, Phase.generate),
))
runner.run()
assert len(seen) == 1
def test_saves_on_interrupt():
def interrupts(data):
raise KeyboardInterrupt()
db = InMemoryExampleDatabase()
runner = ConjectureRunner(
interrupts, settings=settings(database=db), database_key=b'key')
with pytest.raises(KeyboardInterrupt):
runner.run()
assert db.data
def test_uniqueness_is_preserved_when_writing_at_beginning():
seen = set()
def f(data):
data.write(hbytes(1))
n = data.draw_bits(3)
assert n not in seen
seen.add(n)
runner = ConjectureRunner(f, settings=settings(max_examples=50))
runner.run()
assert runner.valid_examples == len(seen)
def test_can_navigate_to_a_valid_example():
def f(data):
i = int_from_bytes(data.draw_bytes(2))
data.draw_bytes(i)
data.mark_interesting()
runner = ConjectureRunner(f, settings=settings(
max_examples=5000, max_iterations=10000,
buffer_size=2,
database=None,
))
runner.run()
assert runner.last_data.status == Status.INTERESTING
return hbytes(runner.last_data.buffer)
def test_detects_flakiness():
failed_once = [False]
count = [0]
def tf(data):
data.draw_bytes(1)
count[0] += 1
if not failed_once[0]:
failed_once[0] = True
data.mark_interesting()
runner = ConjectureRunner(tf)
runner.run()
assert count == [2]
def test_run_with_timeout_while_shrinking():
def f(data):
time.sleep(0.1)
x = data.draw_bytes(32)
if any(x):
data.mark_interesting()
runner = ConjectureRunner(
f, settings=settings(database=None, timeout=0.2))
start = time.time()
runner.run()
assert time.time() <= start + 1
assert runner.interesting_examples
def test_stops_after_max_iterations_when_reading():
key = b'key'
max_iterations = 1
db = ExampleDatabase(':memory:')
for i in range(10):
db.save(key, hbytes([i]))
seen = []
def f(data):
seen.append(data.draw_bytes(1))
data.mark_invalid()
runner = ConjectureRunner(f, settings=settings(
max_examples=1, max_iterations=max_iterations,
database=db,
), database_key=key)
runner.run()
assert len(seen) == max_iterations
def test_can_delete_intervals(monkeypatch):
def generate_new_examples(self):
self.test_function(
ConjectureData.for_buffer(hbytes([255] * 10 + [0])))
monkeypatch.setattr(
ConjectureRunner, 'generate_new_examples', generate_new_examples)
monkeypatch.setattr(
Shrinker, 'shrink', Shrinker.adaptive_example_deletion
)
def f(data):
if data.draw_bits(1):
while data.draw_bits(8):
pass
data.mark_interesting()
runner = ConjectureRunner(f, settings=settings(database=None))
runner.run()
x, = runner.interesting_examples.values()
assert x.buffer == hbytes([1, 0])
def run_test():
if condition is None:
def _condition(x):
return True
condition_string = u''
else:
_condition = condition
condition_string = strip_lambda(
reflection.get_pretty_function_description(condition))
def test_function(data):
try:
value = data.draw(specifier)
except UnsatisfiedAssumption:
data.mark_invalid()
if not _condition(value):
data.mark_invalid()
if predicate(value):
data.mark_interesting()
successes = 0
for _ in range(RUNS):
runner = ConConjectureRunner(test_function,
settings=Settings(max_examples=200,
max_iterations=1000,
max_shrinks=0))
runner.run()
if runner.last_data.status == Status.INTERESTING:
successes += 1
if successes >= REQUIRED_RUNS:
return
event = condition_string
if condition is not None:
event += '|'
event += condition_string
description = (u'P(%s) ~ %d / %d = %.2f < %.2f') % (
event, successes, RUNS, successes / RUNS, (REQUIRED_RUNS / RUNS))
raise HypothesisFalsified(description + u' rejected')
def run_language_test_for(root, data, seed):
random.seed(seed)
def test(local_data):
node = root
while not isinstance(node, Terminal):
if isinstance(node, Write):
local_data.write(hbytes(node.value))
node = node.child
else:
assert isinstance(node, Branch)
c = local_data.draw_bits(node.bits)
try:
node = node.children[c]
except KeyError:
if data is None:
return
node = node.children.setdefault(c, data.draw(nodes))
assert isinstance(node, Terminal)
if node.status == Status.INTERESTING:
local_data.mark_interesting(node.payload)
elif node.status == Status.INVALID:
local_data.mark_invalid()
runner = ConjectureRunner(
test,
settings=settings(
max_examples=1,
buffer_size=512,
database=None,
suppress_health_check=HealthCheck.all(),
verbosity=Verbosity.quiet,
phases=list(Phase),
),
)
try:
runner.run()
finally:
if data is not None:
note(root)
assume(runner.interesting_examples)
def test_saves_data_while_shrinking():
key = b'hi there'
n = 5
db = ExampleDatabase(':memory:')
assert list(db.fetch(key)) == []
seen = set()
def f(data):
x = data.draw_bytes(512)
if sum(x) >= 5000 and len(seen) < n:
seen.add(hbytes(x))
if hbytes(x) in seen:
data.mark_interesting()
runner = ConjectureRunner(
f, settings=settings(database=db), database_key=key)
runner.run()
assert runner.last_data.status == Status.INTERESTING
assert len(seen) == n
in_db = set(db.fetch(key))
assert in_db.issubset(seen)
assert in_db == seen
def test_fully_exhaust_base():
"""In this test we generate all possible values for the first byte but
never get to the point where we exhaust the root of the tree."""
seed_random(0)
seen = set()
def f(data):
seen.add(data.draw_bytes(2))
runner = ConjectureRunner(f,
settings=settings(
max_examples=10000,
max_iterations=10000,
max_shrinks=MAX_SHRINKS,
buffer_size=1024,
database=None,
))
runner.run()
assert len(seen) > 256
assert len({x[0] for x in seen}) == 256
def test_discards_kill_branches():
starts = set()
with deterministic_PRNG():
def test(data):
assert runner.call_count <= 256
while True:
data.start_example(1)
b = data.draw_bits(8)
data.stop_example(b != 0)
if len(data.buffer) == 1:
s = bytes(data.buffer)
assert s not in starts
starts.add(s)
if b == 0:
break
runner = ConjectureRunner(test, settings=SMALL_COUNT_SETTINGS)
runner.run()
assert runner.call_count == 256
def test_uses_tags_in_calculating_pareto_front():
with deterministic_PRNG():
def test(data):
if data.draw_bits(1):
data.start_example(11)
data.draw_bits(8)
data.stop_example()
runner = ConjectureRunner(
test,
settings=settings(
max_examples=10,
database=InMemoryExampleDatabase(),
),
database_key=b"stuff",
)
runner.run()
assert len(runner.pareto_front) == 2
def test_stops_after_max_iterations_when_generating():
key = b'key'
value = b'rubber baby buggy bumpers'
max_iterations = 100
db = ExampleDatabase(':memory:')
db.save(key, value)
seen = []
def f(data):
seen.append(data.draw_bytes(len(value)))
data.mark_invalid()
runner = ConjectureRunner(f, settings=settings(
max_examples=1, max_iterations=max_iterations,
database=db,
), database_key=key)
runner.run()
assert len(seen) == max_iterations
assert value in seen
def test_terminates_shrinks():
shrinks = [-1]
def tf(data):
x = hbytes(data.draw_bytes(100))
if sum(x) >= 500:
shrinks[0] += 1
data.mark_interesting()
runner = ConjectureRunner(tf,
settings=settings(
max_examples=5000,
max_iterations=10000,
max_shrinks=10,
database=None,
))
runner.run()
assert runner.last_data.status == Status.INTERESTING
# There's an extra non-shrinking check step to abort in the presence of
# flakiness
assert shrinks[0] == 11
def test_database_contains_only_pareto_front():
with deterministic_PRNG():
def test(data):
data.target_observations["1"] = data.draw_bits(4)
data.draw_bits(64)
data.target_observations["2"] = data.draw_bits(8)
db = InMemoryExampleDatabase()
runner = ConjectureRunner(
test,
settings=settings(
max_examples=500,
database=db,
suppress_health_check=HealthCheck.all(),
),
database_key=b"stuff",
)
runner.run()
assert len(runner.pareto_front) <= 500
for v in runner.pareto_front:
assert v.status >= Status.VALID
assert len(db.data) == 1
(values, ) = db.data.values()
values = set(values)
assert len(values) == len(runner.pareto_front)
for data in runner.pareto_front:
assert data.buffer in values
assert data in runner.pareto_front
for k in values:
assert runner.cached_test_function(k) in runner.pareto_front
def test_garbage_collects_the_database():
key = b'hi there'
n = 200
db = ExampleDatabase(':memory:')
assert list(db.fetch(key)) == []
seen = set()
go = True
counter = [0]
def f(data):
"""This function is designed to shrink very badly.
So we only occasionally mark things as interesting, and require
a certain amount of complexity to do so.
"""
x = hbytes(data.draw_bytes(512))
if not go:
return
if counter[0] % 10 == 0 and len(seen) < n and sum(x) > 1000:
seen.add(x)
counter[0] += 1
if x in seen:
data.mark_interesting()
local_settings = settings(database=db,
max_shrinks=2 * n,
timeout=unlimited)
runner = ConjectureRunner(f, settings=local_settings, database_key=key)
runner.run()
assert runner.last_data.status == Status.INTERESTING
assert len(seen) == n
assert set(db.fetch(key)) == seen
go = False
runner = ConjectureRunner(f, settings=local_settings, database_key=key)
runner.run()
assert 0 < len(set(db.fetch(key))) < n
def test_fully_exhaust_base(monkeypatch):
"""In this test we generate all possible values for the first byte but
never get to the point where we exhaust the root of the tree."""
seed_random(0)
seen = set()
def f(data):
key = (data.draw_bits(2), data.draw_bits(2))
assert key not in seen
seen.add(key)
runner = ConjectureRunner(f, settings=settings(
max_examples=10000, phases=no_shrink, buffer_size=1024, database=None,
))
for c in hrange(4):
runner.cached_test_function([0, c])
assert 1 in runner.dead
runner.run()
def test_garbage_collects_the_database():
key = b'hi there'
n = 200
db = ExampleDatabase(':memory:')
local_settings = settings(database=db, max_shrinks=n, timeout=unlimited)
runner = ConjectureRunner(slow_shrinker(),
settings=local_settings,
database_key=key)
runner.run()
assert runner.last_data.status == Status.INTERESTING
def in_db():
return set(v for vs in db.data.values() for v in vs)
assert len(in_db()) == n + 1
runner = ConjectureRunner(lambda data: None,
settings=local_settings,
database_key=key)
runner.run()
assert 0 < len(in_db()) < n
def test_garbage_collects_the_database():
key = b'hi there'
n = 200
db = InMemoryExampleDatabase()
local_settings = settings(database=db, max_shrinks=n, timeout=unlimited)
runner = ConjectureRunner(slow_shrinker(),
settings=local_settings,
database_key=key)
runner.run()
assert runner.interesting_examples
def in_db():
return set(v for vs in db.data.values() for v in vs)
assert len(in_db()) == n + 1
runner = ConjectureRunner(lambda data: data.draw_bytes(4),
settings=local_settings,
database_key=key)
runner.run()
assert 0 < len(in_db()) < n
def test_terminates_shrinks(n, monkeypatch):
db = InMemoryExampleDatabase()
def generate_new_examples(self):
self.cached_test_function([255] * 1000)
monkeypatch.setattr(ConjectureRunner, "generate_new_examples",
generate_new_examples)
monkeypatch.setattr(engine_module, "MAX_SHRINKS", n)
runner = ConjectureRunner(
slow_shrinker(),
settings=settings(max_examples=5000, database=db),
random=Random(0),
database_key=b"key",
)
runner.run()
(last_data, ) = runner.interesting_examples.values()
assert last_data.status == Status.INTERESTING
assert runner.shrinks == n
in_db = set(db.data[runner.secondary_key])
assert len(in_db) == n
def test_will_save_covering_examples():
tags = {}
def tagged(data):
b = hbytes(data.draw_bytes(4))
try:
tag = tags[b]
except KeyError:
if len(tags) < 10:
tag = len(tags)
tags[b] = tag
else:
tag = None
if tag is not None:
data.add_tag(tag)
db = InMemoryExampleDatabase()
runner = ConjectureRunner(tagged, settings=settings(
max_examples=100, phases=no_shrink, buffer_size=1024, database=db,
), database_key=b'stuff')
runner.run()
assert len(all_values(db)) == len(tags)
def test_debug_data(capsys):
buf = [0, 1, 2]
def f(data):
for x in hbytes(buf):
if data.draw_bits(8) != x:
data.mark_invalid()
data.start_example(1)
data.stop_example()
data.mark_interesting()
runner = ConjectureRunner(f, settings=settings(
max_examples=5000, buffer_size=1024,
database=None, suppress_health_check=HealthCheck.all(),
verbosity=Verbosity.debug
))
runner.test_function(ConjectureData.for_buffer(buf))
runner.run()
out, _ = capsys.readouterr()
assert re.match(u'\\d+ bytes \\[.*\\] -> ', out)
assert 'INTERESTING' in out
assert '[]' not in out
def test_terminates_shrinks(n, monkeypatch):
db = InMemoryExampleDatabase()
def generate_new_examples(self):
def draw_bytes(data, n):
return hbytes([255] * n)
self.test_function(ConjectureData(
draw_bytes=draw_bytes, max_length=self.settings.buffer_size))
monkeypatch.setattr(
ConjectureRunner, 'generate_new_examples', generate_new_examples)
runner = ConjectureRunner(slow_shrinker(), settings=settings(
max_examples=5000, max_shrinks=n,
database=db, timeout=unlimited,
), random=Random(0), database_key=b'key')
runner.run()
last_data, = runner.interesting_examples.values()
assert last_data.status == Status.INTERESTING
assert runner.shrinks == n
in_db = set(db.data[runner.secondary_key])
assert len(in_db) == n
def test_will_shrink_covering_examples():
best = [None]
replaced = []
def tagged(data):
b = hbytes(data.draw_bytes(4))
if any(b):
data.add_tag('nonzero')
if best[0] is None:
best[0] = b
elif b < best[0]:
replaced.append(best[0])
best[0] = b
db = InMemoryExampleDatabase()
runner = ConjectureRunner(tagged, settings=settings(
max_examples=100, phases=no_shrink, buffer_size=1024, database=db,
), database_key=b'stuff')
runner.run()
saved = set(all_values(db))
assert best[0] in saved
for r in replaced:
assert r not in saved
def test_saves_data_while_shrinking():
key = b'hi there'
n = 5
db = InMemoryExampleDatabase()
assert list(db.fetch(key)) == []
seen = set()
def f(data):
x = data.draw_bytes(512)
if sum(x) >= 5000 and len(seen) < n:
seen.add(hbytes(x))
if hbytes(x) in seen:
data.mark_interesting()
runner = ConjectureRunner(f,
settings=settings(database=db),
database_key=key)
runner.run()
assert runner.interesting_examples
assert len(seen) == n
in_db = non_covering_examples(db)
assert in_db.issubset(seen)
assert in_db == seen
def test_stops_after_max_examples_when_generating_more_bugs(examples):
seen = []
bad = [False, False]
def f(data):
seen.append(data.draw_bits(32))
# Rare, potentially multi-error conditions
if seen[-1] > 2**31:
bad[0] = True
raise ValueError
bad[1] = True
raise Exception
runner = ConjectureRunner(f,
settings=settings(max_examples=examples,
phases=[Phase.generate]))
try:
runner.run()
except Exception:
pass
# No matter what, whether examples is larger or smalller than MAX_TEST_CALLS,
# we stop looking at max_examples. (and re-run each failure for the traceback)
assert len(seen) <= examples + sum(bad)
def run_engine(self):
"""Run the test function many times, on database input and generated
input, using the Conjecture engine.
"""
# Tell pytest to omit the body of this function from tracebacks
__tracebackhide__ = True
try:
database_key = self.wrapped_test._hypothesis_internal_database_key
except AttributeError:
if global_force_seed is None:
database_key = function_digest(self.test)
else:
database_key = None
runner = ConjectureRunner(
self._execute_once_for_engine,
settings=self.settings,
random=self.random,
database_key=database_key,
)
# Use the Conjecture engine to run the test function many times
# on different inputs.
runner.run()
note_engine_for_statistics(runner)
if runner.call_count == 0:
return
if runner.interesting_examples:
self.falsifying_examples = sorted(
runner.interesting_examples.values(),
key=lambda d: sort_key(d.buffer),
reverse=True,
)
else:
if runner.valid_examples == 0:
raise Unsatisfiable(
"Unable to satisfy assumptions of hypothesis %s." %
(get_pretty_function_description(self.test), ))
if not self.falsifying_examples:
return
elif not self.settings.report_multiple_bugs:
# Pretend that we only found one failure, by discarding the others.
del self.falsifying_examples[:-1]
# The engine found one or more failures, so we need to reproduce and
# report them.
self.failed_normally = True
flaky = 0
for falsifying_example in self.falsifying_examples:
info = falsifying_example.extra_information
ran_example = ConjectureData.for_buffer(falsifying_example.buffer)
self.__was_flaky = False
assert info.__expected_exception is not None
try:
self.execute_once(
ran_example,
print_example=not self.is_find,
is_final=True,
expected_failure=(
info.__expected_exception,
info.__expected_traceback,
),
)
except (UnsatisfiedAssumption, StopTest):
report(traceback.format_exc())
self.__flaky(
"Unreliable assumption: An example which satisfied "
"assumptions on the first run now fails it.")
except BaseException as e:
if len(self.falsifying_examples) <= 1:
# There is only one failure, so we can report it by raising
# it directly.
raise
# We are reporting multiple failures, so we need to manually
# print each exception's stack trace and information.
tb = get_trimmed_traceback()
report("".join(traceback.format_exception(type(e), e, tb)))
finally: # pragma: no cover
# Mostly useful for ``find`` and ensuring that objects that
# hold on to a reference to ``data`` know that it's now been
# finished and they shouldn't attempt to draw more data from
# it.
ran_example.freeze()
# This section is in fact entirely covered by the tests in
# test_reproduce_failure, but it seems to trigger a lovely set
# of coverage bugs: The branches show up as uncovered (despite
# definitely being covered - you can add an assert False else
# branch to verify this and see it fail - and additionally the
# second branch still complains about lack of coverage even if
# you add a pragma: no cover to it!
# See https://bitbucket.org/ned/coveragepy/issues/623/
if self.settings.print_blob:
report(("\nYou can reproduce this example by temporarily "
"adding @reproduce_failure(%r, %r) as a decorator "
"on your test case") %
(__version__,
encode_failure(falsifying_example.buffer)))
if self.__was_flaky:
flaky += 1
# If we only have one example then we should have raised an error or
# flaky prior to this point.
assert len(self.falsifying_examples) > 1
if flaky > 0:
raise Flaky(
("Hypothesis found %d distinct failures, but %d of them "
"exhibited some sort of flaky behaviour.") %
(len(self.falsifying_examples), flaky))
else:
raise MultipleFailures(("Hypothesis found %d distinct failures.") %
(len(self.falsifying_examples)))
def run(self):
# Tell pytest to omit the body of this function from tracebacks
__tracebackhide__ = True
database_key = str_to_bytes(fully_qualified_name(self.test))
self.start_time = time.time()
runner = ConjectureRunner(
self.evaluate_test_data,
settings=self.settings,
random=self.random,
database_key=database_key,
)
runner.run()
note_engine_for_statistics(runner)
run_time = time.time() - self.start_time
timed_out = runner.exit_reason == ExitReason.timeout
if runner.last_data is None:
return
if runner.interesting_examples:
self.falsifying_examples = sorted(
[d for d in runner.interesting_examples.values()],
key=lambda d: sort_key(d.buffer),
reverse=True)
else:
if timed_out:
note_deprecation((
'Your tests are hitting the settings timeout (%.2fs). '
'This functionality will go away in a future release '
'and you should not rely on it. Instead, try setting '
'max_examples to be some value lower than %d (the number '
'of examples your test successfully ran here). Or, if you '
'would prefer your tests to run to completion, regardless '
'of how long they take, you can set the timeout value to '
'hypothesis.unlimited.') % (self.settings.timeout,
runner.valid_examples),
self.settings)
if runner.valid_examples < min(
self.settings.min_satisfying_examples,
self.settings.max_examples,
) and not (runner.exit_reason == ExitReason.finished
and self.at_least_one_success):
if timed_out:
raise Timeout(
('Ran out of time before finding a satisfying '
'example for '
'%s. Only found %d examples in ' + '%.2fs.') %
(get_pretty_function_description(
self.test), runner.valid_examples, run_time))
else:
raise Unsatisfiable(
('Unable to satisfy assumptions of hypothesis '
'%s. Only %d examples considered '
'satisfied assumptions') % (
get_pretty_function_description(self.test),
runner.valid_examples,
))
if not self.falsifying_examples:
return
flaky = 0
for falsifying_example in self.falsifying_examples:
self.__was_flaky = False
raised_exception = False
try:
with self.settings:
self.test_runner(
ConjectureData.for_buffer(falsifying_example.buffer),
reify_and_execute(self.search_strategy,
self.test,
print_example=True,
is_final=True))
except (UnsatisfiedAssumption, StopTest):
report(traceback.format_exc())
self.__flaky(
'Unreliable assumption: An example which satisfied '
'assumptions on the first run now fails it.')
except:
if len(self.falsifying_examples) <= 1:
raise
raised_exception = True
report(traceback.format_exc())
if not raised_exception:
report(
'Failed to reproduce exception. Expected: \n' +
falsifying_example.__expected_exception, )
filter_message = (
'Unreliable test data: Failed to reproduce a failure '
'and then when it came to recreating the example in '
'order to print the test data with a flaky result '
'the example was filtered out (by e.g. a '
'call to filter in your strategy) when we didn\'t '
'expect it to be.')
try:
self.test_runner(
ConjectureData.for_buffer(falsifying_example.buffer),
reify_and_execute(self.search_strategy,
test_is_flaky(
self.test,
self.repr_for_last_exception),
print_example=True,
is_final=True))
except (UnsatisfiedAssumption, StopTest):
self.__flaky(filter_message)
except Flaky as e:
if len(self.falsifying_examples) > 1:
self.__flaky(e.args[0])
else:
raise
if self.__was_flaky:
flaky += 1
# If we only have one example then we should have raised an error or
# flaky prior to this point.
assert len(self.falsifying_examples) > 1
if flaky > 0:
raise Flaky(
('Hypothesis found %d distinct failures, but %d of them '
'exhibited some sort of flaky behaviour.') %
(len(self.falsifying_examples), flaky))
else:
raise MultipleFailures(('Hypothesis found %d distinct failures.') %
(len(self.falsifying_examples, )))
def run(self):
database_key = str_to_bytes(fully_qualified_name(self.test))
start_time = time.time()
runner = ConjectureRunner(
self.evaluate_test_data,
settings=self.settings,
random=self.random,
database_key=database_key,
)
runner.run()
note_engine_for_statistics(runner)
run_time = time.time() - start_time
timed_out = (self.settings.timeout > 0
and run_time >= self.settings.timeout)
if runner.last_data is None:
return
if runner.last_data.status == Status.INTERESTING:
self.falsifying_example = runner.last_data.buffer
if self.settings.database is not None:
self.settings.database.save(database_key,
self.falsifying_example)
else:
if runner.valid_examples < min(
self.settings.min_satisfying_examples,
self.settings.max_examples,
) and not (runner.exit_reason == ExitReason.finished
and self.at_least_one_success):
if timed_out:
raise Timeout(
('Ran out of time before finding a satisfying '
'example for '
'%s. Only found %d examples in ' + '%.2fs.') %
(get_pretty_function_description(
self.test), runner.valid_examples, run_time))
else:
raise Unsatisfiable(
('Unable to satisfy assumptions of hypothesis '
'%s. Only %d examples considered '
'satisfied assumptions') % (
get_pretty_function_description(self.test),
runner.valid_examples,
))
if self.falsifying_example is None:
return
assert self.last_exception is not None
try:
with self.settings:
self.test_runner(
ConjectureData.for_buffer(self.falsifying_example),
reify_and_execute(self.search_strategy,
self.test,
print_example=True,
is_final=True))
except (UnsatisfiedAssumption, StopTest):
report(traceback.format_exc())
raise Flaky('Unreliable assumption: An example which satisfied '
'assumptions on the first run now fails it.')
report(
'Failed to reproduce exception. Expected: \n' +
self.last_exception, )
filter_message = (
'Unreliable test data: Failed to reproduce a failure '
'and then when it came to recreating the example in '
'order to print the test data with a flaky result '
'the example was filtered out (by e.g. a '
'call to filter in your strategy) when we didn\'t '
'expect it to be.')
try:
self.test_runner(
ConjectureData.for_buffer(self.falsifying_example),
reify_and_execute(self.search_strategy,
test_is_flaky(self.test,
self.repr_for_last_exception),
print_example=True,
is_final=True))
except (UnsatisfiedAssumption, StopTest):
raise Flaky(filter_message)
def find(specifier, condition, settings=None, random=None, database_key=None):
"""Returns the minimal example from the given strategy ``specifier`` that
matches the predicate function ``condition``."""
settings = settings or Settings(
max_examples=2000,
min_satisfying_examples=0,
max_shrinks=2000,
)
settings = Settings(settings, perform_health_check=False)
if database_key is None and settings.database is not None:
database_key = function_digest(condition)
if not isinstance(specifier, SearchStrategy):
raise InvalidArgument(
'Expected SearchStrategy but got %r of type %s' % (
specifier, type(specifier).__name__
))
specifier.validate()
search = specifier
random = random or new_random()
successful_examples = [0]
last_data = [None]
def template_condition(data):
with BuildContext(data):
try:
data.is_find = True
result = data.draw(search)
data.note(result)
success = condition(result)
except UnsatisfiedAssumption:
data.mark_invalid()
if success:
successful_examples[0] += 1
if settings.verbosity == Verbosity.verbose:
if not successful_examples[0]:
report(lambda: u'Trying example %s' % (
nicerepr(result),
))
elif success:
if successful_examples[0] == 1:
report(lambda: u'Found satisfying example %s' % (
nicerepr(result),
))
last_data[0] = data
elif (
sort_key(hbytes(data.buffer)) <
sort_key(last_data[0].buffer)
):
report(lambda: u'Shrunk example to %s' % (
nicerepr(result),
))
last_data[0] = data
if success and not data.frozen:
data.mark_interesting()
start = time.time()
runner = ConjectureRunner(
template_condition, settings=settings, random=random,
database_key=database_key,
)
runner.run()
note_engine_for_statistics(runner)
run_time = time.time() - start
if runner.interesting_examples:
data = ConjectureData.for_buffer(
list(runner.interesting_examples.values())[0].buffer)
with BuildContext(data):
return data.draw(search)
if (
runner.valid_examples <= settings.min_satisfying_examples and
runner.exit_reason != ExitReason.finished
):
if settings.timeout > 0 and run_time > settings.timeout:
raise Timeout((
'Ran out of time before finding enough valid examples for '
'%s. Only %d valid examples found in %.2f seconds.'
) % (
get_pretty_function_description(condition),
runner.valid_examples, run_time))
else:
raise Unsatisfiable((
'Unable to satisfy assumptions of '
'%s. Only %d examples considered satisfied assumptions'
) % (
get_pretty_function_description(condition),
runner.valid_examples,))
raise NoSuchExample(get_pretty_function_description(condition))
def run(self):
# Tell pytest to omit the body of this function from tracebacks
__tracebackhide__ = True
if global_force_seed is None:
database_key = str_to_bytes(fully_qualified_name(self.test))
else:
database_key = None
self.start_time = time.time()
global in_given
runner = ConjectureRunner(
self.evaluate_test_data,
settings=self.settings,
random=self.random,
database_key=database_key,
)
if in_given or self.collector is None:
runner.run()
else: # pragma: no cover
in_given = True
original_trace = sys.gettrace()
try:
sys.settrace(None)
runner.run()
finally:
in_given = False
sys.settrace(original_trace)
self.used_examples_from_database = \
runner.used_examples_from_database
note_engine_for_statistics(runner)
run_time = time.time() - self.start_time
self.used_examples_from_database = runner.used_examples_from_database
if runner.used_examples_from_database:
if self.settings.derandomize:
note_deprecation(
'In future derandomize will imply database=None, but your '
'test is currently using examples from the database. To '
'get the future behaviour, update your settings to '
'include database=None.')
if self.__had_seed:
note_deprecation(
'In future use of @seed will imply database=None in your '
'settings, but your test is currently using examples from '
'the database. To get the future behaviour, update your '
'settings for this test to include database=None.')
timed_out = runner.exit_reason == ExitReason.timeout
if runner.call_count == 0:
return
if runner.interesting_examples:
self.falsifying_examples = sorted(
[d for d in runner.interesting_examples.values()],
key=lambda d: sort_key(d.buffer),
reverse=True)
else:
if timed_out:
note_deprecation((
'Your tests are hitting the settings timeout (%.2fs). '
'This functionality will go away in a future release '
'and you should not rely on it. Instead, try setting '
'max_examples to be some value lower than %d (the number '
'of examples your test successfully ran here). Or, if you '
'would prefer your tests to run to completion, regardless '
'of how long they take, you can set the timeout value to '
'hypothesis.unlimited.') % (self.settings.timeout,
runner.valid_examples),
self.settings)
if runner.valid_examples == 0:
if timed_out:
raise Timeout(
('Ran out of time before finding a satisfying '
'example for %s. Only found %d examples in %.2fs.') %
(get_pretty_function_description(
self.test), runner.valid_examples, run_time))
else:
raise Unsatisfiable(
'Unable to satisfy assumptions of hypothesis %s.' %
(get_pretty_function_description(self.test), ))
if not self.falsifying_examples:
return
self.failed_normally = True
flaky = 0
for falsifying_example in self.falsifying_examples:
ran_example = ConjectureData.for_buffer(falsifying_example.buffer)
self.__was_flaky = False
assert falsifying_example.__expected_exception is not None
try:
self.execute(ran_example,
print_example=True,
is_final=True,
expected_failure=(
falsifying_example.__expected_exception,
falsifying_example.__expected_traceback,
))
except (UnsatisfiedAssumption, StopTest):
report(traceback.format_exc())
self.__flaky(
'Unreliable assumption: An example which satisfied '
'assumptions on the first run now fails it.')
except BaseException:
if len(self.falsifying_examples) <= 1:
raise
report(traceback.format_exc())
finally: # pragma: no cover
# This section is in fact entirely covered by the tests in
# test_reproduce_failure, but it seems to trigger a lovely set
# of coverage bugs: The branches show up as uncovered (despite
# definitely being covered - you can add an assert False else
# branch to verify this and see it fail - and additionally the
# second branch still complains about lack of coverage even if
# you add a pragma: no cover to it!
# See https://bitbucket.org/ned/coveragepy/issues/623/
if self.settings.print_blob is not PrintSettings.NEVER:
failure_blob = encode_failure(falsifying_example.buffer)
# Have to use the example we actually ran, not the original
# falsifying example! Otherwise we won't catch problems
# where the repr of the generated example doesn't parse.
can_use_repr = ran_example.can_reproduce_example_from_repr
if (self.settings.print_blob is PrintSettings.ALWAYS or
(self.settings.print_blob is PrintSettings.INFER
and not can_use_repr and len(failure_blob) < 200)):
report((
'\n'
'You can reproduce this example by temporarily '
'adding @reproduce_failure(%r, %r) as a decorator '
'on your test case') % (
__version__,
failure_blob,
))
if self.__was_flaky:
flaky += 1
# If we only have one example then we should have raised an error or
# flaky prior to this point.
assert len(self.falsifying_examples) > 1
if flaky > 0:
raise Flaky(
('Hypothesis found %d distinct failures, but %d of them '
'exhibited some sort of flaky behaviour.') %
(len(self.falsifying_examples), flaky))
else:
raise MultipleFailures(('Hypothesis found %d distinct failures.') %
(len(self.falsifying_examples, )))
def run(self):
# Tell pytest to omit the body of this function from tracebacks
__tracebackhide__ = True
if global_force_seed is None:
database_key = function_digest(self.test)
else:
database_key = None
runner = ConjectureRunner(
self.evaluate_test_data,
settings=self.settings,
random=self.random,
database_key=database_key,
)
try:
runner.run()
finally:
self.used_examples_from_database = runner.used_examples_from_database
note_engine_for_statistics(runner)
self.used_examples_from_database = runner.used_examples_from_database
if runner.call_count == 0:
return
if runner.interesting_examples:
self.falsifying_examples = sorted(
[d for d in runner.interesting_examples.values()],
key=lambda d: sort_key(d.buffer),
reverse=True,
)
else:
if runner.valid_examples == 0:
raise Unsatisfiable(
"Unable to satisfy assumptions of hypothesis %s." %
(get_pretty_function_description(self.test), ))
if not self.falsifying_examples:
return
self.failed_normally = True
flaky = 0
for falsifying_example in self.falsifying_examples:
ran_example = ConjectureData.for_buffer(falsifying_example.buffer)
self.__was_flaky = False
assert falsifying_example.__expected_exception is not None
try:
self.execute(
ran_example,
print_example=True,
is_final=True,
expected_failure=(
falsifying_example.__expected_exception,
falsifying_example.__expected_traceback,
),
)
except (UnsatisfiedAssumption, StopTest):
report(traceback.format_exc())
self.__flaky(
"Unreliable assumption: An example which satisfied "
"assumptions on the first run now fails it.")
except BaseException as e:
if len(self.falsifying_examples) <= 1:
raise
tb = get_trimmed_traceback()
report("".join(traceback.format_exception(type(e), e, tb)))
finally: # pragma: no cover
# This section is in fact entirely covered by the tests in
# test_reproduce_failure, but it seems to trigger a lovely set
# of coverage bugs: The branches show up as uncovered (despite
# definitely being covered - you can add an assert False else
# branch to verify this and see it fail - and additionally the
# second branch still complains about lack of coverage even if
# you add a pragma: no cover to it!
# See https://bitbucket.org/ned/coveragepy/issues/623/
if self.settings.print_blob is not PrintSettings.NEVER:
failure_blob = encode_failure(falsifying_example.buffer)
# Have to use the example we actually ran, not the original
# falsifying example! Otherwise we won't catch problems
# where the repr of the generated example doesn't parse.
can_use_repr = ran_example.can_reproduce_example_from_repr
if self.settings.print_blob is PrintSettings.ALWAYS or (
self.settings.print_blob is PrintSettings.INFER
and self.settings.verbosity >= Verbosity.normal
and not can_use_repr and len(failure_blob) < 200):
report((
"\nYou can reproduce this example by temporarily "
"adding @reproduce_failure(%r, %r) as a decorator "
"on your test case") % (__version__, failure_blob))
if self.__was_flaky:
flaky += 1
# If we only have one example then we should have raised an error or
# flaky prior to this point.
assert len(self.falsifying_examples) > 1
if flaky > 0:
raise Flaky(
("Hypothesis found %d distinct failures, but %d of them "
"exhibited some sort of flaky behaviour.") %
(len(self.falsifying_examples), flaky))
else:
raise MultipleFailures(("Hypothesis found %d distinct failures.") %
(len(self.falsifying_examples)))
def learner_for(strategy):
"""Returns an LStar learner that predicts whether a buffer
corresponds to a discard free choice sequence leading to
a valid value for this strategy."""
try:
return LEARNERS[strategy]
except KeyError:
pass
def test_function(data):
data.draw(strategy)
data.mark_interesting()
runner = ConjectureRunner(
test_function,
settings=settings(
database=None,
verbosity=Verbosity.quiet,
suppress_health_check=HealthCheck.all(),
),
random=Random(0),
ignore_limits=True,
)
def predicate(s):
result = runner.cached_test_function(s)
if result.status < Status.VALID:
return False
if result.has_discards:
return False
return result.buffer == s
learner = LStar(predicate)
runner.run()
(v,) = runner.interesting_examples.values()
# We make sure the learner has properly learned small examples.
# This is all fairly ad hoc but is mostly designed to get it
# to understand what the smallest example is and avoid any
# loops at the beginning of the DFA that don't really exist.
learner.learn(v.buffer)
for n in [1, 2, 3]:
for _ in range(5):
learner.learn(uniform(runner.random, n) + v.buffer)
prev = -1
while learner.generation != prev:
prev = learner.generation
for _ in range(10):
s = uniform(runner.random, len(v.buffer)) + bytes(BUFFER_SIZE)
learner.learn(s)
data = runner.cached_test_function(s)
if data.status >= Status.VALID:
learner.learn(data.buffer)
LEARNERS[strategy] = learner
return learner
def find(
specifier, # type: SearchStrategy
condition, # type: Callable[[Any], bool]
settings=None, # type: Settings
random=None, # type: Any
database_key=None, # type: bytes
):
# type: (...) -> Any
"""Returns the minimal example from the given strategy ``specifier`` that
matches the predicate function ``condition``."""
if settings is None:
settings = Settings(max_examples=2000)
settings = Settings(settings, suppress_health_check=HealthCheck.all())
if database_key is None and settings.database is not None:
database_key = function_digest(condition)
if not isinstance(specifier, SearchStrategy):
raise InvalidArgument("Expected SearchStrategy but got %r of type %s" %
(specifier, type(specifier).__name__))
specifier.validate()
search = specifier
random = random or new_random()
successful_examples = [0]
last_data = [None]
last_repr = [None]
def template_condition(data):
with BuildContext(data):
try:
data.is_find = True
with deterministic_PRNG():
result = data.draw(search)
data.note(result)
success = condition(result)
except UnsatisfiedAssumption:
data.mark_invalid()
if success:
successful_examples[0] += 1
if settings.verbosity >= Verbosity.verbose:
if not successful_examples[0]:
report(u"Tried non-satisfying example %s" %
(nicerepr(result), ))
elif success:
if successful_examples[0] == 1:
last_repr[0] = nicerepr(result)
report(u"Found satisfying example %s" % (last_repr[0], ))
last_data[0] = data
elif (sort_key(hbytes(data.buffer)) < sort_key(
last_data[0].buffer)
) and nicerepr(result) != last_repr[0]:
last_repr[0] = nicerepr(result)
report(u"Shrunk example to %s" % (last_repr[0], ))
last_data[0] = data
if success and not data.frozen:
data.mark_interesting()
runner = ConjectureRunner(template_condition,
settings=settings,
random=random,
database_key=database_key)
runner.run()
note_engine_for_statistics(runner)
if runner.interesting_examples:
data = ConjectureData.for_buffer(
list(runner.interesting_examples.values())[0].buffer)
with BuildContext(data):
with deterministic_PRNG():
return data.draw(search)
if runner.valid_examples == 0 and (runner.exit_reason !=
ExitReason.finished):
raise Unsatisfiable("Unable to satisfy assumptions of %s." %
(get_pretty_function_description(condition), ))
raise NoSuchExample(get_pretty_function_description(condition))
