def test_can_reduce_poison_from_any_subtree(size, seed):
"""This test validates that we can minimize to any leaf node of a binary
tree, regardless of where in the tree the leaf is."""
random = Random(seed)
# Initially we create the minimal tree of size n, regardless of whether it
# is poisoned (which it won't be - the poison event essentially never
# happens when drawing uniformly at random).
# Choose p so that the expected size of the tree is equal to the desired
# size.
p = 1.0 / (2.0 - 1.0 / size)
strat = PoisonedTree(p)
def test_function(data):
v = data.draw(strat)
if len(v) >= size:
data.mark_interesting()
runner = ConjectureRunner(test_function, random=random, settings=TEST_SETTINGS)
runner.generate_new_examples()
runner.shrink_interesting_examples()
(data,) = runner.interesting_examples.values()
assert len(ConjectureData.for_buffer(data.buffer).draw(strat)) == size
starts = [b.start for b in data.blocks if b.length == 2]
assert len(starts) % 2 == 0
for i in range(0, len(starts), 2):
# Now for each leaf position in the tree we try inserting a poison
# value artificially. Additionally, we add a marker to the end that
# must be preserved. The marker means that we are not allow to rely on
# discarding the end of the buffer to get the desired shrink.
u = starts[i]
marker = bytes([1, 2, 3, 4])
def test_function_with_poison(data):
v = data.draw(strat)
m = data.draw_bytes(len(marker))
if POISON in v and m == marker:
data.mark_interesting()
runner = ConjectureRunner(
test_function_with_poison, random=random, settings=TEST_SETTINGS
)
runner.cached_test_function(
data.buffer[:u] + bytes([255]) * 4 + data.buffer[u + 4 :] + marker
)
assert runner.interesting_examples
runner.shrink_interesting_examples()
(shrunk,) = runner.interesting_examples.values()
assert ConjectureData.for_buffer(shrunk.buffer).draw(strat) == (POISON,)
def test_shrink_after_max_iterations():
"""If we find a bug, keep looking for more, and then hit the test call
limit, we should still proceed to shrinking.
"""
max_examples = 10
max_iterations = max_examples * 10
fail_at = max_iterations - 5
invalid = set()
bad = set()
post_failure_calls = [0]
def test(data):
if bad:
post_failure_calls[0] += 1
value = data.draw_bits(16)
if value in invalid:
data.mark_invalid()
if value in bad or (not bad and len(invalid) == fail_at):
bad.add(value)
data.mark_interesting()
invalid.add(value)
data.mark_invalid()
# This shouldn't need to be deterministic, but it makes things much easier
# to debug if anything goes wrong.
with deterministic_PRNG():
runner = ConjectureRunner(
test,
settings=settings(
TEST_SETTINGS,
max_examples=max_examples,
phases=[Phase.generate, Phase.shrink],
report_multiple_bugs=True,
),
)
runner.shrink_interesting_examples = Mock(
name="shrink_interesting_examples")
runner.run()
# First, verify our test assumptions: we found a bug, kept running, and
# then hit the test call limit.
assert runner.interesting_examples
assert post_failure_calls[0] >= (max_iterations - fail_at) - 1
assert runner.call_count >= max_iterations
assert runner.valid_examples == 0
# Now check that we still performed shrinking, even after hitting the
# test call limit.
assert runner.shrink_interesting_examples.call_count == 1
assert runner.exit_reason == ExitReason.finished
def test_does_not_shrink_multiple_bugs_when_told_not_to():
def test(data):
m = data.draw_bits(8)
n = data.draw_bits(8)
if m > 0:
data.mark_interesting(1)
if n > 5:
data.mark_interesting(2)
with deterministic_PRNG():
runner = ConjectureRunner(
test, settings=settings(TEST_SETTINGS, report_multiple_bugs=False)
)
runner.cached_test_function([255, 255])
runner.shrink_interesting_examples()
results = {d.buffer for d in runner.interesting_examples.values()}
assert len(results.intersection({bytes([0, 1]), bytes([1, 0])})) == 1
def test_will_not_reset_the_tree_after_interesting_example(monkeypatch):
monkeypatch.setattr(engine_module, 'CACHE_RESET_FREQUENCY', 3)
def f(data):
if data.draw_bits(8) == 7:
data.mark_interesting()
with deterministic_PRNG():
runner = ConjectureRunner(f, settings=settings(
database=None, suppress_health_check=HealthCheck.all(),
))
def step(n):
runner.test_function(ConjectureData.for_buffer([n]))
step(0)
step(1)
assert len(runner.tree) > 1
step(7)
assert len(runner.tree) > 1
t = len(runner.tree)
runner.shrink_interesting_examples()
assert len(runner.tree) > t
def test_can_reduce_poison_from_any_subtree(size, seed):
"""This test validates that we can minimize to any leaf node of a binary
tree, regardless of where in the tree the leaf is."""
random = Random(seed)
# Initially we create the minimal tree of size n, regardless of whether it
# is poisoned (which it won't be - the poison event essentially never
# happens when drawing uniformly at random).
# Choose p so that the expected size of the tree is equal to the desired
# size.
p = 1.0 / (2.0 - 1.0 / size)
strat = PoisonedTree(p)
def test_function(data):
v = data.draw(strat)
if len(v) >= size:
data.mark_interesting()
runner = ConjectureRunner(
test_function, random=random, settings=settings(TEST_SETTINGS, buffer_size=LOTS)
)
while not runner.interesting_examples:
runner.test_function(
runner.new_conjecture_data(lambda data, n: uniform(random, n))
)
runner.shrink_interesting_examples()
data, = runner.interesting_examples.values()
assert len(ConjectureData.for_buffer(data.buffer).draw(strat)) == size
starts = [b.start for b in data.blocks if b.length == 2]
assert len(starts) % 2 == 0
for i in hrange(0, len(starts), 2):
# Now for each leaf position in the tree we try inserting a poison
# value artificially. Additionally, we add a marker to the end that
# must be preserved. The marker means that we are not allow to rely on
# discarding the end of the buffer to get the desired shrink.
u = starts[i]
marker = hbytes([1, 2, 3, 4])
def test_function_with_poison(data):
v = data.draw(strat)
m = data.draw_bytes(len(marker))
if POISON in v and m == marker:
data.mark_interesting()
runner = ConjectureRunner(
test_function_with_poison, random=random, settings=TEST_SETTINGS
)
runner.cached_test_function(
data.buffer[:u] + hbytes([255]) * 4 + data.buffer[u + 4 :] + marker
)
assert runner.interesting_examples
runner.shrink_interesting_examples()
shrunk, = runner.interesting_examples.values()
assert ConjectureData.for_buffer(shrunk.buffer).draw(strat) == (POISON,)
