def test_always_reduces_integers_to_smallest_suitable_sizes(problem):
n, blob = problem
blob = hbytes(blob)
try:
d = ConjectureData.for_buffer(blob)
k = d.draw(st.integers())
stop = blob[len(d.buffer)]
except (StopTest, IndexError):
reject()
assume(k > n)
assume(stop > 0)
def f(data):
k = data.draw(st.integers())
data.output = repr(k)
if data.draw_bits(8) == stop and k >= n:
data.mark_interesting()
runner = ConjectureRunner(f,
random=Random(0),
settings=settings(
suppress_health_check=HealthCheck.all(),
timeout=unlimited,
phases=(Phase.shrink, ),
database=None,
verbosity=Verbosity.quiet))
runner.test_function(ConjectureData.for_buffer(blob))
assert runner.interesting_examples
v, = runner.interesting_examples.values()
shrinker = runner.new_shrinker(v, lambda x: x.status == Status.INTERESTING)
shrinker.single_greedy_shrink_iteration = \
shrinker.minimize_individual_blocks
shrinker.shrink()
v = shrinker.shrink_target
m = ConjectureData.for_buffer(v.buffer).draw(st.integers())
assert m == n
# Upper bound on the length needed is calculated as follows:
# * We have an initial byte at the beginning to decide the length of the
# integer.
# * We have a terminal byte as the stop value.
# * The rest is the integer payload. This should be n. Including the sign
# bit, n needs (1 + n.bit_length()) / 8 bytes (rounded up). But we only
# have power of two sizes, so it may be up to a factor of two more than
# that.
bits_needed = 1 + n.bit_length()
actual_bits_needed = min(
[s for s in WideRangeIntStrategy.sizes if s >= bits_needed])
bytes_needed = actual_bits_needed // 8
# 3 extra bytes: two for the sampler, one for the capping value.
assert len(v.buffer) == 3 + bytes_needed
def test_exhaustive_enumeration(prefix, bits, seed):
seen = set()
def f(data):
if prefix:
data.write(hbytes(prefix))
assert len(data.buffer) == len(prefix)
k = data.draw_bits(bits)
assert k not in seen
seen.add(k)
size = 2 ** bits
seen_prefixes = set()
runner = ConjectureRunner(
f, settings=settings(database=None, max_examples=size),
random=Random(seed),
)
with pytest.raises(RunIsComplete):
runner.cached_test_function(b'')
for _ in hrange(size):
p = runner.generate_novel_prefix()
assert p not in seen_prefixes
seen_prefixes.add(p)
data = ConjectureData.for_buffer(
hbytes(p + hbytes(2 + len(prefix))))
runner.test_function(data)
assert data.status == Status.VALID
node = 0
for b in data.buffer:
node = runner.tree[node][b]
assert node in runner.dead
assert len(seen) == size
def test_database_uses_values_from_secondary_key():
key = b'key'
database = InMemoryExampleDatabase()
def f(data):
if data.draw_bits(8) >= 5:
data.mark_interesting()
else:
data.mark_invalid()
runner = ConjectureRunner(f, settings=settings(
max_examples=1, buffer_size=1024,
database=database, suppress_health_check=HealthCheck.all(),
), database_key=key)
for i in range(10):
database.save(runner.secondary_key, hbytes([i]))
runner.test_function(ConjectureData.for_buffer(hbytes([10])))
assert runner.interesting_examples
assert len(set(database.fetch(key))) == 1
assert len(set(database.fetch(runner.secondary_key))) == 10
runner.clear_secondary_key()
assert len(set(database.fetch(key))) == 1
assert set(
map(int_from_bytes, database.fetch(runner.secondary_key))
) == set(range(6, 11))
v, = runner.interesting_examples.values()
assert list(v.buffer) == [5]
def test_debug_data(capsys):
buf = [0, 1, 2]
def f(data):
for x in hbytes(buf):
if data.draw_bits(8) != x:
data.mark_invalid()
data.start_example(1)
data.stop_example()
data.mark_interesting()
runner = ConjectureRunner(
f,
settings=settings(
max_examples=5000,
buffer_size=1024,
database=None,
suppress_health_check=HealthCheck.all(),
verbosity=Verbosity.debug,
),
)
runner.test_function(ConjectureData.for_buffer(buf))
runner.run()
out, _ = capsys.readouterr()
assert re.match(u"\\d+ bytes \\[.*\\] -> ", out)
assert "INTERESTING" in out
assert "[]" not in out
def test_database_clears_secondary_key():
key = b"key"
database = InMemoryExampleDatabase()
def f(data):
if data.draw_bits(8) == 10:
data.mark_interesting()
else:
data.mark_invalid()
runner = ConjectureRunner(
f,
settings=settings(
max_examples=1,
buffer_size=1024,
database=database,
suppress_health_check=HealthCheck.all(),
),
database_key=key,
)
for i in range(10):
database.save(runner.secondary_key, hbytes([i]))
runner.test_function(ConjectureData.for_buffer(hbytes([10])))
assert runner.interesting_examples
assert len(set(database.fetch(key))) == 1
assert len(set(database.fetch(runner.secondary_key))) == 10
runner.clear_secondary_key()
assert len(set(database.fetch(key))) == 1
assert len(set(database.fetch(runner.secondary_key))) == 0
def test_exhaustive_enumeration_of_partial_buffer():
seen = set()
def f(data):
k = data.draw_bytes(2)
assert k[1] == 0
assert k not in seen
seen.add(k)
seen_prefixes = set()
runner = ConjectureRunner(
f, settings=settings(database=None, max_examples=256, buffer_size=2),
random=Random(0),
)
with pytest.raises(RunIsComplete):
runner.cached_test_function(b'')
for _ in hrange(256):
p = runner.generate_novel_prefix()
assert p not in seen_prefixes
seen_prefixes.add(p)
data = ConjectureData.for_buffer(hbytes(p + hbytes(2)))
runner.test_function(data)
assert data.status == Status.VALID
node = 0
for b in data.buffer:
node = runner.tree[node][b]
assert node in runner.dead
assert len(seen) == 256
def test_exhaustive_enumeration_of_partial_buffer():
seen = set()
def f(data):
k = data.draw_bytes(2)
assert k[1] == 0
assert k not in seen
seen.add(k)
seen_prefixes = set()
runner = ConjectureRunner(
f,
settings=settings(database=None, max_examples=256, buffer_size=2),
random=Random(0),
)
with pytest.raises(RunIsComplete):
runner.cached_test_function(b'')
for _ in hrange(256):
p = runner.generate_novel_prefix()
assert p not in seen_prefixes
seen_prefixes.add(p)
data = ConjectureData.for_buffer(hbytes(p + hbytes(2)))
runner.test_function(data)
assert data.status == Status.VALID
node = 0
for b in data.buffer:
node = runner.tree[node][b]
assert node in runner.dead
assert len(seen) == 256
def test_detects_too_small_block_starts():
def f(data):
data.draw_bytes(8)
data.mark_interesting()
runner = ConjectureRunner(f, settings=settings(database=None))
r = ConjectureData.for_buffer(hbytes(8))
runner.test_function(r)
assert r.status == Status.INTERESTING
assert not runner.prescreen_buffer(hbytes([255] * 7))
def test_always_reduces_integers_to_smallest_suitable_sizes(problem):
n, blob = problem
blob = hbytes(blob)
try:
d = ConjectureData.for_buffer(blob)
k = d.draw(st.integers())
stop = blob[len(d.buffer)]
except (StopTest, IndexError):
reject()
assume(k > n)
assume(stop > 0)
def f(data):
k = data.draw(st.integers())
data.output = repr(k)
if data.draw_bits(8) == stop and k >= n:
data.mark_interesting()
runner = ConjectureRunner(f, random=Random(0), settings=settings(
suppress_health_check=HealthCheck.all(), timeout=unlimited,
phases=(Phase.shrink,), database=None, verbosity=Verbosity.debug
), database_key=None)
runner.test_function(ConjectureData.for_buffer(blob))
assert runner.interesting_examples
v, = runner.interesting_examples.values()
shrinker = runner.new_shrinker(v, lambda x: x.status == Status.INTERESTING)
shrinker.clear_passes()
shrinker.add_new_pass('minimize_individual_blocks')
shrinker.shrink()
v = shrinker.shrink_target
m = ConjectureData.for_buffer(v.buffer).draw(st.integers())
assert m == n
# Upper bound on the length needed is calculated as follows:
# * We have an initial byte at the beginning to decide the length of the
# integer.
# * We have a terminal byte as the stop value.
# * The rest is the integer payload. This should be n. Including the sign
# bit, n needs (1 + n.bit_length()) / 8 bytes (rounded up). But we only
# have power of two sizes, so it may be up to a factor of two more than
# that.
bits_needed = 1 + n.bit_length()
actual_bits_needed = min(
[s for s in WideRangeIntStrategy.sizes if s >= bits_needed])
bytes_needed = actual_bits_needed // 8
# 3 extra bytes: two for the sampler, one for the capping value.
assert len(v.buffer) == 3 + bytes_needed
def accept(f):
with deterministic_PRNG():
runner = ConjectureRunner(f, settings=settings(
max_examples=5000, buffer_size=1024,
database=None, suppress_health_check=HealthCheck.all(),
))
runner.test_function(ConjectureData.for_buffer(start))
assert runner.interesting_examples
last_data, = runner.interesting_examples.values()
return runner.new_shrinker(
last_data, lambda d: d.status == Status.INTERESTING
)
def test_avoids_zig_zag_trap(p):
b, marker, lower_bound = p
b = hbytes(b)
marker = hbytes(marker)
n_bits = 8 * (len(b) + 1)
def test_function(data):
m = data.draw_bits(n_bits)
if m < lower_bound:
data.mark_invalid()
n = data.draw_bits(n_bits)
if data.draw_bytes(len(marker)) != marker:
data.mark_invalid()
if abs(m - n) == 1:
data.mark_interesting()
runner = ConjectureRunner(test_function,
database_key=None,
settings=settings(database=None,
max_shrinks=100,
verbosity=Verbosity.debug))
runner.debug = note
original_debug_data = runner.debug_data
def debug_interesting(data):
if data.status == Status.INTERESTING:
original_debug_data(data)
runner.debug_data = debug_interesting
runner.test_function(
ConjectureData.for_buffer(b + hbytes([0]) + b + hbytes([1]) + marker))
assert runner.interesting_examples
runner.run()
v, = runner.interesting_examples.values()
data = ConjectureData.for_buffer(v.buffer)
m = data.draw_bits(n_bits)
n = data.draw_bits(n_bits)
assert m == lower_bound
if m == 0:
assert n == 1
else:
assert n == m - 1
def test_avoids_zig_zag_trap(p):
b, marker, lower_bound = p
random.seed(0)
b = hbytes(b)
marker = hbytes(marker)
n_bits = 8 * (len(b) + 1)
def test_function(data):
m = data.draw_bits(n_bits)
if m < lower_bound:
data.mark_invalid()
n = data.draw_bits(n_bits)
if data.draw_bytes(len(marker)) != marker:
data.mark_invalid()
if abs(m - n) == 1:
data.mark_interesting()
runner = ConjectureRunner(
test_function, database_key=None, settings=settings(
base_settings,
phases=(Phase.generate, Phase.shrink)
)
)
runner.test_function(ConjectureData.for_buffer(
b + hbytes([0]) + b + hbytes([1]) + marker))
assert runner.interesting_examples
runner.run()
v, = runner.interesting_examples.values()
data = ConjectureData.for_buffer(v.buffer)
m = data.draw_bits(n_bits)
n = data.draw_bits(n_bits)
assert m == lower_bound
if m == 0:
assert n == 1
else:
assert n == m - 1
budget = 2 * n_bits * ceil(log(n_bits, 2)) + 2
assert runner.shrinks <= budget
def float_runner(start, condition):
def parse_buf(b):
return flt.lex_to_float(int_from_bytes(b))
def test_function(data):
f = flt.draw_float(data)
if condition(f):
data.mark_interesting()
runner = ConjectureRunner(test_function)
runner.test_function(
ConjectureData.for_buffer(
int_to_bytes(flt.float_to_lex(start), 8) + hbytes(1)))
assert runner.interesting_examples
return runner
def test_cached_with_masked_byte_agrees_with_results(byte_a, byte_b):
def f(data):
data.draw_bits(2)
runner = ConjectureRunner(f)
cached_a = runner.cached_test_function(hbytes([byte_a]))
cached_b = runner.cached_test_function(hbytes([byte_b]))
data_b = ConjectureData.for_buffer(hbytes([byte_b]))
runner.test_function(data_b)
# If the cache found an old result, then it should match the real result.
# If it did not, then it must be because A and B were different.
assert (cached_a is cached_b) == (cached_a.buffer == data_b.buffer)
def test_cached_with_masked_byte_agrees_with_results(byte_a, byte_b):
def f(data):
data.draw_bits(2)
runner = ConjectureRunner(f)
cached_a = runner.cached_test_function(hbytes([byte_a]))
cached_b = runner.cached_test_function(hbytes([byte_b]))
data_b = ConjectureData.for_buffer(hbytes([byte_b]))
runner.test_function(data_b)
# If the cache found an old result, then it should match the real result.
# If it did not, then it must be because A and B were different.
assert (cached_a is cached_b) == (cached_a.buffer == data_b.buffer)
def accept(tf):
runner = ConjectureRunner(tf, settings=TEST_SETTINGS, random=Random(0))
ran_examples = []
for e in examples:
e = hbytes(e)
data = ConjectureData.for_buffer(e)
try:
runner.test_function(data)
except RunIsComplete:
pass
ran_examples.append((e, data))
for e, d in ran_examples:
rewritten, status = runner.tree.rewrite(e)
assert status == d.status
assert rewritten == d.buffer
return runner
def test_always_reduces_integers_to_smallest_suitable_sizes(problem):
n, blob = problem
try:
d = ConjectureData.for_buffer(blob)
k = d.draw(st.integers())
stop = blob[len(d.buffer)]
except (StopTest, IndexError):
reject()
assume(k > n)
assume(stop > 0)
def f(data):
k = data.draw(st.integers())
data.output = repr(k)
if data.draw_bits(8) == stop and k >= n:
data.mark_interesting()
runner = ConjectureRunner(f, random=Random(0), settings=settings(
suppress_health_check=HealthCheck.all(), timeout=unlimited,
phases=(Phase.shrink,), database=None, verbosity=Verbosity.quiet
))
runner.test_function(ConjectureData.for_buffer(blob))
assert runner.interesting_examples
runner.run()
v, = runner.interesting_examples.values()
runner.debug = note
runner.debug_data(v)
m = ConjectureData.for_buffer(v.buffer).draw(st.integers())
assert m == n
# Upper bound on the length needed is calculated as follows:
# * We have an initial byte at the beginning to decide the length of the
# integer.
# * We have a terminal byte as the stop value.
# * The rest is the integer payload. This should be n. Including the sign
# bit, n needs (1 + n.bit_length()) / 8 bytes (rounded up). But we only
# have power of two sizes, so it may be up to a factor of two more than
# that.
assert len(v.buffer) <= 2 + 2 * max(1, ceil((1 + n.bit_length()) / 8))
def test_detects_too_small_block_starts():
call_count = [0]
def f(data):
assert call_count[0] == 0
call_count[0] += 1
data.draw_bytes(8)
data.mark_interesting()
runner = ConjectureRunner(f, settings=settings(database=None))
r = ConjectureData.for_buffer(hbytes(8))
runner.test_function(r)
assert r.status == Status.INTERESTING
assert call_count[0] == 1
r2 = runner.cached_test_function(hbytes([255] * 7))
assert r2.status == Status.OVERRUN
assert call_count[0] == 1
def test_prescreen_with_masked_byte_agrees_with_results(byte_a, byte_b):
def f(data):
data.draw_bits(2)
runner = ConjectureRunner(f)
data_a = ConjectureData.for_buffer(hbytes([byte_a]))
data_b = ConjectureData.for_buffer(hbytes([byte_b]))
runner.test_function(data_a)
prescreen_b = runner.prescreen_buffer(hbytes([byte_b]))
# Always test buffer B, to check whether the prescreen was correct.
runner.test_function(data_b)
# If the prescreen passed, then the buffers should be different.
# If it failed, then the buffers should be the same.
assert prescreen_b == (data_a.buffer != data_b.buffer)
def test_prescreen_with_masked_byte_agrees_with_results(byte_a, byte_b):
def f(data):
data.draw_bits(2)
runner = ConjectureRunner(f)
data_a = ConjectureData.for_buffer(hbytes([byte_a]))
data_b = ConjectureData.for_buffer(hbytes([byte_b]))
runner.test_function(data_a)
prescreen_b = runner.prescreen_buffer(hbytes([byte_b]))
# Always test buffer B, to check whether the prescreen was correct.
runner.test_function(data_b)
# If the prescreen passed, then the buffers should be different.
# If it failed, then the buffers should be the same.
assert prescreen_b == (data_a.buffer != data_b.buffer)
def test_only_calls_discard_at_top_level_pass():
def tree(data):
data.start_example('tree')
result = 1
if data.draw_bits(1):
result += max(tree(data), tree(data))
data.stop_example()
return result
def f(data):
if tree(data) == 3:
data.mark_interesting()
runner = ConjectureRunner(f,
settings=settings(
max_examples=1,
buffer_size=1024,
database=None,
suppress_health_check=HealthCheck.all(),
))
runner.test_function(ConjectureData.for_buffer([
1,
0,
1,
0,
0,
]))
assert runner.interesting_examples
last_data, = runner.interesting_examples.values()
shrinker = runner.new_shrinker(last_data,
lambda d: d.status == Status.INTERESTING)
shrinker.remove_discarded = MagicMock(return_value=None)
shrinker.adaptive_example_deletion()
assert shrinker.remove_discarded.call_count == 1
def test_debug_data(capsys):
buf = [0, 1, 2]
def f(data):
for x in hbytes(buf):
if data.draw_bits(8) != x:
data.mark_invalid()
data.start_example(1)
data.stop_example()
data.mark_interesting()
runner = ConjectureRunner(f, settings=settings(
max_examples=5000, buffer_size=1024,
database=None, suppress_health_check=HealthCheck.all(),
verbosity=Verbosity.debug
))
runner.test_function(ConjectureData.for_buffer(buf))
runner.run()
out, _ = capsys.readouterr()
assert re.match(u'\\d+ bytes \\[.*\\] -> ', out)
assert 'INTERESTING' in out
assert '[]' not in out
def test_can_reduce_poison_from_any_subtree(size, seed):
"""This test validates that we can minimize to any leaf node of a binary
tree, regardless of where in the tree the leaf is."""
random = Random(seed)
# Initially we create the minimal tree of size n, regardless of whether it
# is poisoned (which it won't be - the poison event essentially never
# happens when drawing uniformly at random).
# Choose p so that the expected size of the tree is equal to the desired
# size.
p = 1.0 / (2.0 - 1.0 / size)
strat = PoisonedTree(p)
def test_function(data):
v = data.draw(strat)
if len(v) >= size:
data.mark_interesting()
runner = ConjectureRunner(test_function,
random=random,
settings=settings(TEST_SETTINGS,
buffer_size=LOTS))
while not runner.interesting_examples:
runner.test_function(
runner.new_conjecture_data(lambda data, n: uniform(random, n)))
runner.shrink_interesting_examples()
data, = runner.interesting_examples.values()
assert len(ConjectureData.for_buffer(data.buffer).draw(strat)) == size
starts = [b.start for b in data.blocks if b.length == 2]
assert len(starts) % 2 == 0
for i in hrange(0, len(starts), 2):
# Now for each leaf position in the tree we try inserting a poison
# value artificially. Additionally, we add a marker to the end that
# must be preserved. The marker means that we are not allow to rely on
# discarding the end of the buffer to get the desired shrink.
u = starts[i]
marker = hbytes([1, 2, 3, 4])
def test_function_with_poison(data):
v = data.draw(strat)
m = data.draw_bytes(len(marker))
if POISON in v and m == marker:
data.mark_interesting()
runner = ConjectureRunner(test_function_with_poison,
random=random,
settings=TEST_SETTINGS)
runner.cached_test_function(data.buffer[:u] + hbytes([255]) * 4 +
data.buffer[u + 4:] + marker)
assert runner.interesting_examples
runner.shrink_interesting_examples()
shrunk, = runner.interesting_examples.values()
assert ConjectureData.for_buffer(
shrunk.buffer).draw(strat) == (POISON, )
def test_can_reduce_poison_from_any_subtree(size, seed):
"""This test validates that we can minimize to any leaf node of a binary
tree, regardless of where in the tree the leaf is."""
random = Random(seed)
# Initially we create the minimal tree of size n, regardless of whether it
# is poisoned (which it won't be - the poison event essentially never
# happens when drawing uniformly at random).
# Choose p so that the expected size of the tree is equal to the desired
# size.
p = 1.0 / (2.0 - 1.0 / size)
strat = PoisonedTree(p)
def test_function(data):
v = data.draw(strat)
if len(v) >= size:
data.mark_interesting()
runner = ConjectureRunner(
test_function, random=random, settings=settings(TEST_SETTINGS, buffer_size=LOTS)
)
while not runner.interesting_examples:
runner.test_function(
runner.new_conjecture_data(lambda data, n: uniform(random, n))
)
runner.shrink_interesting_examples()
data, = runner.interesting_examples.values()
assert len(ConjectureData.for_buffer(data.buffer).draw(strat)) == size
starts = [b.start for b in data.blocks if b.length == 2]
assert len(starts) % 2 == 0
for i in hrange(0, len(starts), 2):
# Now for each leaf position in the tree we try inserting a poison
# value artificially. Additionally, we add a marker to the end that
# must be preserved. The marker means that we are not allow to rely on
# discarding the end of the buffer to get the desired shrink.
u = starts[i]
marker = hbytes([1, 2, 3, 4])
def test_function_with_poison(data):
v = data.draw(strat)
m = data.draw_bytes(len(marker))
if POISON in v and m == marker:
data.mark_interesting()
runner = ConjectureRunner(
test_function_with_poison, random=random, settings=TEST_SETTINGS
)
runner.cached_test_function(
data.buffer[:u] + hbytes([255]) * 4 + data.buffer[u + 4 :] + marker
)
assert runner.interesting_examples
runner.shrink_interesting_examples()
shrunk, = runner.interesting_examples.values()
assert ConjectureData.for_buffer(shrunk.buffer).draw(strat) == (POISON,)
