def test_always_reduces_integers_to_smallest_suitable_sizes(problem): n, blob = problem blob = hbytes(blob) try: d = ConjectureData.for_buffer(blob) k = d.draw(st.integers()) stop = blob[len(d.buffer)] except (StopTest, IndexError): reject() assume(k > n) assume(stop > 0) def f(data): k = data.draw(st.integers()) data.output = repr(k) if data.draw_bits(8) == stop and k >= n: data.mark_interesting() runner = ConjectureRunner(f, random=Random(0), settings=settings( suppress_health_check=HealthCheck.all(), timeout=unlimited, phases=(Phase.shrink, ), database=None, verbosity=Verbosity.quiet)) runner.test_function(ConjectureData.for_buffer(blob)) assert runner.interesting_examples v, = runner.interesting_examples.values() shrinker = runner.new_shrinker(v, lambda x: x.status == Status.INTERESTING) shrinker.single_greedy_shrink_iteration = \ shrinker.minimize_individual_blocks shrinker.shrink() v = shrinker.shrink_target m = ConjectureData.for_buffer(v.buffer).draw(st.integers()) assert m == n # Upper bound on the length needed is calculated as follows: # * We have an initial byte at the beginning to decide the length of the # integer. # * We have a terminal byte as the stop value. # * The rest is the integer payload. This should be n. Including the sign # bit, n needs (1 + n.bit_length()) / 8 bytes (rounded up). But we only # have power of two sizes, so it may be up to a factor of two more than # that. bits_needed = 1 + n.bit_length() actual_bits_needed = min( [s for s in WideRangeIntStrategy.sizes if s >= bits_needed]) bytes_needed = actual_bits_needed // 8 # 3 extra bytes: two for the sampler, one for the capping value. assert len(v.buffer) == 3 + bytes_needed
def test_exhaustive_enumeration(prefix, bits, seed): seen = set() def f(data): if prefix: data.write(hbytes(prefix)) assert len(data.buffer) == len(prefix) k = data.draw_bits(bits) assert k not in seen seen.add(k) size = 2 ** bits seen_prefixes = set() runner = ConjectureRunner( f, settings=settings(database=None, max_examples=size), random=Random(seed), ) with pytest.raises(RunIsComplete): runner.cached_test_function(b'') for _ in hrange(size): p = runner.generate_novel_prefix() assert p not in seen_prefixes seen_prefixes.add(p) data = ConjectureData.for_buffer( hbytes(p + hbytes(2 + len(prefix)))) runner.test_function(data) assert data.status == Status.VALID node = 0 for b in data.buffer: node = runner.tree[node][b] assert node in runner.dead assert len(seen) == size
def test_database_uses_values_from_secondary_key(): key = b'key' database = InMemoryExampleDatabase() def f(data): if data.draw_bits(8) >= 5: data.mark_interesting() else: data.mark_invalid() runner = ConjectureRunner(f, settings=settings( max_examples=1, buffer_size=1024, database=database, suppress_health_check=HealthCheck.all(), ), database_key=key) for i in range(10): database.save(runner.secondary_key, hbytes([i])) runner.test_function(ConjectureData.for_buffer(hbytes([10]))) assert runner.interesting_examples assert len(set(database.fetch(key))) == 1 assert len(set(database.fetch(runner.secondary_key))) == 10 runner.clear_secondary_key() assert len(set(database.fetch(key))) == 1 assert set( map(int_from_bytes, database.fetch(runner.secondary_key)) ) == set(range(6, 11)) v, = runner.interesting_examples.values() assert list(v.buffer) == [5]
def test_debug_data(capsys): buf = [0, 1, 2] def f(data): for x in hbytes(buf): if data.draw_bits(8) != x: data.mark_invalid() data.start_example(1) data.stop_example() data.mark_interesting() runner = ConjectureRunner( f, settings=settings( max_examples=5000, buffer_size=1024, database=None, suppress_health_check=HealthCheck.all(), verbosity=Verbosity.debug, ), ) runner.test_function(ConjectureData.for_buffer(buf)) runner.run() out, _ = capsys.readouterr() assert re.match(u"\\d+ bytes \\[.*\\] -> ", out) assert "INTERESTING" in out assert "[]" not in out
def test_database_clears_secondary_key(): key = b"key" database = InMemoryExampleDatabase() def f(data): if data.draw_bits(8) == 10: data.mark_interesting() else: data.mark_invalid() runner = ConjectureRunner( f, settings=settings( max_examples=1, buffer_size=1024, database=database, suppress_health_check=HealthCheck.all(), ), database_key=key, ) for i in range(10): database.save(runner.secondary_key, hbytes([i])) runner.test_function(ConjectureData.for_buffer(hbytes([10]))) assert runner.interesting_examples assert len(set(database.fetch(key))) == 1 assert len(set(database.fetch(runner.secondary_key))) == 10 runner.clear_secondary_key() assert len(set(database.fetch(key))) == 1 assert len(set(database.fetch(runner.secondary_key))) == 0
def test_exhaustive_enumeration_of_partial_buffer(): seen = set() def f(data): k = data.draw_bytes(2) assert k[1] == 0 assert k not in seen seen.add(k) seen_prefixes = set() runner = ConjectureRunner( f, settings=settings(database=None, max_examples=256, buffer_size=2), random=Random(0), ) with pytest.raises(RunIsComplete): runner.cached_test_function(b'') for _ in hrange(256): p = runner.generate_novel_prefix() assert p not in seen_prefixes seen_prefixes.add(p) data = ConjectureData.for_buffer(hbytes(p + hbytes(2))) runner.test_function(data) assert data.status == Status.VALID node = 0 for b in data.buffer: node = runner.tree[node][b] assert node in runner.dead assert len(seen) == 256
def test_exhaustive_enumeration_of_partial_buffer(): seen = set() def f(data): k = data.draw_bytes(2) assert k[1] == 0 assert k not in seen seen.add(k) seen_prefixes = set() runner = ConjectureRunner( f, settings=settings(database=None, max_examples=256, buffer_size=2), random=Random(0), ) with pytest.raises(RunIsComplete): runner.cached_test_function(b'') for _ in hrange(256): p = runner.generate_novel_prefix() assert p not in seen_prefixes seen_prefixes.add(p) data = ConjectureData.for_buffer(hbytes(p + hbytes(2))) runner.test_function(data) assert data.status == Status.VALID node = 0 for b in data.buffer: node = runner.tree[node][b] assert node in runner.dead assert len(seen) == 256
def test_detects_too_small_block_starts(): def f(data): data.draw_bytes(8) data.mark_interesting() runner = ConjectureRunner(f, settings=settings(database=None)) r = ConjectureData.for_buffer(hbytes(8)) runner.test_function(r) assert r.status == Status.INTERESTING assert not runner.prescreen_buffer(hbytes([255] * 7))
def test_always_reduces_integers_to_smallest_suitable_sizes(problem): n, blob = problem blob = hbytes(blob) try: d = ConjectureData.for_buffer(blob) k = d.draw(st.integers()) stop = blob[len(d.buffer)] except (StopTest, IndexError): reject() assume(k > n) assume(stop > 0) def f(data): k = data.draw(st.integers()) data.output = repr(k) if data.draw_bits(8) == stop and k >= n: data.mark_interesting() runner = ConjectureRunner(f, random=Random(0), settings=settings( suppress_health_check=HealthCheck.all(), timeout=unlimited, phases=(Phase.shrink,), database=None, verbosity=Verbosity.debug ), database_key=None) runner.test_function(ConjectureData.for_buffer(blob)) assert runner.interesting_examples v, = runner.interesting_examples.values() shrinker = runner.new_shrinker(v, lambda x: x.status == Status.INTERESTING) shrinker.clear_passes() shrinker.add_new_pass('minimize_individual_blocks') shrinker.shrink() v = shrinker.shrink_target m = ConjectureData.for_buffer(v.buffer).draw(st.integers()) assert m == n # Upper bound on the length needed is calculated as follows: # * We have an initial byte at the beginning to decide the length of the # integer. # * We have a terminal byte as the stop value. # * The rest is the integer payload. This should be n. Including the sign # bit, n needs (1 + n.bit_length()) / 8 bytes (rounded up). But we only # have power of two sizes, so it may be up to a factor of two more than # that. bits_needed = 1 + n.bit_length() actual_bits_needed = min( [s for s in WideRangeIntStrategy.sizes if s >= bits_needed]) bytes_needed = actual_bits_needed // 8 # 3 extra bytes: two for the sampler, one for the capping value. assert len(v.buffer) == 3 + bytes_needed
def accept(f): with deterministic_PRNG(): runner = ConjectureRunner(f, settings=settings( max_examples=5000, buffer_size=1024, database=None, suppress_health_check=HealthCheck.all(), )) runner.test_function(ConjectureData.for_buffer(start)) assert runner.interesting_examples last_data, = runner.interesting_examples.values() return runner.new_shrinker( last_data, lambda d: d.status == Status.INTERESTING )
def test_avoids_zig_zag_trap(p): b, marker, lower_bound = p b = hbytes(b) marker = hbytes(marker) n_bits = 8 * (len(b) + 1) def test_function(data): m = data.draw_bits(n_bits) if m < lower_bound: data.mark_invalid() n = data.draw_bits(n_bits) if data.draw_bytes(len(marker)) != marker: data.mark_invalid() if abs(m - n) == 1: data.mark_interesting() runner = ConjectureRunner(test_function, database_key=None, settings=settings(database=None, max_shrinks=100, verbosity=Verbosity.debug)) runner.debug = note original_debug_data = runner.debug_data def debug_interesting(data): if data.status == Status.INTERESTING: original_debug_data(data) runner.debug_data = debug_interesting runner.test_function( ConjectureData.for_buffer(b + hbytes([0]) + b + hbytes([1]) + marker)) assert runner.interesting_examples runner.run() v, = runner.interesting_examples.values() data = ConjectureData.for_buffer(v.buffer) m = data.draw_bits(n_bits) n = data.draw_bits(n_bits) assert m == lower_bound if m == 0: assert n == 1 else: assert n == m - 1
def test_avoids_zig_zag_trap(p): b, marker, lower_bound = p random.seed(0) b = hbytes(b) marker = hbytes(marker) n_bits = 8 * (len(b) + 1) def test_function(data): m = data.draw_bits(n_bits) if m < lower_bound: data.mark_invalid() n = data.draw_bits(n_bits) if data.draw_bytes(len(marker)) != marker: data.mark_invalid() if abs(m - n) == 1: data.mark_interesting() runner = ConjectureRunner( test_function, database_key=None, settings=settings( base_settings, phases=(Phase.generate, Phase.shrink) ) ) runner.test_function(ConjectureData.for_buffer( b + hbytes([0]) + b + hbytes([1]) + marker)) assert runner.interesting_examples runner.run() v, = runner.interesting_examples.values() data = ConjectureData.for_buffer(v.buffer) m = data.draw_bits(n_bits) n = data.draw_bits(n_bits) assert m == lower_bound if m == 0: assert n == 1 else: assert n == m - 1 budget = 2 * n_bits * ceil(log(n_bits, 2)) + 2 assert runner.shrinks <= budget
def float_runner(start, condition): def parse_buf(b): return flt.lex_to_float(int_from_bytes(b)) def test_function(data): f = flt.draw_float(data) if condition(f): data.mark_interesting() runner = ConjectureRunner(test_function) runner.test_function( ConjectureData.for_buffer( int_to_bytes(flt.float_to_lex(start), 8) + hbytes(1))) assert runner.interesting_examples return runner
def test_cached_with_masked_byte_agrees_with_results(byte_a, byte_b): def f(data): data.draw_bits(2) runner = ConjectureRunner(f) cached_a = runner.cached_test_function(hbytes([byte_a])) cached_b = runner.cached_test_function(hbytes([byte_b])) data_b = ConjectureData.for_buffer(hbytes([byte_b])) runner.test_function(data_b) # If the cache found an old result, then it should match the real result. # If it did not, then it must be because A and B were different. assert (cached_a is cached_b) == (cached_a.buffer == data_b.buffer)
def test_cached_with_masked_byte_agrees_with_results(byte_a, byte_b): def f(data): data.draw_bits(2) runner = ConjectureRunner(f) cached_a = runner.cached_test_function(hbytes([byte_a])) cached_b = runner.cached_test_function(hbytes([byte_b])) data_b = ConjectureData.for_buffer(hbytes([byte_b])) runner.test_function(data_b) # If the cache found an old result, then it should match the real result. # If it did not, then it must be because A and B were different. assert (cached_a is cached_b) == (cached_a.buffer == data_b.buffer)
def accept(tf): runner = ConjectureRunner(tf, settings=TEST_SETTINGS, random=Random(0)) ran_examples = [] for e in examples: e = hbytes(e) data = ConjectureData.for_buffer(e) try: runner.test_function(data) except RunIsComplete: pass ran_examples.append((e, data)) for e, d in ran_examples: rewritten, status = runner.tree.rewrite(e) assert status == d.status assert rewritten == d.buffer return runner
def test_always_reduces_integers_to_smallest_suitable_sizes(problem): n, blob = problem try: d = ConjectureData.for_buffer(blob) k = d.draw(st.integers()) stop = blob[len(d.buffer)] except (StopTest, IndexError): reject() assume(k > n) assume(stop > 0) def f(data): k = data.draw(st.integers()) data.output = repr(k) if data.draw_bits(8) == stop and k >= n: data.mark_interesting() runner = ConjectureRunner(f, random=Random(0), settings=settings( suppress_health_check=HealthCheck.all(), timeout=unlimited, phases=(Phase.shrink,), database=None, verbosity=Verbosity.quiet )) runner.test_function(ConjectureData.for_buffer(blob)) assert runner.interesting_examples runner.run() v, = runner.interesting_examples.values() runner.debug = note runner.debug_data(v) m = ConjectureData.for_buffer(v.buffer).draw(st.integers()) assert m == n # Upper bound on the length needed is calculated as follows: # * We have an initial byte at the beginning to decide the length of the # integer. # * We have a terminal byte as the stop value. # * The rest is the integer payload. This should be n. Including the sign # bit, n needs (1 + n.bit_length()) / 8 bytes (rounded up). But we only # have power of two sizes, so it may be up to a factor of two more than # that. assert len(v.buffer) <= 2 + 2 * max(1, ceil((1 + n.bit_length()) / 8))
def test_detects_too_small_block_starts(): call_count = [0] def f(data): assert call_count[0] == 0 call_count[0] += 1 data.draw_bytes(8) data.mark_interesting() runner = ConjectureRunner(f, settings=settings(database=None)) r = ConjectureData.for_buffer(hbytes(8)) runner.test_function(r) assert r.status == Status.INTERESTING assert call_count[0] == 1 r2 = runner.cached_test_function(hbytes([255] * 7)) assert r2.status == Status.OVERRUN assert call_count[0] == 1
def test_prescreen_with_masked_byte_agrees_with_results(byte_a, byte_b): def f(data): data.draw_bits(2) runner = ConjectureRunner(f) data_a = ConjectureData.for_buffer(hbytes([byte_a])) data_b = ConjectureData.for_buffer(hbytes([byte_b])) runner.test_function(data_a) prescreen_b = runner.prescreen_buffer(hbytes([byte_b])) # Always test buffer B, to check whether the prescreen was correct. runner.test_function(data_b) # If the prescreen passed, then the buffers should be different. # If it failed, then the buffers should be the same. assert prescreen_b == (data_a.buffer != data_b.buffer)
def test_prescreen_with_masked_byte_agrees_with_results(byte_a, byte_b): def f(data): data.draw_bits(2) runner = ConjectureRunner(f) data_a = ConjectureData.for_buffer(hbytes([byte_a])) data_b = ConjectureData.for_buffer(hbytes([byte_b])) runner.test_function(data_a) prescreen_b = runner.prescreen_buffer(hbytes([byte_b])) # Always test buffer B, to check whether the prescreen was correct. runner.test_function(data_b) # If the prescreen passed, then the buffers should be different. # If it failed, then the buffers should be the same. assert prescreen_b == (data_a.buffer != data_b.buffer)
def test_only_calls_discard_at_top_level_pass(): def tree(data): data.start_example('tree') result = 1 if data.draw_bits(1): result += max(tree(data), tree(data)) data.stop_example() return result def f(data): if tree(data) == 3: data.mark_interesting() runner = ConjectureRunner(f, settings=settings( max_examples=1, buffer_size=1024, database=None, suppress_health_check=HealthCheck.all(), )) runner.test_function(ConjectureData.for_buffer([ 1, 0, 1, 0, 0, ])) assert runner.interesting_examples last_data, = runner.interesting_examples.values() shrinker = runner.new_shrinker(last_data, lambda d: d.status == Status.INTERESTING) shrinker.remove_discarded = MagicMock(return_value=None) shrinker.adaptive_example_deletion() assert shrinker.remove_discarded.call_count == 1
def test_debug_data(capsys): buf = [0, 1, 2] def f(data): for x in hbytes(buf): if data.draw_bits(8) != x: data.mark_invalid() data.start_example(1) data.stop_example() data.mark_interesting() runner = ConjectureRunner(f, settings=settings( max_examples=5000, buffer_size=1024, database=None, suppress_health_check=HealthCheck.all(), verbosity=Verbosity.debug )) runner.test_function(ConjectureData.for_buffer(buf)) runner.run() out, _ = capsys.readouterr() assert re.match(u'\\d+ bytes \\[.*\\] -> ', out) assert 'INTERESTING' in out assert '[]' not in out
def test_can_reduce_poison_from_any_subtree(size, seed): """This test validates that we can minimize to any leaf node of a binary tree, regardless of where in the tree the leaf is.""" random = Random(seed) # Initially we create the minimal tree of size n, regardless of whether it # is poisoned (which it won't be - the poison event essentially never # happens when drawing uniformly at random). # Choose p so that the expected size of the tree is equal to the desired # size. p = 1.0 / (2.0 - 1.0 / size) strat = PoisonedTree(p) def test_function(data): v = data.draw(strat) if len(v) >= size: data.mark_interesting() runner = ConjectureRunner(test_function, random=random, settings=settings(TEST_SETTINGS, buffer_size=LOTS)) while not runner.interesting_examples: runner.test_function( runner.new_conjecture_data(lambda data, n: uniform(random, n))) runner.shrink_interesting_examples() data, = runner.interesting_examples.values() assert len(ConjectureData.for_buffer(data.buffer).draw(strat)) == size starts = [b.start for b in data.blocks if b.length == 2] assert len(starts) % 2 == 0 for i in hrange(0, len(starts), 2): # Now for each leaf position in the tree we try inserting a poison # value artificially. Additionally, we add a marker to the end that # must be preserved. The marker means that we are not allow to rely on # discarding the end of the buffer to get the desired shrink. u = starts[i] marker = hbytes([1, 2, 3, 4]) def test_function_with_poison(data): v = data.draw(strat) m = data.draw_bytes(len(marker)) if POISON in v and m == marker: data.mark_interesting() runner = ConjectureRunner(test_function_with_poison, random=random, settings=TEST_SETTINGS) runner.cached_test_function(data.buffer[:u] + hbytes([255]) * 4 + data.buffer[u + 4:] + marker) assert runner.interesting_examples runner.shrink_interesting_examples() shrunk, = runner.interesting_examples.values() assert ConjectureData.for_buffer( shrunk.buffer).draw(strat) == (POISON, )
def test_can_reduce_poison_from_any_subtree(size, seed): """This test validates that we can minimize to any leaf node of a binary tree, regardless of where in the tree the leaf is.""" random = Random(seed) # Initially we create the minimal tree of size n, regardless of whether it # is poisoned (which it won't be - the poison event essentially never # happens when drawing uniformly at random). # Choose p so that the expected size of the tree is equal to the desired # size. p = 1.0 / (2.0 - 1.0 / size) strat = PoisonedTree(p) def test_function(data): v = data.draw(strat) if len(v) >= size: data.mark_interesting() runner = ConjectureRunner( test_function, random=random, settings=settings(TEST_SETTINGS, buffer_size=LOTS) ) while not runner.interesting_examples: runner.test_function( runner.new_conjecture_data(lambda data, n: uniform(random, n)) ) runner.shrink_interesting_examples() data, = runner.interesting_examples.values() assert len(ConjectureData.for_buffer(data.buffer).draw(strat)) == size starts = [b.start for b in data.blocks if b.length == 2] assert len(starts) % 2 == 0 for i in hrange(0, len(starts), 2): # Now for each leaf position in the tree we try inserting a poison # value artificially. Additionally, we add a marker to the end that # must be preserved. The marker means that we are not allow to rely on # discarding the end of the buffer to get the desired shrink. u = starts[i] marker = hbytes([1, 2, 3, 4]) def test_function_with_poison(data): v = data.draw(strat) m = data.draw_bytes(len(marker)) if POISON in v and m == marker: data.mark_interesting() runner = ConjectureRunner( test_function_with_poison, random=random, settings=TEST_SETTINGS ) runner.cached_test_function( data.buffer[:u] + hbytes([255]) * 4 + data.buffer[u + 4 :] + marker ) assert runner.interesting_examples runner.shrink_interesting_examples() shrunk, = runner.interesting_examples.values() assert ConjectureData.for_buffer(shrunk.buffer).draw(strat) == (POISON,)