def test_going_from_interesting_to_invalid_is_flaky(): tree = DataTree() data = ConjectureData.for_buffer(b"", observer=tree.new_observer()) with pytest.raises(StopTest): data.conclude_test(Status.INTERESTING) data = ConjectureData.for_buffer(b"", observer=tree.new_observer()) with pytest.raises(Flaky): data.conclude_test(Status.INVALID)
def test_always_reduces_integers_to_smallest_suitable_sizes(problem): n, blob = problem blob = hbytes(blob) try: d = ConjectureData.for_buffer(blob) k = d.draw(st.integers()) stop = blob[len(d.buffer)] except (StopTest, IndexError): reject() assume(k > n) assume(stop > 0) def f(data): k = data.draw(st.integers()) data.output = repr(k) if data.draw_bits(8) == stop and k >= n: data.mark_interesting() runner = ConjectureRunner(f, random=Random(0), settings=settings( suppress_health_check=HealthCheck.all(), timeout=unlimited, phases=(Phase.shrink,), database=None, verbosity=Verbosity.debug ), database_key=None) runner.test_function(ConjectureData.for_buffer(blob)) assert runner.interesting_examples v, = runner.interesting_examples.values() shrinker = runner.new_shrinker(v, lambda x: x.status == Status.INTERESTING) shrinker.clear_passes() shrinker.add_new_pass('minimize_individual_blocks') shrinker.shrink() v = shrinker.shrink_target m = ConjectureData.for_buffer(v.buffer).draw(st.integers()) assert m == n # Upper bound on the length needed is calculated as follows: # * We have an initial byte at the beginning to decide the length of the # integer. # * We have a terminal byte as the stop value. # * The rest is the integer payload. This should be n. Including the sign # bit, n needs (1 + n.bit_length()) / 8 bytes (rounded up). But we only # have power of two sizes, so it may be up to a factor of two more than # that. bits_needed = 1 + n.bit_length() actual_bits_needed = min( [s for s in WideRangeIntStrategy.sizes if s >= bits_needed]) bytes_needed = actual_bits_needed // 8 # 3 extra bytes: two for the sampler, one for the capping value. assert len(v.buffer) == 3 + bytes_needed
def test_changing_n_bits_is_flaky_in_prefix(): tree = DataTree() data = ConjectureData.for_buffer(b"\1", observer=tree.new_observer()) data.draw_bits(1) with pytest.raises(StopTest): data.conclude_test(Status.INTERESTING) data = ConjectureData.for_buffer(b"\1", observer=tree.new_observer()) with pytest.raises(Flaky): data.draw_bits(2)
def test_concluding_with_overrun_at_prefix_is_not_flaky(): tree = DataTree() data = ConjectureData.for_buffer(b"\1", observer=tree.new_observer()) data.draw_bits(1) with pytest.raises(StopTest): data.conclude_test(Status.INTERESTING) data = ConjectureData.for_buffer(b"", observer=tree.new_observer()) with pytest.raises(StopTest): data.conclude_test(Status.OVERRUN)
def test_changing_value_of_forced_is_flaky(): tree = DataTree() data = ConjectureData.for_buffer(b"\1", observer=tree.new_observer()) data.draw_bits(1, forced=1) with pytest.raises(StopTest): data.conclude_test(Status.INTERESTING) data = ConjectureData.for_buffer(b"\1\0", observer=tree.new_observer()) with pytest.raises(Flaky): data.draw_bits(1, forced=0)
def test_coin_biased_towards_falsehood(): p = 1.0 / 500 for i in range(255): assert not cu.biased_coin(ConjectureData.for_buffer([i]), p) second_order = [ cu.biased_coin(ConjectureData.for_buffer([255, i]), p) for i in range(255) ] assert False in second_order assert True in second_order
def test_will_avoid_exhausted_branches_for_necessary_prefix(): tree = DataTree() data = ConjectureData.for_buffer([0], observer=tree.new_observer()) data.draw_bits(1) data.freeze() data = ConjectureData.for_buffer([1, 1], observer=tree.new_observer()) data.draw_bits(1) data.draw_bits(8) data.freeze() assert list(tree.find_necessary_prefix_for_novelty()) == [1]
def test_extending_past_conclusion_is_flaky(): tree = DataTree() data = ConjectureData.for_buffer(b"\1", observer=tree.new_observer()) data.draw_bits(1) with pytest.raises(StopTest): data.conclude_test(Status.INTERESTING) data = ConjectureData.for_buffer(b"\1\0", observer=tree.new_observer()) data.draw_bits(1) with pytest.raises(Flaky): data.draw_bits(1)
def test_draw_write_round_trip(f): d = ConjectureData.for_buffer(hbytes(10)) flt.write_float(d, f) d2 = ConjectureData.for_buffer(d.buffer) g = flt.draw_float(d2) if f == f: assert f == g assert float_to_int(f) == float_to_int(g) d3 = ConjectureData.for_buffer(d2.buffer) flt.draw_float(d3) assert d3.buffer == d2.buffer
def test_foreign_key_primary(self, buf): # Regression test for #1307 company_strategy = from_model(Company, name=just("test")) strategy = from_model( CompanyExtension, company=company_strategy, self_modifying=just(2) ) try: ConjectureData.for_buffer(buf).draw(strategy) except HypothesisException: reject() # Draw again with the same buffer. This will cause a duplicate # primary key. ConjectureData.for_buffer(buf).draw(strategy) assert CompanyExtension.objects.all().count() == 1
def test_child_becomes_exhausted_after_split(): tree = DataTree() data = ConjectureData.for_buffer([0, 0], observer=tree.new_observer()) data.draw_bits(8) data.draw_bits(8, forced=0) data.freeze() data = ConjectureData.for_buffer([1, 0], observer=tree.new_observer()) data.draw_bits(8) data.draw_bits(8) data.freeze() assert not tree.is_exhausted assert tree.root.transition.children[0].is_exhausted
def test_avoids_zig_zag_trap(p): b, marker, lower_bound = p random.seed(0) b = hbytes(b) marker = hbytes(marker) n_bits = 8 * (len(b) + 1) def test_function(data): m = data.draw_bits(n_bits) if m < lower_bound: data.mark_invalid() n = data.draw_bits(n_bits) if data.draw_bytes(len(marker)) != marker: data.mark_invalid() if abs(m - n) == 1: data.mark_interesting() runner = ConjectureRunner( test_function, database_key=None, settings=settings( base_settings, phases=(Phase.generate, Phase.shrink) ) ) runner.test_function(ConjectureData.for_buffer( b + hbytes([0]) + b + hbytes([1]) + marker)) assert runner.interesting_examples runner.run() v, = runner.interesting_examples.values() data = ConjectureData.for_buffer(v.buffer) m = data.draw_bits(n_bits) n = data.draw_bits(n_bits) assert m == lower_bound if m == 0: assert n == 1 else: assert n == m - 1 budget = 2 * n_bits * ceil(log(n_bits, 2)) + 2 assert runner.shrinks <= budget
def test_will_generate_novel_prefix_to_avoid_exhausted_branches(): tree = DataTree() data = ConjectureData.for_buffer([1], observer=tree.new_observer()) data.draw_bits(1) data.freeze() data = ConjectureData.for_buffer([0, 1], observer=tree.new_observer()) data.draw_bits(1) data.draw_bits(8) data.freeze() prefix = list(tree.generate_novel_prefix(Random(0))) assert len(prefix) == 2 assert prefix[0] == 0
def test_exhaustive_enumeration(prefix, bits, seed): seen = set() def f(data): if prefix: data.write(hbytes(prefix)) assert len(data.buffer) == len(prefix) k = data.draw_bits(bits) assert k not in seen seen.add(k) size = 2 ** bits seen_prefixes = set() runner = ConjectureRunner( f, settings=settings(database=None, max_examples=size), random=Random(seed), ) with pytest.raises(RunIsComplete): runner.cached_test_function(b'') for _ in hrange(size): p = runner.generate_novel_prefix() assert p not in seen_prefixes seen_prefixes.add(p) data = ConjectureData.for_buffer( hbytes(p + hbytes(2 + len(prefix)))) runner.test_function(data) assert data.status == Status.VALID node = 0 for b in data.buffer: node = runner.tree[node][b] assert node in runner.dead assert len(seen) == size
def sample(seed, count, max_size): random = Random(seed) names = os.listdir(raw) names.sort() random.shuffle(names) printed = 0 for n in names: with open(os.path.join(raw, n), 'rb') as i: buffer = i.read() if max_size > 0 and len(buffer) > max_size: continue initial_data = ConjectureData.for_buffer(buffer) prog = os.path.join(programs, n + '.c') with open(prog, 'r') as i: already_good = is_valid(prog) if not already_good: try: gen(initial_data, prog) except StopTest: continue if not interesting_reason(prog, printing=False): continue print(n) printed += 1 if printed >= count: break
def test_database_uses_values_from_secondary_key(): key = b'key' database = InMemoryExampleDatabase() def f(data): if data.draw_bits(8) >= 5: data.mark_interesting() else: data.mark_invalid() runner = ConjectureRunner(f, settings=settings( max_examples=1, buffer_size=1024, database=database, suppress_health_check=HealthCheck.all(), ), database_key=key) for i in range(10): database.save(runner.secondary_key, hbytes([i])) runner.test_function(ConjectureData.for_buffer(hbytes([10]))) assert runner.interesting_examples assert len(set(database.fetch(key))) == 1 assert len(set(database.fetch(runner.secondary_key))) == 10 runner.clear_secondary_key() assert len(set(database.fetch(key))) == 1 assert set( map(int_from_bytes, database.fetch(runner.secondary_key)) ) == set(range(6, 11)) v, = runner.interesting_examples.values() assert list(v.buffer) == [5]
def test_draw_past_end_sets_overflow(): x = ConjectureData.for_buffer(bytes(5)) with pytest.raises(StopTest) as e: x.draw_bytes(6) assert e.value.testcounter == x.testcounter assert x.frozen assert x.status == Status.OVERRUN
def _run(self): self.last_data = None self.start_time = time.time() self.reuse_existing_examples() self.generate_new_examples() if ( Phase.shrink not in self.settings.phases or not self.interesting_examples ): self.exit_with(ExitReason.finished) for prev_data in sorted( self.interesting_examples.values(), key=lambda d: sort_key(d.buffer) ): assert prev_data.status == Status.INTERESTING data = ConjectureData.for_buffer(prev_data.buffer) self.test_function(data) if data.status != Status.INTERESTING: self.exit_with(ExitReason.flaky) while len(self.shrunk_examples) < len(self.interesting_examples): target, self.last_data = min([ (k, v) for k, v in self.interesting_examples.items() if k not in self.shrunk_examples], key=lambda kv: (sort_key(kv[1].buffer), sort_key(repr(kv[0]))), ) self.debug('Shrinking %r' % (target,)) assert self.last_data.interesting_origin == target self.shrink() self.shrunk_examples.add(target) self.exit_with(ExitReason.finished)
def test_can_observe_draws(): class LoggingObserver(DataObserver): def __init__(self): self.log = [] def draw_bits(self, *args): self.log.append(("draw",) + args) def conclude_test(self, *args): assert x.frozen self.log.append(("concluded",) + args) observer = LoggingObserver() x = ConjectureData.for_buffer(hbytes([1, 2, 3]), observer=observer) x.draw_bits(1) x.draw_bits(7, forced=10) x.draw_bits(8) with pytest.raises(StopTest): x.conclude_test(Status.INTERESTING, interesting_origin="neat") assert observer.log == [ ("draw", 1, False, 1), ("draw", 7, True, 10), ("draw", 8, False, 3), ("concluded", Status.INTERESTING, "neat"), ]
def test_can_get_odd_number_of_bits(): counts = Counter() for i in range(256): x = cu.getrandbits(ConjectureData.for_buffer([i]), 3) assert 0 <= x <= 7 counts[x] += 1 assert len(set(counts.values())) == 1
def incorporate_new_buffer(self, buffer): if buffer in self.seen: return False assert self.last_data.status == Status.INTERESTING if ( self.settings.timeout > 0 and time.time() >= self.start_time + self.settings.timeout ): self.exit_reason = ExitReason.timeout raise RunIsComplete() buffer = buffer[:self.last_data.index] if sort_key(buffer) >= sort_key(self.last_data.buffer): return False assert sort_key(buffer) <= sort_key(self.last_data.buffer) data = ConjectureData.for_buffer(buffer) self.test_function(data) if self.consider_new_test_data(data): self.shrinks += 1 self.last_data = data if self.shrinks >= self.settings.max_shrinks: self.exit_reason = ExitReason.max_shrinks raise RunIsComplete() self.last_data = data self.changed += 1 return True return False
def test_exhaustive_enumeration_of_partial_buffer(): seen = set() def f(data): k = data.draw_bytes(2) assert k[1] == 0 assert k not in seen seen.add(k) seen_prefixes = set() runner = ConjectureRunner( f, settings=settings(database=None, max_examples=256, buffer_size=2), random=Random(0), ) with pytest.raises(RunIsComplete): runner.cached_test_function(b'') for _ in hrange(256): p = runner.generate_novel_prefix() assert p not in seen_prefixes seen_prefixes.add(p) data = ConjectureData.for_buffer(hbytes(p + hbytes(2))) runner.test_function(data) assert data.status == Status.VALID node = 0 for b in data.buffer: node = runner.tree[node][b] assert node in runner.dead assert len(seen) == 256
def test_saves_data_while_shrinking(monkeypatch): key = b'hi there' n = 5 db = InMemoryExampleDatabase() assert list(db.fetch(key)) == [] seen = set() monkeypatch.setattr( ConjectureRunner, 'generate_new_examples', lambda runner: runner.test_function( ConjectureData.for_buffer([255] * 10))) def f(data): x = data.draw_bytes(10) if sum(x) >= 2000 and len(seen) < n: seen.add(hbytes(x)) if hbytes(x) in seen: data.mark_interesting() runner = ConjectureRunner( f, settings=settings(database=db), database_key=key) runner.run() assert runner.interesting_examples assert len(seen) == n in_db = non_covering_examples(db) assert in_db.issubset(seen) assert in_db == seen
def cleanup(): files = glob('hypothesis-examples/raw/*') Random().shuffle(files) for f in files: try: with open(f, 'rb') as i: data = ConjectureData.for_buffer(i.read()) except FileNotFoundError: continue with tempfile.TemporaryDirectory() as d: prog = os.path.join(d, 'test.c') try: gen(data, prog) except StopTest: continue name = os.path.basename(f) if not is_interesting(prog): print(f"Removing {name}") rmf(os.path.join(raw, name)) rmf(os.path.join(programs, name + '.c'))
def test_prescreen_with_masked_byte_agrees_with_results(byte_a, byte_b): def f(data): data.draw_bits(2) runner = ConjectureRunner(f) data_a = ConjectureData.for_buffer(hbytes([byte_a])) data_b = ConjectureData.for_buffer(hbytes([byte_b])) runner.test_function(data_a) prescreen_b = runner.prescreen_buffer(hbytes([byte_b])) # Always test buffer B, to check whether the prescreen was correct. runner.test_function(data_b) # If the prescreen passed, then the buffers should be different. # If it failed, then the buffers should be the same. assert prescreen_b == (data_a.buffer != data_b.buffer)
def test_can_write_empty_string(): d = ConjectureData.for_buffer([1, 1, 1]) d.draw_bits(1) d.write(hbytes()) d.draw_bits(1) d.draw_bits(0, forced=0) d.draw_bits(1) assert d.buffer == hbytes([1, 1, 1])
def test_compact_blocks_during_generation(): d = ConjectureData.for_buffer([1] * 10) for _ in hrange(5): d.draw_bits(1) assert len(list(d.blocks)) == 5 for _ in hrange(5): d.draw_bits(1) assert len(list(d.blocks)) == 10
def reuse_existing_examples(self): """If appropriate (we have a database and have been told to use it), try to reload existing examples from the database. If there are a lot we don't try all of them. We always try the smallest example in the database (which is guaranteed to be the last failure) and the largest (which is usually the seed example which the last failure came from but we don't enforce that). We then take a random sampling of the remainder and try those. Any examples that are no longer interesting are cleared out. """ if self.has_existing_examples(): self.debug('Reusing examples from database') # We have to do some careful juggling here. We have two database # corpora: The primary and secondary. The primary corpus is a # small set of minimized examples each of which has at one point # demonstrated a distinct bug. We want to retry all of these. # We also have a secondary corpus of examples that have at some # point demonstrated interestingness (currently only ones that # were previously non-minimal examples of a bug, but this will # likely expand in future). These are a good source of potentially # interesting examples, but there are a lot of them, so we down # sample the secondary corpus to a more manageable size. corpus = sorted( self.settings.database.fetch(self.database_key), key=sort_key ) desired_size = max(2, ceil(0.1 * self.settings.max_examples)) for extra_key in [self.secondary_key, self.covering_key]: if len(corpus) < desired_size: extra_corpus = list( self.settings.database.fetch(extra_key), ) shortfall = desired_size - len(corpus) if len(extra_corpus) <= shortfall: extra = extra_corpus else: extra = self.random.sample(extra_corpus, shortfall) extra.sort(key=sort_key) corpus.extend(extra) for existing in corpus: self.last_data = ConjectureData.for_buffer(existing) try: self.test_function(self.last_data) finally: if self.last_data.status != Status.INTERESTING: self.settings.database.delete( self.database_key, existing) self.settings.database.delete( self.secondary_key, existing)
def test_detects_too_small_block_starts(): def f(data): data.draw_bytes(8) data.mark_interesting() runner = ConjectureRunner(f, settings=settings(database=None)) r = ConjectureData.for_buffer(hbytes(8)) runner.test_function(r) assert r.status == Status.INTERESTING assert not runner.prescreen_buffer(hbytes([255] * 7))
def test_last_block_length(): d = ConjectureData.for_buffer([0] * 15) with pytest.raises(IndexError): d.blocks.last_block_length for n in hrange(1, 5 + 1): d.draw_bits(n * 8) assert d.blocks.last_block_length == n
def test_will_error_on_find(): d = ConjectureData.for_buffer(hbytes(0)) d.is_find = True with pytest.raises(InvalidArgument): d.draw(st.data())
def test_can_note_non_str(): d = ConjectureData.for_buffer(hbytes(0)) x = object() d.note(x) assert repr(x) in d.output
def test_drawing_zero_bits_is_free(): x = ConjectureData.for_buffer(hbytes()) assert x.draw_bits(0) == 0
def test_sampler_does_not_draw_minimum_if_zero(): sampler = cu.Sampler([0, 2, 47]) assert sampler.sample(ConjectureData.for_buffer([0, 0])) != 0
def test_too_small_to_be_useful_coin(): assert not cu.biased_coin(ConjectureData.for_buffer([1]), 0.5**65)
def new_conjecture_data_for_buffer(self, buffer): return ConjectureData.for_buffer(buffer, observer=self.tree.new_observer())
def test_buffer_draws_as_self(buf): x = ConjectureData.for_buffer(buf) assert x.draw_bytes(len(buf)) == buf
def test_can_mark_interesting(): x = ConjectureData.for_buffer(hbytes()) with pytest.raises(StopTest): x.mark_interesting() assert x.frozen assert x.status == Status.INTERESTING
def test_can_draw_zero_bytes(): x = ConjectureData.for_buffer(b"") for _ in range(10): assert x.draw_bytes(0) == b""
def test_can_mark_invalid(): x = ConjectureData.for_buffer(hbytes()) with pytest.raises(StopTest): x.mark_invalid() assert x.frozen assert x.status == Status.INVALID
def test_sampler_shrinks(): sampler = cu.Sampler([4.0, 8.0, 1.0, 1.0, 0.5]) assert sampler.sample(ConjectureData.for_buffer([0] * 3)) == 0
def test_center_in_middle_above(): assert ( cu.integer_range(ConjectureData.for_buffer([1, 0]), lower=0, upper=10, center=5) == 5 )
def test_can_note_str_as_non_repr(): d = ConjectureData.for_buffer(hbytes(0)) d.note(u"foo") assert d.output == u"foo"
def test_integer_range_lower_equals_upper(): data = ConjectureData.for_buffer([0]) assert cu.integer_range(data, lower=0, upper=0, center=0) == 0 assert len(data.buffer) == 1
def test_notes_repr(): x = ConjectureData.for_buffer(b"") x.note(b"hi") assert repr(b"hi") in x.output
def test_can_double_freeze(): x = ConjectureData.for_buffer(b"hi") x.freeze() assert x.frozen x.freeze() assert x.frozen
def test_choice(): assert cu.choice(ConjectureData.for_buffer([1]), [1, 2, 3]) == 2
def is_failure_inducing(b): try: return strat._attempt_one_draw( ConjectureData.for_buffer(b)) is None except StopTest: return False
def test_empty_strategy_is_invalid(): d = ConjectureData.for_buffer(hbytes(0)) with pytest.raises(StopTest): d.draw(st.nothing()) assert d.status == Status.INVALID
def test_blocks_lengths(): d = ConjectureData.for_buffer(hbytes(7)) d.draw_bits(32) d.draw_bits(16) d.draw_bits(1) assert [b.length for b in d.blocks] == [4, 2, 1]
def test_examples_support_negative_indexing(): d = ConjectureData.for_buffer(hbytes(2)) d.draw_bits(1) d.draw_bits(1) d.freeze() assert d.examples[-1].length == 1
def test_cannot_draw_after_freeze(): x = ConjectureData.for_buffer(b"hi") x.draw_bytes(1) x.freeze() with pytest.raises(Frozen): x.draw_bytes(1)
def test_fractional_float(): assert cu.fractional_float(ConjectureData.for_buffer([0] * 8)) == 0.0
def test_events_are_noted(): d = ConjectureData.for_buffer(()) d.note_event("hello") assert "hello" in d.events
def test_can_override_label(): d = ConjectureData.for_buffer(hbytes(2)) d.draw(st.booleans(), label=7) d.freeze() assert any(ex.label == 7 for ex in d.examples)
def test_biased_coin_can_be_forced(): assert cu.biased_coin(ConjectureData.for_buffer([0]), p=0.5, forced=True) assert not cu.biased_coin(ConjectureData.for_buffer([1]), p=0.5, forced=False)
def test_assert_biased_coin_always_treats_one_as_true(): assert cu.biased_coin(ConjectureData.for_buffer([0, 1]), p=1.0 / 257)
def test_can_draw_arbitrary_fractions(p, b): try: cu.biased_coin(ConjectureData.for_buffer(b), p) except StopTest: reject()
def test_integer_range_center_default(): assert ( cu.integer_range(ConjectureData.for_buffer([0]), lower=0, upper=10, center=None) == 0 )
def test_result_is_overrun(): d = ConjectureData.for_buffer(hbytes(0)) with pytest.raises(StopTest): d.draw_bits(1) assert d.as_result() is Overrun