def test_going_from_interesting_to_invalid_is_flaky():
tree = DataTree()
data = ConjectureData.for_buffer(b"", observer=tree.new_observer())
with pytest.raises(StopTest):
data.conclude_test(Status.INTERESTING)
data = ConjectureData.for_buffer(b"", observer=tree.new_observer())
with pytest.raises(Flaky):
data.conclude_test(Status.INVALID)
def test_always_reduces_integers_to_smallest_suitable_sizes(problem):
n, blob = problem
blob = hbytes(blob)
try:
d = ConjectureData.for_buffer(blob)
k = d.draw(st.integers())
stop = blob[len(d.buffer)]
except (StopTest, IndexError):
reject()
assume(k > n)
assume(stop > 0)
def f(data):
k = data.draw(st.integers())
data.output = repr(k)
if data.draw_bits(8) == stop and k >= n:
data.mark_interesting()
runner = ConjectureRunner(f, random=Random(0), settings=settings(
suppress_health_check=HealthCheck.all(), timeout=unlimited,
phases=(Phase.shrink,), database=None, verbosity=Verbosity.debug
), database_key=None)
runner.test_function(ConjectureData.for_buffer(blob))
assert runner.interesting_examples
v, = runner.interesting_examples.values()
shrinker = runner.new_shrinker(v, lambda x: x.status == Status.INTERESTING)
shrinker.clear_passes()
shrinker.add_new_pass('minimize_individual_blocks')
shrinker.shrink()
v = shrinker.shrink_target
m = ConjectureData.for_buffer(v.buffer).draw(st.integers())
assert m == n
# Upper bound on the length needed is calculated as follows:
# * We have an initial byte at the beginning to decide the length of the
# integer.
# * We have a terminal byte as the stop value.
# * The rest is the integer payload. This should be n. Including the sign
# bit, n needs (1 + n.bit_length()) / 8 bytes (rounded up). But we only
# have power of two sizes, so it may be up to a factor of two more than
# that.
bits_needed = 1 + n.bit_length()
actual_bits_needed = min(
[s for s in WideRangeIntStrategy.sizes if s >= bits_needed])
bytes_needed = actual_bits_needed // 8
# 3 extra bytes: two for the sampler, one for the capping value.
assert len(v.buffer) == 3 + bytes_needed
def test_changing_n_bits_is_flaky_in_prefix():
tree = DataTree()
data = ConjectureData.for_buffer(b"\1", observer=tree.new_observer())
data.draw_bits(1)
with pytest.raises(StopTest):
data.conclude_test(Status.INTERESTING)
data = ConjectureData.for_buffer(b"\1", observer=tree.new_observer())
with pytest.raises(Flaky):
data.draw_bits(2)
def test_concluding_with_overrun_at_prefix_is_not_flaky():
tree = DataTree()
data = ConjectureData.for_buffer(b"\1", observer=tree.new_observer())
data.draw_bits(1)
with pytest.raises(StopTest):
data.conclude_test(Status.INTERESTING)
data = ConjectureData.for_buffer(b"", observer=tree.new_observer())
with pytest.raises(StopTest):
data.conclude_test(Status.OVERRUN)
def test_changing_value_of_forced_is_flaky():
tree = DataTree()
data = ConjectureData.for_buffer(b"\1", observer=tree.new_observer())
data.draw_bits(1, forced=1)
with pytest.raises(StopTest):
data.conclude_test(Status.INTERESTING)
data = ConjectureData.for_buffer(b"\1\0", observer=tree.new_observer())
with pytest.raises(Flaky):
data.draw_bits(1, forced=0)
def test_coin_biased_towards_falsehood():
p = 1.0 / 500
for i in range(255):
assert not cu.biased_coin(ConjectureData.for_buffer([i]), p)
second_order = [
cu.biased_coin(ConjectureData.for_buffer([255, i]), p) for i in range(255)
]
assert False in second_order
assert True in second_order
def test_will_avoid_exhausted_branches_for_necessary_prefix():
tree = DataTree()
data = ConjectureData.for_buffer([0], observer=tree.new_observer())
data.draw_bits(1)
data.freeze()
data = ConjectureData.for_buffer([1, 1], observer=tree.new_observer())
data.draw_bits(1)
data.draw_bits(8)
data.freeze()
assert list(tree.find_necessary_prefix_for_novelty()) == [1]
def test_extending_past_conclusion_is_flaky():
tree = DataTree()
data = ConjectureData.for_buffer(b"\1", observer=tree.new_observer())
data.draw_bits(1)
with pytest.raises(StopTest):
data.conclude_test(Status.INTERESTING)
data = ConjectureData.for_buffer(b"\1\0", observer=tree.new_observer())
data.draw_bits(1)
with pytest.raises(Flaky):
data.draw_bits(1)
def test_draw_write_round_trip(f):
d = ConjectureData.for_buffer(hbytes(10))
flt.write_float(d, f)
d2 = ConjectureData.for_buffer(d.buffer)
g = flt.draw_float(d2)
if f == f:
assert f == g
assert float_to_int(f) == float_to_int(g)
d3 = ConjectureData.for_buffer(d2.buffer)
flt.draw_float(d3)
assert d3.buffer == d2.buffer
def test_foreign_key_primary(self, buf):
# Regression test for #1307
company_strategy = from_model(Company, name=just("test"))
strategy = from_model(
CompanyExtension, company=company_strategy, self_modifying=just(2)
)
try:
ConjectureData.for_buffer(buf).draw(strategy)
except HypothesisException:
reject()
# Draw again with the same buffer. This will cause a duplicate
# primary key.
ConjectureData.for_buffer(buf).draw(strategy)
assert CompanyExtension.objects.all().count() == 1
def test_child_becomes_exhausted_after_split():
tree = DataTree()
data = ConjectureData.for_buffer([0, 0], observer=tree.new_observer())
data.draw_bits(8)
data.draw_bits(8, forced=0)
data.freeze()
data = ConjectureData.for_buffer([1, 0], observer=tree.new_observer())
data.draw_bits(8)
data.draw_bits(8)
data.freeze()
assert not tree.is_exhausted
assert tree.root.transition.children[0].is_exhausted
def test_avoids_zig_zag_trap(p):
b, marker, lower_bound = p
random.seed(0)
b = hbytes(b)
marker = hbytes(marker)
n_bits = 8 * (len(b) + 1)
def test_function(data):
m = data.draw_bits(n_bits)
if m < lower_bound:
data.mark_invalid()
n = data.draw_bits(n_bits)
if data.draw_bytes(len(marker)) != marker:
data.mark_invalid()
if abs(m - n) == 1:
data.mark_interesting()
runner = ConjectureRunner(
test_function, database_key=None, settings=settings(
base_settings,
phases=(Phase.generate, Phase.shrink)
)
)
runner.test_function(ConjectureData.for_buffer(
b + hbytes([0]) + b + hbytes([1]) + marker))
assert runner.interesting_examples
runner.run()
v, = runner.interesting_examples.values()
data = ConjectureData.for_buffer(v.buffer)
m = data.draw_bits(n_bits)
n = data.draw_bits(n_bits)
assert m == lower_bound
if m == 0:
assert n == 1
else:
assert n == m - 1
budget = 2 * n_bits * ceil(log(n_bits, 2)) + 2
assert runner.shrinks <= budget
def test_will_generate_novel_prefix_to_avoid_exhausted_branches():
tree = DataTree()
data = ConjectureData.for_buffer([1], observer=tree.new_observer())
data.draw_bits(1)
data.freeze()
data = ConjectureData.for_buffer([0, 1], observer=tree.new_observer())
data.draw_bits(1)
data.draw_bits(8)
data.freeze()
prefix = list(tree.generate_novel_prefix(Random(0)))
assert len(prefix) == 2
assert prefix[0] == 0
def test_exhaustive_enumeration(prefix, bits, seed):
seen = set()
def f(data):
if prefix:
data.write(hbytes(prefix))
assert len(data.buffer) == len(prefix)
k = data.draw_bits(bits)
assert k not in seen
seen.add(k)
size = 2 ** bits
seen_prefixes = set()
runner = ConjectureRunner(
f, settings=settings(database=None, max_examples=size),
random=Random(seed),
)
with pytest.raises(RunIsComplete):
runner.cached_test_function(b'')
for _ in hrange(size):
p = runner.generate_novel_prefix()
assert p not in seen_prefixes
seen_prefixes.add(p)
data = ConjectureData.for_buffer(
hbytes(p + hbytes(2 + len(prefix))))
runner.test_function(data)
assert data.status == Status.VALID
node = 0
for b in data.buffer:
node = runner.tree[node][b]
assert node in runner.dead
assert len(seen) == size
def sample(seed, count, max_size):
random = Random(seed)
names = os.listdir(raw)
names.sort()
random.shuffle(names)
printed = 0
for n in names:
with open(os.path.join(raw, n), 'rb') as i:
buffer = i.read()
if max_size > 0 and len(buffer) > max_size:
continue
initial_data = ConjectureData.for_buffer(buffer)
prog = os.path.join(programs, n + '.c')
with open(prog, 'r') as i:
already_good = is_valid(prog)
if not already_good:
try:
gen(initial_data, prog)
except StopTest:
continue
if not interesting_reason(prog, printing=False):
continue
print(n)
printed += 1
if printed >= count:
break
def test_database_uses_values_from_secondary_key():
key = b'key'
database = InMemoryExampleDatabase()
def f(data):
if data.draw_bits(8) >= 5:
data.mark_interesting()
else:
data.mark_invalid()
runner = ConjectureRunner(f, settings=settings(
max_examples=1, buffer_size=1024,
database=database, suppress_health_check=HealthCheck.all(),
), database_key=key)
for i in range(10):
database.save(runner.secondary_key, hbytes([i]))
runner.test_function(ConjectureData.for_buffer(hbytes([10])))
assert runner.interesting_examples
assert len(set(database.fetch(key))) == 1
assert len(set(database.fetch(runner.secondary_key))) == 10
runner.clear_secondary_key()
assert len(set(database.fetch(key))) == 1
assert set(
map(int_from_bytes, database.fetch(runner.secondary_key))
) == set(range(6, 11))
v, = runner.interesting_examples.values()
assert list(v.buffer) == [5]
def test_draw_past_end_sets_overflow():
x = ConjectureData.for_buffer(bytes(5))
with pytest.raises(StopTest) as e:
x.draw_bytes(6)
assert e.value.testcounter == x.testcounter
assert x.frozen
assert x.status == Status.OVERRUN
def _run(self):
self.last_data = None
self.start_time = time.time()
self.reuse_existing_examples()
self.generate_new_examples()
if (
Phase.shrink not in self.settings.phases or
not self.interesting_examples
):
self.exit_with(ExitReason.finished)
for prev_data in sorted(
self.interesting_examples.values(),
key=lambda d: sort_key(d.buffer)
):
assert prev_data.status == Status.INTERESTING
data = ConjectureData.for_buffer(prev_data.buffer)
self.test_function(data)
if data.status != Status.INTERESTING:
self.exit_with(ExitReason.flaky)
while len(self.shrunk_examples) < len(self.interesting_examples):
target, self.last_data = min([
(k, v) for k, v in self.interesting_examples.items()
if k not in self.shrunk_examples],
key=lambda kv: (sort_key(kv[1].buffer), sort_key(repr(kv[0]))),
)
self.debug('Shrinking %r' % (target,))
assert self.last_data.interesting_origin == target
self.shrink()
self.shrunk_examples.add(target)
self.exit_with(ExitReason.finished)
def test_can_observe_draws():
class LoggingObserver(DataObserver):
def __init__(self):
self.log = []
def draw_bits(self, *args):
self.log.append(("draw",) + args)
def conclude_test(self, *args):
assert x.frozen
self.log.append(("concluded",) + args)
observer = LoggingObserver()
x = ConjectureData.for_buffer(hbytes([1, 2, 3]), observer=observer)
x.draw_bits(1)
x.draw_bits(7, forced=10)
x.draw_bits(8)
with pytest.raises(StopTest):
x.conclude_test(Status.INTERESTING, interesting_origin="neat")
assert observer.log == [
("draw", 1, False, 1),
("draw", 7, True, 10),
("draw", 8, False, 3),
("concluded", Status.INTERESTING, "neat"),
]
def test_can_get_odd_number_of_bits():
counts = Counter()
for i in range(256):
x = cu.getrandbits(ConjectureData.for_buffer([i]), 3)
assert 0 <= x <= 7
counts[x] += 1
assert len(set(counts.values())) == 1
def incorporate_new_buffer(self, buffer):
if buffer in self.seen:
return False
assert self.last_data.status == Status.INTERESTING
if (
self.settings.timeout > 0 and
time.time() >= self.start_time + self.settings.timeout
):
self.exit_reason = ExitReason.timeout
raise RunIsComplete()
buffer = buffer[:self.last_data.index]
if sort_key(buffer) >= sort_key(self.last_data.buffer):
return False
assert sort_key(buffer) <= sort_key(self.last_data.buffer)
data = ConjectureData.for_buffer(buffer)
self.test_function(data)
if self.consider_new_test_data(data):
self.shrinks += 1
self.last_data = data
if self.shrinks >= self.settings.max_shrinks:
self.exit_reason = ExitReason.max_shrinks
raise RunIsComplete()
self.last_data = data
self.changed += 1
return True
return False
def test_exhaustive_enumeration_of_partial_buffer():
seen = set()
def f(data):
k = data.draw_bytes(2)
assert k[1] == 0
assert k not in seen
seen.add(k)
seen_prefixes = set()
runner = ConjectureRunner(
f, settings=settings(database=None, max_examples=256, buffer_size=2),
random=Random(0),
)
with pytest.raises(RunIsComplete):
runner.cached_test_function(b'')
for _ in hrange(256):
p = runner.generate_novel_prefix()
assert p not in seen_prefixes
seen_prefixes.add(p)
data = ConjectureData.for_buffer(hbytes(p + hbytes(2)))
runner.test_function(data)
assert data.status == Status.VALID
node = 0
for b in data.buffer:
node = runner.tree[node][b]
assert node in runner.dead
assert len(seen) == 256
def test_saves_data_while_shrinking(monkeypatch):
key = b'hi there'
n = 5
db = InMemoryExampleDatabase()
assert list(db.fetch(key)) == []
seen = set()
monkeypatch.setattr(
ConjectureRunner, 'generate_new_examples',
lambda runner: runner.test_function(
ConjectureData.for_buffer([255] * 10)))
def f(data):
x = data.draw_bytes(10)
if sum(x) >= 2000 and len(seen) < n:
seen.add(hbytes(x))
if hbytes(x) in seen:
data.mark_interesting()
runner = ConjectureRunner(
f, settings=settings(database=db), database_key=key)
runner.run()
assert runner.interesting_examples
assert len(seen) == n
in_db = non_covering_examples(db)
assert in_db.issubset(seen)
assert in_db == seen
def cleanup():
files = glob('hypothesis-examples/raw/*')
Random().shuffle(files)
for f in files:
try:
with open(f, 'rb') as i:
data = ConjectureData.for_buffer(i.read())
except FileNotFoundError:
continue
with tempfile.TemporaryDirectory() as d:
prog = os.path.join(d, 'test.c')
try:
gen(data, prog)
except StopTest:
continue
name = os.path.basename(f)
if not is_interesting(prog):
print(f"Removing {name}")
rmf(os.path.join(raw, name))
rmf(os.path.join(programs, name + '.c'))
def test_prescreen_with_masked_byte_agrees_with_results(byte_a, byte_b):
def f(data):
data.draw_bits(2)
runner = ConjectureRunner(f)
data_a = ConjectureData.for_buffer(hbytes([byte_a]))
data_b = ConjectureData.for_buffer(hbytes([byte_b]))
runner.test_function(data_a)
prescreen_b = runner.prescreen_buffer(hbytes([byte_b]))
# Always test buffer B, to check whether the prescreen was correct.
runner.test_function(data_b)
# If the prescreen passed, then the buffers should be different.
# If it failed, then the buffers should be the same.
assert prescreen_b == (data_a.buffer != data_b.buffer)
def test_can_write_empty_string():
d = ConjectureData.for_buffer([1, 1, 1])
d.draw_bits(1)
d.write(hbytes())
d.draw_bits(1)
d.draw_bits(0, forced=0)
d.draw_bits(1)
assert d.buffer == hbytes([1, 1, 1])
def test_compact_blocks_during_generation():
d = ConjectureData.for_buffer([1] * 10)
for _ in hrange(5):
d.draw_bits(1)
assert len(list(d.blocks)) == 5
for _ in hrange(5):
d.draw_bits(1)
assert len(list(d.blocks)) == 10
def reuse_existing_examples(self):
"""If appropriate (we have a database and have been told to use it),
try to reload existing examples from the database.
If there are a lot we don't try all of them. We always try the
smallest example in the database (which is guaranteed to be the
last failure) and the largest (which is usually the seed example
which the last failure came from but we don't enforce that). We
then take a random sampling of the remainder and try those. Any
examples that are no longer interesting are cleared out.
"""
if self.has_existing_examples():
self.debug('Reusing examples from database')
# We have to do some careful juggling here. We have two database
# corpora: The primary and secondary. The primary corpus is a
# small set of minimized examples each of which has at one point
# demonstrated a distinct bug. We want to retry all of these.
# We also have a secondary corpus of examples that have at some
# point demonstrated interestingness (currently only ones that
# were previously non-minimal examples of a bug, but this will
# likely expand in future). These are a good source of potentially
# interesting examples, but there are a lot of them, so we down
# sample the secondary corpus to a more manageable size.
corpus = sorted(
self.settings.database.fetch(self.database_key),
key=sort_key
)
desired_size = max(2, ceil(0.1 * self.settings.max_examples))
for extra_key in [self.secondary_key, self.covering_key]:
if len(corpus) < desired_size:
extra_corpus = list(
self.settings.database.fetch(extra_key),
)
shortfall = desired_size - len(corpus)
if len(extra_corpus) <= shortfall:
extra = extra_corpus
else:
extra = self.random.sample(extra_corpus, shortfall)
extra.sort(key=sort_key)
corpus.extend(extra)
for existing in corpus:
self.last_data = ConjectureData.for_buffer(existing)
try:
self.test_function(self.last_data)
finally:
if self.last_data.status != Status.INTERESTING:
self.settings.database.delete(
self.database_key, existing)
self.settings.database.delete(
self.secondary_key, existing)
def test_detects_too_small_block_starts():
def f(data):
data.draw_bytes(8)
data.mark_interesting()
runner = ConjectureRunner(f, settings=settings(database=None))
r = ConjectureData.for_buffer(hbytes(8))
runner.test_function(r)
assert r.status == Status.INTERESTING
assert not runner.prescreen_buffer(hbytes([255] * 7))
def test_last_block_length():
d = ConjectureData.for_buffer([0] * 15)
with pytest.raises(IndexError):
d.blocks.last_block_length
for n in hrange(1, 5 + 1):
d.draw_bits(n * 8)
assert d.blocks.last_block_length == n
def test_will_error_on_find():
d = ConjectureData.for_buffer(hbytes(0))
d.is_find = True
with pytest.raises(InvalidArgument):
d.draw(st.data())
def test_can_note_non_str():
d = ConjectureData.for_buffer(hbytes(0))
x = object()
d.note(x)
assert repr(x) in d.output
def test_drawing_zero_bits_is_free():
x = ConjectureData.for_buffer(hbytes())
assert x.draw_bits(0) == 0
def test_sampler_does_not_draw_minimum_if_zero():
sampler = cu.Sampler([0, 2, 47])
assert sampler.sample(ConjectureData.for_buffer([0, 0])) != 0
def test_too_small_to_be_useful_coin():
assert not cu.biased_coin(ConjectureData.for_buffer([1]), 0.5**65)
def new_conjecture_data_for_buffer(self, buffer):
return ConjectureData.for_buffer(buffer,
observer=self.tree.new_observer())
def test_buffer_draws_as_self(buf):
x = ConjectureData.for_buffer(buf)
assert x.draw_bytes(len(buf)) == buf
def test_can_mark_interesting():
x = ConjectureData.for_buffer(hbytes())
with pytest.raises(StopTest):
x.mark_interesting()
assert x.frozen
assert x.status == Status.INTERESTING
def test_can_draw_zero_bytes():
x = ConjectureData.for_buffer(b"")
for _ in range(10):
assert x.draw_bytes(0) == b""
def test_can_mark_invalid():
x = ConjectureData.for_buffer(hbytes())
with pytest.raises(StopTest):
x.mark_invalid()
assert x.frozen
assert x.status == Status.INVALID
def test_sampler_shrinks():
sampler = cu.Sampler([4.0, 8.0, 1.0, 1.0, 0.5])
assert sampler.sample(ConjectureData.for_buffer([0] * 3)) == 0
def test_center_in_middle_above():
assert (
cu.integer_range(ConjectureData.for_buffer([1, 0]), lower=0, upper=10, center=5)
== 5
)
def test_can_note_str_as_non_repr():
d = ConjectureData.for_buffer(hbytes(0))
d.note(u"foo")
assert d.output == u"foo"
def test_integer_range_lower_equals_upper():
data = ConjectureData.for_buffer([0])
assert cu.integer_range(data, lower=0, upper=0, center=0) == 0
assert len(data.buffer) == 1
def test_notes_repr():
x = ConjectureData.for_buffer(b"")
x.note(b"hi")
assert repr(b"hi") in x.output
def test_can_double_freeze():
x = ConjectureData.for_buffer(b"hi")
x.freeze()
assert x.frozen
x.freeze()
assert x.frozen
def test_choice():
assert cu.choice(ConjectureData.for_buffer([1]), [1, 2, 3]) == 2
def is_failure_inducing(b):
try:
return strat._attempt_one_draw(
ConjectureData.for_buffer(b)) is None
except StopTest:
return False
def test_empty_strategy_is_invalid():
d = ConjectureData.for_buffer(hbytes(0))
with pytest.raises(StopTest):
d.draw(st.nothing())
assert d.status == Status.INVALID
def test_blocks_lengths():
d = ConjectureData.for_buffer(hbytes(7))
d.draw_bits(32)
d.draw_bits(16)
d.draw_bits(1)
assert [b.length for b in d.blocks] == [4, 2, 1]
def test_examples_support_negative_indexing():
d = ConjectureData.for_buffer(hbytes(2))
d.draw_bits(1)
d.draw_bits(1)
d.freeze()
assert d.examples[-1].length == 1
def test_cannot_draw_after_freeze():
x = ConjectureData.for_buffer(b"hi")
x.draw_bytes(1)
x.freeze()
with pytest.raises(Frozen):
x.draw_bytes(1)
def test_fractional_float():
assert cu.fractional_float(ConjectureData.for_buffer([0] * 8)) == 0.0
def test_events_are_noted():
d = ConjectureData.for_buffer(())
d.note_event("hello")
assert "hello" in d.events
def test_can_override_label():
d = ConjectureData.for_buffer(hbytes(2))
d.draw(st.booleans(), label=7)
d.freeze()
assert any(ex.label == 7 for ex in d.examples)
def test_biased_coin_can_be_forced():
assert cu.biased_coin(ConjectureData.for_buffer([0]), p=0.5, forced=True)
assert not cu.biased_coin(ConjectureData.for_buffer([1]), p=0.5, forced=False)
def test_assert_biased_coin_always_treats_one_as_true():
assert cu.biased_coin(ConjectureData.for_buffer([0, 1]), p=1.0 / 257)
def test_can_draw_arbitrary_fractions(p, b):
try:
cu.biased_coin(ConjectureData.for_buffer(b), p)
except StopTest:
reject()
def test_integer_range_center_default():
assert (
cu.integer_range(ConjectureData.for_buffer([0]), lower=0, upper=10, center=None)
== 0
)
def test_result_is_overrun():
d = ConjectureData.for_buffer(hbytes(0))
with pytest.raises(StopTest):
d.draw_bits(1)
assert d.as_result() is Overrun
