def test_database_uses_values_from_secondary_key():
key = b"key"
database = InMemoryExampleDatabase()
def f(data):
if data.draw_bits(8) >= 5:
data.mark_interesting()
else:
data.mark_invalid()
runner = ConjectureRunner(
f,
settings=settings(
max_examples=1,
buffer_size=1024,
database=database,
suppress_health_check=HealthCheck.all(),
),
database_key=key,
)
for i in range(10):
database.save(runner.secondary_key, hbytes([i]))
runner.test_function(ConjectureData.for_buffer(hbytes([10])))
assert runner.interesting_examples
assert len(set(database.fetch(key))) == 1
assert len(set(database.fetch(runner.secondary_key))) == 10
runner.clear_secondary_key()
assert len(set(database.fetch(key))) == 1
assert set(map(int_from_bytes,
database.fetch(runner.secondary_key))) == set(range(6, 11))
v, = runner.interesting_examples.values()
assert list(v.buffer) == [5]
def test_does_not_duplicate_elements():
def test(data):
data.target_observations["m"] = data.draw_bits(8)
runner = ConjectureRunner(
test,
settings=settings(TEST_SETTINGS, database=InMemoryExampleDatabase()),
database_key=b"stuff",
)
d1 = runner.cached_test_function([1]).as_result()
assert len(runner.pareto_front) == 1
# This can happen in practice if we e.g. reexecute a test because it has
# expired from the cache. It's easier just to test it directly though
# rather than simulate the failure mode.
is_pareto = runner.pareto_front.add(d1)
assert is_pareto
assert len(runner.pareto_front) == 1
def test_will_save_covering_examples():
tags = {}
def tagged(data):
b = hbytes(data.draw_bytes(4))
try:
tag = tags[b]
except KeyError:
if len(tags) < 10:
tag = len(tags)
tags[b] = tag
else:
tag = None
if tag is not None:
data.add_tag(tag)
db = InMemoryExampleDatabase()
runner = ConjectureRunner(tagged, settings=settings(
max_examples=100, max_shrinks=0, buffer_size=1024, database=db,
), database_key=b'stuff')
runner.run()
assert len(all_values(db)) == len(tags)
def test_fully_exhaust_base(monkeypatch):
"""In this test we generate all possible values for the first byte but
never get to the point where we exhaust the root of the tree."""
seed_random(0)
seen = set()
def f(data):
key = data.draw_bytes(2)
assert key not in seen
seen.add(key)
runner = ConjectureRunner(f, settings=settings(
max_examples=10000, max_shrinks=0, buffer_size=1024, database=None,
))
for c in hrange(256):
runner.cached_test_function([0, c])
assert 1 in runner.dead
runner.run()
def test_optimises_the_pareto_front():
def test(data):
count = 0
while data.draw_bits(8):
count += 1
data.target_observations[""] = min(count, 5)
runner = ConjectureRunner(
test,
settings=settings(max_examples=10000,
database=InMemoryExampleDatabase()),
database_key=b"stuff",
)
runner.cached_test_function([255] * 20 + [0])
runner.pareto_optimise()
assert len(runner.pareto_front) == 6
for i, data in enumerate(runner.pareto_front):
assert list(data.buffer) == [1] * i + [0]
def test_database_contains_only_pareto_front():
with deterministic_PRNG():
def test(data):
data.target_observations["1"] = data.draw_bits(4)
data.draw_bits(64)
data.target_observations["2"] = data.draw_bits(8)
db = InMemoryExampleDatabase()
runner = ConjectureRunner(
test,
settings=settings(
max_examples=500, database=db, suppress_health_check=HealthCheck.all()
),
database_key=b"stuff",
)
runner.run()
assert len(runner.pareto_front) <= 500
for v in runner.pareto_front:
assert v.status >= Status.VALID
assert len(db.data) == 1
(values,) = db.data.values()
values = set(values)
assert len(values) == len(runner.pareto_front)
for data in runner.pareto_front:
assert data.buffer in values
assert data in runner.pareto_front
for k in values:
assert runner.cached_test_function(k) in runner.pareto_front
def test_stops_optimising_once_interesting():
hi = 2**16 - 1
def test(data):
n = data.draw_bits(16)
data.target_observations[""] = n
if n < hi:
data.mark_interesting()
runner = ConjectureRunner(
test,
settings=settings(
max_examples=10000,
database=InMemoryExampleDatabase(),
),
database_key=b"stuff",
)
data = runner.cached_test_function([255] * 2)
assert data.status == Status.VALID
runner.pareto_optimise()
assert runner.call_count <= 20
assert runner.interesting_examples
def test_does_not_optimise_the_pareto_front_if_interesting():
def test(data):
n = data.draw_bits(8)
data.target_observations[""] = n
if n == 255:
data.mark_interesting()
runner = ConjectureRunner(
test,
settings=settings(
max_examples=10000,
database=InMemoryExampleDatabase(),
),
database_key=b"stuff",
)
runner.cached_test_function([0])
runner.pareto_optimise = None
runner.optimise_targets()
assert runner.interesting_examples
def test_replaces_all_dominated():
def test(data):
data.target_observations["m"] = 3 - data.draw_bits(2)
data.target_observations["n"] = 3 - data.draw_bits(2)
runner = ConjectureRunner(
test,
settings=settings(TEST_SETTINGS, database=InMemoryExampleDatabase()),
database_key=b"stuff",
)
d1 = runner.cached_test_function([0, 1]).as_result()
d2 = runner.cached_test_function([1, 0]).as_result()
assert len(runner.pareto_front) == 2
assert runner.pareto_front[0] == d1
assert runner.pareto_front[1] == d2
d3 = runner.cached_test_function([0, 0]).as_result()
assert len(runner.pareto_front) == 1
assert runner.pareto_front[0] == d3
def test_terminates_shrinks(n, monkeypatch):
db = InMemoryExampleDatabase()
def generate_new_examples(self):
self.cached_test_function([255] * 1000)
monkeypatch.setattr(
ConjectureRunner, "generate_new_examples", generate_new_examples
)
monkeypatch.setattr(engine_module, "MAX_SHRINKS", n)
runner = ConjectureRunner(
slow_shrinker(),
settings=settings(max_examples=5000, database=db),
random=Random(0),
database_key=b"key",
)
runner.run()
(last_data,) = runner.interesting_examples.values()
assert last_data.status == Status.INTERESTING
assert runner.shrinks == n
in_db = set(db.data[runner.secondary_key])
assert len(in_db) == n
def test_will_shrink_covering_examples():
best = [None]
replaced = []
def tagged(data):
b = hbytes(data.draw_bytes(4))
if any(b):
data.add_tag('nonzero')
if best[0] is None:
best[0] = b
elif b < best[0]:
replaced.append(best[0])
best[0] = b
db = InMemoryExampleDatabase()
runner = ConjectureRunner(tagged, settings=settings(
max_examples=100, phases=no_shrink, buffer_size=1024, database=db,
), database_key=b'stuff')
runner.run()
saved = set(all_values(db))
assert best[0] in saved
for r in replaced:
assert r not in saved
def test_terminates_shrinks(n, monkeypatch):
db = InMemoryExampleDatabase()
def generate_new_examples(self):
def draw_bytes(data, n):
return hbytes([255] * n)
self.test_function(ConjectureData(
draw_bytes=draw_bytes, max_length=self.settings.buffer_size))
monkeypatch.setattr(
ConjectureRunner, 'generate_new_examples', generate_new_examples)
runner = ConjectureRunner(slow_shrinker(), settings=settings(
max_examples=5000, max_shrinks=n,
database=db, timeout=unlimited,
), random=Random(0), database_key=b'key')
runner.run()
last_data, = runner.interesting_examples.values()
assert last_data.status == Status.INTERESTING
assert runner.shrinks == n
in_db = set(db.data[runner.secondary_key])
assert len(in_db) == n
def test_debug_data(capsys):
buf = [0, 1, 2]
def f(data):
for x in hbytes(buf):
if data.draw_bits(8) != x:
data.mark_invalid()
data.start_example(1)
data.stop_example()
data.mark_interesting()
runner = ConjectureRunner(f, settings=settings(
max_examples=5000, buffer_size=1024,
database=None, suppress_health_check=HealthCheck.all(),
verbosity=Verbosity.debug
))
runner.test_function(ConjectureData.for_buffer(buf))
runner.run()
out, _ = capsys.readouterr()
assert re.match(u'\\d+ bytes \\[.*\\] -> ', out)
assert 'INTERESTING' in out
assert '[]' not in out
def test_will_not_reset_the_tree_after_interesting_example(monkeypatch):
monkeypatch.setattr(engine_module, 'CACHE_RESET_FREQUENCY', 3)
def f(data):
if data.draw_bits(8) == 7:
data.mark_interesting()
with deterministic_PRNG():
runner = ConjectureRunner(f, settings=settings(
database=None, suppress_health_check=HealthCheck.all(),
))
def step(n):
runner.test_function(ConjectureData.for_buffer([n]))
step(0)
step(1)
assert len(runner.tree) > 1
step(7)
assert len(runner.tree) > 1
t = len(runner.tree)
runner.shrink_interesting_examples()
assert len(runner.tree) > t
def test_stops_after_max_examples_when_generating_more_bugs(examples):
seen = []
bad = [False, False]
def f(data):
seen.append(data.draw_bits(32))
# Rare, potentially multi-error conditions
if seen[-1] > 2**31:
bad[0] = True
raise ValueError
bad[1] = True
raise Exception
runner = ConjectureRunner(f,
settings=settings(max_examples=examples,
phases=[Phase.generate]))
try:
runner.run()
except Exception:
pass
# No matter what, whether examples is larger or smalller than MAX_TEST_CALLS,
# we stop looking at max_examples. (and re-run each failure for the traceback)
assert len(seen) <= examples + sum(bad)
def test_saves_data_while_shrinking():
key = b'hi there'
n = 5
db = ExampleDatabase(':memory:')
assert list(db.fetch(key)) == []
seen = set()
def f(data):
x = data.draw_bytes(512)
if sum(x) >= 5000 and len(seen) < n:
seen.add(hbytes(x))
if hbytes(x) in seen:
data.mark_interesting()
runner = ConjectureRunner(f,
settings=settings(database=db),
database_key=key)
runner.run()
assert runner.last_data.status == Status.INTERESTING
assert len(seen) == n
in_db = set(v for vs in db.data.values() for v in vs)
assert in_db.issubset(seen)
assert in_db == seen
def test_stops_after_max_iterations_when_reading():
key = b'key'
max_iterations = 1
db = ExampleDatabase(':memory:')
for i in range(10):
db.save(key, hbytes([i]))
seen = []
def f(data):
seen.append(data.draw_bytes(1))
data.mark_invalid()
runner = ConjectureRunner(f,
settings=settings(
max_examples=1,
max_iterations=max_iterations,
database=db,
),
database_key=key)
runner.run()
assert len(seen) == max_iterations
def test_saves_data_while_shrinking():
key = b'hi there'
n = 5
db = InMemoryExampleDatabase()
assert list(db.fetch(key)) == []
seen = set()
def f(data):
x = data.draw_bytes(512)
if sum(x) >= 5000 and len(seen) < n:
seen.add(hbytes(x))
if hbytes(x) in seen:
data.mark_interesting()
runner = ConjectureRunner(f,
settings=settings(database=db),
database_key=key)
runner.run()
assert runner.interesting_examples
assert len(seen) == n
in_db = non_covering_examples(db)
assert in_db.issubset(seen)
assert in_db == seen
def test_optimises_multiple_targets():
with deterministic_PRNG():
def test(data):
n = data.draw_bits(8)
m = data.draw_bits(8)
if n + m > 256:
data.mark_invalid()
data.target_observations["m"] = m
data.target_observations["n"] = n
data.target_observations["m + n"] = m + n
runner = ConjectureRunner(test, settings=TEST_SETTINGS)
runner.cached_test_function([200, 0])
runner.cached_test_function([0, 200])
try:
runner.optimise_targets()
except RunIsComplete:
pass
assert runner.best_observed_targets["m"] == 255
assert runner.best_observed_targets["n"] == 255
assert runner.best_observed_targets["m + n"] == 256
def test_branch_ending_in_write():
seen = set()
def tf(data):
count = 0
while data.draw_bits(1):
count += 1
if count > 1:
data.draw_bits(1, forced=0)
b = hbytes(data.buffer)
assert b not in seen
seen.add(b)
with deterministic_PRNG():
runner = ConjectureRunner(tf, settings=TEST_SETTINGS)
for _ in hrange(100):
prefix = runner.generate_novel_prefix()
attempt = prefix + hbytes(2)
data = runner.cached_test_function(attempt)
assert data.status == Status.VALID
assert attempt.startswith(data.buffer)
def test_stops_after_max_iterations_when_generating():
key = b'key'
value = b'rubber baby buggy bumpers'
max_iterations = 100
db = ExampleDatabase(':memory:')
db.save(key, value)
seen = []
def f(data):
seen.append(data.draw_bytes(len(value)))
data.mark_invalid()
runner = ConjectureRunner(f,
settings=settings(
max_examples=1,
max_iterations=max_iterations,
database=db,
),
database_key=key)
runner.run()
assert len(seen) == max_iterations
assert value in seen
def test_can_find_endpoints_of_a_range(lower, upper, score_up):
with deterministic_PRNG():
def test(data):
n = data.draw_bits(16)
if n < lower or n > upper:
data.mark_invalid()
if not score_up:
n = -n
data.target_observations["n"] = n
runner = ConjectureRunner(
test, settings=settings(TEST_SETTINGS, max_examples=1000)
)
runner.cached_test_function(int_to_bytes((lower + upper) // 2, 2))
try:
runner.optimise_targets()
except RunIsComplete:
pass
if score_up:
assert runner.best_observed_targets["n"] == upper
else:
assert runner.best_observed_targets["n"] == -lower
def test_can_patch_up_examples():
with deterministic_PRNG():
def test(data):
data.start_example(42)
m = data.draw_bits(6)
data.target_observations["m"] = m
for _ in range(m):
data.draw_bits(1)
data.stop_example()
for i in range(4):
if i != data.draw_bits(8):
data.mark_invalid()
runner = ConjectureRunner(test, settings=TEST_SETTINGS)
d = runner.cached_test_function([0, 0, 1, 2, 3, 4])
assert d.status == Status.VALID
try:
runner.optimise_targets()
except RunIsComplete:
pass
assert runner.best_observed_targets["m"] == 63
def test_can_reduce_poison_from_any_subtree(size, seed):
"""This test validates that we can minimize to any leaf node of a binary
tree, regardless of where in the tree the leaf is."""
random = Random(seed)
# Initially we create the minimal tree of size n, regardless of whether it
# is poisoned (which it won't be - the poison event essentially never
# happens when drawing uniformly at random).
# Choose p so that the expected size of the tree is equal to the desired
# size.
p = 1.0 / (2.0 - 1.0 / size)
strat = PoisonedTree(p)
def test_function(data):
v = data.draw(strat)
if len(v) >= size:
data.mark_interesting()
runner = ConjectureRunner(test_function,
random=random,
settings=settings(TEST_SETTINGS,
buffer_size=LOTS))
while not runner.interesting_examples:
runner.test_function(
runner.new_conjecture_data(lambda data, n: uniform(random, n)))
runner.shrink_interesting_examples()
data, = runner.interesting_examples.values()
assert len(ConjectureData.for_buffer(data.buffer).draw(strat)) == size
starts = [b.start for b in data.blocks if b.length == 2]
assert len(starts) % 2 == 0
for i in hrange(0, len(starts), 2):
# Now for each leaf position in the tree we try inserting a poison
# value artificially. Additionally, we add a marker to the end that
# must be preserved. The marker means that we are not allow to rely on
# discarding the end of the buffer to get the desired shrink.
u = starts[i]
marker = hbytes([1, 2, 3, 4])
def test_function_with_poison(data):
v = data.draw(strat)
m = data.draw_bytes(len(marker))
if POISON in v and m == marker:
data.mark_interesting()
runner = ConjectureRunner(test_function_with_poison,
random=random,
settings=TEST_SETTINGS)
runner.cached_test_function(data.buffer[:u] + hbytes([255]) * 4 +
data.buffer[u + 4:] + marker)
assert runner.interesting_examples
runner.shrink_interesting_examples()
shrunk, = runner.interesting_examples.values()
assert ConjectureData.for_buffer(
shrunk.buffer).draw(strat) == (POISON, )
def run_engine(self):
"""Run the test function many times, on database input and generated
input, using the Conjecture engine.
"""
# Tell pytest to omit the body of this function from tracebacks
__tracebackhide__ = True
try:
database_key = self.wrapped_test._hypothesis_internal_database_key
except AttributeError:
if global_force_seed is None:
database_key = function_digest(self.test)
else:
database_key = None
runner = ConjectureRunner(
self._execute_once_for_engine,
settings=self.settings,
random=self.random,
database_key=database_key,
)
# Use the Conjecture engine to run the test function many times
# on different inputs.
runner.run()
note_engine_for_statistics(runner)
if runner.call_count == 0:
return
if runner.interesting_examples:
self.falsifying_examples = sorted(
runner.interesting_examples.values(),
key=lambda d: sort_key(d.buffer),
reverse=True,
)
else:
if runner.valid_examples == 0:
raise Unsatisfiable(
"Unable to satisfy assumptions of hypothesis %s." %
(get_pretty_function_description(self.test), ))
if not self.falsifying_examples:
return
elif not self.settings.report_multiple_bugs:
# Pretend that we only found one failure, by discarding the others.
del self.falsifying_examples[:-1]
# The engine found one or more failures, so we need to reproduce and
# report them.
self.failed_normally = True
flaky = 0
for falsifying_example in self.falsifying_examples:
info = falsifying_example.extra_information
ran_example = ConjectureData.for_buffer(falsifying_example.buffer)
self.__was_flaky = False
assert info.__expected_exception is not None
try:
self.execute_once(
ran_example,
print_example=not self.is_find,
is_final=True,
expected_failure=(
info.__expected_exception,
info.__expected_traceback,
),
)
except (UnsatisfiedAssumption, StopTest):
report(traceback.format_exc())
self.__flaky(
"Unreliable assumption: An example which satisfied "
"assumptions on the first run now fails it.")
except BaseException as e:
if len(self.falsifying_examples) <= 1:
# There is only one failure, so we can report it by raising
# it directly.
raise
# We are reporting multiple failures, so we need to manually
# print each exception's stack trace and information.
tb = get_trimmed_traceback()
report("".join(traceback.format_exception(type(e), e, tb)))
finally: # pragma: no cover
# Mostly useful for ``find`` and ensuring that objects that
# hold on to a reference to ``data`` know that it's now been
# finished and they shouldn't attempt to draw more data from
# it.
ran_example.freeze()
# This section is in fact entirely covered by the tests in
# test_reproduce_failure, but it seems to trigger a lovely set
# of coverage bugs: The branches show up as uncovered (despite
# definitely being covered - you can add an assert False else
# branch to verify this and see it fail - and additionally the
# second branch still complains about lack of coverage even if
# you add a pragma: no cover to it!
# See https://bitbucket.org/ned/coveragepy/issues/623/
if self.settings.print_blob:
report(("\nYou can reproduce this example by temporarily "
"adding @reproduce_failure(%r, %r) as a decorator "
"on your test case") %
(__version__,
encode_failure(falsifying_example.buffer)))
if self.__was_flaky:
flaky += 1
# If we only have one example then we should have raised an error or
# flaky prior to this point.
assert len(self.falsifying_examples) > 1
if flaky > 0:
raise Flaky(
("Hypothesis found %d distinct failures, but %d of them "
"exhibited some sort of flaky behaviour.") %
(len(self.falsifying_examples), flaky))
else:
raise MultipleFailures(("Hypothesis found %d distinct failures.") %
(len(self.falsifying_examples)))
def find(
specifier, # type: SearchStrategy
condition, # type: Callable[[Any], bool]
settings=None, # type: Settings
random=None, # type: Any
database_key=None, # type: bytes
):
# type: (...) -> Any
"""Returns the minimal example from the given strategy ``specifier`` that
matches the predicate function ``condition``."""
if settings is None:
settings = Settings(max_examples=2000)
settings = Settings(settings, suppress_health_check=HealthCheck.all())
if database_key is None and settings.database is not None:
database_key = function_digest(condition)
if not isinstance(specifier, SearchStrategy):
raise InvalidArgument("Expected SearchStrategy but got %r of type %s" %
(specifier, type(specifier).__name__))
specifier.validate()
search = specifier
random = random or new_random()
successful_examples = [0]
last_data = [None]
last_repr = [None]
def template_condition(data):
with BuildContext(data):
try:
data.is_find = True
with deterministic_PRNG():
result = data.draw(search)
data.note(result)
success = condition(result)
except UnsatisfiedAssumption:
data.mark_invalid()
if success:
successful_examples[0] += 1
if settings.verbosity >= Verbosity.verbose:
if not successful_examples[0]:
report(u"Tried non-satisfying example %s" %
(nicerepr(result), ))
elif success:
if successful_examples[0] == 1:
last_repr[0] = nicerepr(result)
report(u"Found satisfying example %s" % (last_repr[0], ))
last_data[0] = data
elif (sort_key(hbytes(data.buffer)) < sort_key(
last_data[0].buffer)
) and nicerepr(result) != last_repr[0]:
last_repr[0] = nicerepr(result)
report(u"Shrunk example to %s" % (last_repr[0], ))
last_data[0] = data
if success and not data.frozen:
data.mark_interesting()
runner = ConjectureRunner(template_condition,
settings=settings,
random=random,
database_key=database_key)
runner.run()
note_engine_for_statistics(runner)
if runner.interesting_examples:
data = ConjectureData.for_buffer(
list(runner.interesting_examples.values())[0].buffer)
with BuildContext(data):
with deterministic_PRNG():
return data.draw(search)
if runner.valid_examples == 0 and (runner.exit_reason !=
ExitReason.finished):
raise Unsatisfiable("Unable to satisfy assumptions of %s." %
(get_pretty_function_description(condition), ))
raise NoSuchExample(get_pretty_function_description(condition))
def run(self):
# Tell pytest to omit the body of this function from tracebacks
__tracebackhide__ = True
if global_force_seed is None:
database_key = function_digest(self.test)
else:
database_key = None
runner = ConjectureRunner(
self.evaluate_test_data,
settings=self.settings,
random=self.random,
database_key=database_key,
)
try:
runner.run()
finally:
self.used_examples_from_database = runner.used_examples_from_database
note_engine_for_statistics(runner)
self.used_examples_from_database = runner.used_examples_from_database
if runner.call_count == 0:
return
if runner.interesting_examples:
self.falsifying_examples = sorted(
[d for d in runner.interesting_examples.values()],
key=lambda d: sort_key(d.buffer),
reverse=True,
)
else:
if runner.valid_examples == 0:
raise Unsatisfiable(
"Unable to satisfy assumptions of hypothesis %s." %
(get_pretty_function_description(self.test), ))
if not self.falsifying_examples:
return
self.failed_normally = True
flaky = 0
for falsifying_example in self.falsifying_examples:
ran_example = ConjectureData.for_buffer(falsifying_example.buffer)
self.__was_flaky = False
assert falsifying_example.__expected_exception is not None
try:
self.execute(
ran_example,
print_example=True,
is_final=True,
expected_failure=(
falsifying_example.__expected_exception,
falsifying_example.__expected_traceback,
),
)
except (UnsatisfiedAssumption, StopTest):
report(traceback.format_exc())
self.__flaky(
"Unreliable assumption: An example which satisfied "
"assumptions on the first run now fails it.")
except BaseException as e:
if len(self.falsifying_examples) <= 1:
raise
tb = get_trimmed_traceback()
report("".join(traceback.format_exception(type(e), e, tb)))
finally: # pragma: no cover
# This section is in fact entirely covered by the tests in
# test_reproduce_failure, but it seems to trigger a lovely set
# of coverage bugs: The branches show up as uncovered (despite
# definitely being covered - you can add an assert False else
# branch to verify this and see it fail - and additionally the
# second branch still complains about lack of coverage even if
# you add a pragma: no cover to it!
# See https://bitbucket.org/ned/coveragepy/issues/623/
if self.settings.print_blob is not PrintSettings.NEVER:
failure_blob = encode_failure(falsifying_example.buffer)
# Have to use the example we actually ran, not the original
# falsifying example! Otherwise we won't catch problems
# where the repr of the generated example doesn't parse.
can_use_repr = ran_example.can_reproduce_example_from_repr
if self.settings.print_blob is PrintSettings.ALWAYS or (
self.settings.print_blob is PrintSettings.INFER
and self.settings.verbosity >= Verbosity.normal
and not can_use_repr and len(failure_blob) < 200):
report((
"\nYou can reproduce this example by temporarily "
"adding @reproduce_failure(%r, %r) as a decorator "
"on your test case") % (__version__, failure_blob))
if self.__was_flaky:
flaky += 1
# If we only have one example then we should have raised an error or
# flaky prior to this point.
assert len(self.falsifying_examples) > 1
if flaky > 0:
raise Flaky(
("Hypothesis found %d distinct failures, but %d of them "
"exhibited some sort of flaky behaviour.") %
(len(self.falsifying_examples), flaky))
else:
raise MultipleFailures(("Hypothesis found %d distinct failures.") %
(len(self.falsifying_examples)))
def normalize(
base_name,
test_function,
*,
required_successes=100,
allowed_to_update=False,
max_dfas=10,
):
"""Attempt to ensure that this test function successfully normalizes - i.e.
whenever it declares a test case to be interesting, we are able
to shrink that to the same interesting test case (which logically should
be the shortlex minimal interesting test case, though we may not be able
to detect if it is).
Will run until we have seen ``required_successes`` many interesting test
cases in a row normalize to the same value.
If ``allowed_to_update`` is True, whenever we fail to normalize we will
learn a new DFA-based shrink pass that allows us to make progress. Any
learned DFAs will be written back into the learned DFA file at the end
of this function. If ``allowed_to_update`` is False, this will raise an
error as soon as it encounters a failure to normalize.
Additionally, if more than ``max_dfas` DFAs are required to normalize
this test function, this function will raise an error - it's essentially
designed for small patches that other shrink passes don't cover, and
if it's learning too many patches then you need a better shrink pass
than this can provide.
"""
# Need import inside the function to avoid circular imports
from hypothesis.internal.conjecture.engine import BUFFER_SIZE, ConjectureRunner
runner = ConjectureRunner(
test_function,
settings=settings(database=None,
suppress_health_check=HealthCheck.all()),
ignore_limits=True,
)
seen = set()
dfas_added = 0
found_interesting = False
consecutive_successes = 0
failures_to_find_interesting = 0
while consecutive_successes < required_successes:
attempt = runner.cached_test_function(b"", extend=BUFFER_SIZE)
if attempt.status < Status.INTERESTING:
failures_to_find_interesting += 1
assert (found_interesting or failures_to_find_interesting <= 1000
), "Test function seems to have no interesting test cases"
continue
found_interesting = True
target = attempt.interesting_origin
def shrinking_predicate(d):
return d.status == Status.INTERESTING and d.interesting_origin == target
if target not in seen:
seen.add(target)
runner.shrink(attempt, shrinking_predicate)
continue
previous = fully_shrink(runner, runner.interesting_examples[target],
shrinking_predicate)
current = fully_shrink(runner, attempt, shrinking_predicate)
if current.buffer == previous.buffer:
consecutive_successes += 1
continue
consecutive_successes = 0
if not allowed_to_update:
raise FailedToNormalise(
"Shrinker failed to normalize %r to %r and we are not allowed to learn new DFAs."
% (previous.buffer, current.buffer))
if dfas_added >= max_dfas:
raise FailedToNormalise(
"Test function is too hard to learn: Added %d DFAs and still not done."
% (dfas_added, ))
dfas_added += 1
new_dfa = learn_a_new_dfa(runner, previous.buffer, current.buffer,
shrinking_predicate)
name = (base_name + "-" +
hashlib.sha256(repr(new_dfa).encode("utf-8")).hexdigest()[:10])
# If there is a name collision this DFA should already be being
# used for shrinking, so we should have already been able to shrink
# v further.
assert name not in SHRINKING_DFAS
SHRINKING_DFAS[name] = new_dfa
if dfas_added > 0:
# We've learned one or more DFAs in the course of normalising, so now
# we update the file to record those for posterity.
update_learned_dfas()
def learner_for(strategy):
"""Returns an LStar learner that predicts whether a buffer
corresponds to a discard free choice sequence leading to
a valid value for this strategy."""
try:
return LEARNERS[strategy]
except KeyError:
pass
def test_function(data):
data.draw(strategy)
data.mark_interesting()
runner = ConjectureRunner(
test_function,
settings=settings(
database=None,
verbosity=Verbosity.quiet,
suppress_health_check=HealthCheck.all(),
),
random=Random(0),
ignore_limits=True,
)
def predicate(s):
result = runner.cached_test_function(s)
if result.status < Status.VALID:
return False
if result.has_discards:
return False
return result.buffer == s
learner = LStar(predicate)
runner.run()
(v,) = runner.interesting_examples.values()
# We make sure the learner has properly learned small examples.
# This is all fairly ad hoc but is mostly designed to get it
# to understand what the smallest example is and avoid any
# loops at the beginning of the DFA that don't really exist.
learner.learn(v.buffer)
for n in [1, 2, 3]:
for _ in range(5):
learner.learn(uniform(runner.random, n) + v.buffer)
prev = -1
while learner.generation != prev:
prev = learner.generation
for _ in range(10):
s = uniform(runner.random, len(v.buffer)) + bytes(BUFFER_SIZE)
learner.learn(s)
data = runner.cached_test_function(s)
if data.status >= Status.VALID:
learner.learn(data.buffer)
LEARNERS[strategy] = learner
return learner
def test_overruns_if_not_enough_bytes_for_block():
runner = ConjectureRunner(lambda data: data.draw_bytes(2),
settings=TEST_SETTINGS,
random=Random(0))
runner.cached_test_function(b"\0\0")
assert runner.tree.rewrite(b"\0")[1] == Status.OVERRUN
