def run(data):
if not hasattr(data, 'can_reproduce_example_from_repr'):
data.can_reproduce_example_from_repr = True
with local_settings(self.settings):
with BuildContext(data, is_final=is_final):
with deterministic_PRNG():
args, kwargs = data.draw(self.search_strategy)
if expected_failure is not None:
text_repr[0] = arg_string(test, args, kwargs)
if print_example:
example = '%s(%s)' % (
test.__name__, arg_string(test, args, kwargs))
try:
ast.parse(example)
except SyntaxError:
data.can_reproduce_example_from_repr = False
report('Falsifying example: %s' % (example,))
elif current_verbosity() >= Verbosity.verbose:
report(
lambda: 'Trying example: %s(%s)' % (
test.__name__, arg_string(test, args, kwargs)))
with deterministic_PRNG():
return test(*args, **kwargs)
def template_condition(data):
with BuildContext(data):
try:
data.is_find = True
with deterministic_PRNG():
result = data.draw(search)
data.note(result)
success = condition(result)
except UnsatisfiedAssumption:
data.mark_invalid()
if success:
successful_examples[0] += 1
if settings.verbosity >= Verbosity.verbose:
if not successful_examples[0]:
report(
u'Tried non-satisfying example %s' % (nicerepr(result),))
elif success:
if successful_examples[0] == 1:
last_repr[0] = nicerepr(result)
report(u'Found satisfying example %s' % (last_repr[0],))
last_data[0] = data
elif (
sort_key(hbytes(data.buffer)) <
sort_key(last_data[0].buffer)
) and nicerepr(result) != last_repr[0]:
last_repr[0] = nicerepr(result)
report(u'Shrunk example to %s' % (last_repr[0],))
last_data[0] = data
if success and not data.frozen:
data.mark_interesting()
def run_to_data(f):
with deterministic_PRNG():
runner = ConjectureRunner(f, settings=TEST_SETTINGS)
runner.run()
assert runner.interesting_examples
last_data, = runner.interesting_examples.values()
return last_data
def test_exhaust_space():
with deterministic_PRNG():
runner = ConjectureRunner(
lambda data: data.draw_bits(1), settings=TEST_SETTINGS
)
runner.run()
assert runner.tree.is_exhausted
assert runner.valid_examples == 2
def test_find_does_not_pollute_state():
with deterministic_PRNG():
find(st.random_module(), lambda r: True)
state_a = random.getstate()
find(st.random_module(), lambda r: True)
state_b = random.getstate()
assert state_a != state_b
def run_to_buffer(f):
with deterministic_PRNG():
runner = ConjectureRunner(f, settings=settings(
max_examples=5000, buffer_size=1024,
database=None, suppress_health_check=HealthCheck.all(),
))
runner.run()
assert runner.interesting_examples
last_data, = runner.interesting_examples.values()
return hbytes(last_data.buffer)
def accept(f):
with deterministic_PRNG():
runner = ConjectureRunner(f, settings=settings(
max_examples=5000, buffer_size=1024,
database=None, suppress_health_check=HealthCheck.all(),
))
runner.test_function(ConjectureData.for_buffer(start))
assert runner.interesting_examples
last_data, = runner.interesting_examples.values()
return runner.new_shrinker(
last_data, lambda d: d.status == Status.INTERESTING
)
def test_one_dead_branch():
with deterministic_PRNG():
seen = set()
@run_to_buffer
def x(data):
i = data.draw_bytes(1)[0]
if i > 0:
data.mark_invalid()
i = data.draw_bytes(1)[0]
if len(seen) < 255:
seen.add(i)
elif i not in seen:
data.mark_interesting()
def test_given_does_not_pollute_state():
with deterministic_PRNG():
@given(st.random_module())
def test(r):
pass
test()
state_a = random.getstate()
test()
state_b = random.getstate()
assert state_a != state_b
def test_large_initial_write():
big = hbytes(b'\xff') * 512
def f(data):
data.write(big)
data.draw_bits(63)
with deterministic_PRNG():
runner = ConjectureRunner(f, settings=settings(
max_examples=5000, buffer_size=1024,
database=None, suppress_health_check=HealthCheck.all(),
))
runner.run()
assert runner.exit_reason == ExitReason.finished
def test_cached_test_function_returns_right_value():
count = [0]
def tf(data):
count[0] += 1
data.draw_bits(2)
data.mark_interesting()
with deterministic_PRNG():
runner = ConjectureRunner(tf, settings=TEST_SETTINGS)
for _ in hrange(2):
for b in (b"\0", b"\1"):
d = runner.cached_test_function(b)
assert d.status == Status.INTERESTING
assert d.buffer == b
assert count[0] == 2
def test_cached_test_function_does_not_reinvoke_on_prefix():
call_count = [0]
def test_function(data):
call_count[0] += 1
data.draw_bits(8)
data.write(hbytes([7]))
data.draw_bits(8)
with deterministic_PRNG():
runner = ConjectureRunner(test_function, settings=TEST_SETTINGS)
data = runner.cached_test_function(hbytes(3))
assert data.status == Status.VALID
for n in [2, 1, 0]:
prefix_data = runner.cached_test_function(hbytes(n))
assert prefix_data is Overrun
assert call_count[0] == 1
def test_populates_the_pareto_front():
with deterministic_PRNG():
def test(data):
data.target_observations[""] = data.draw_bits(4)
runner = ConjectureRunner(
test,
settings=settings(
max_examples=5000,
database=InMemoryExampleDatabase(),
suppress_health_check=HealthCheck.all(),
),
database_key=b"stuff",
)
runner.run()
assert len(runner.pareto_front) == 2**4
def test_uses_tags_in_calculating_pareto_front():
with deterministic_PRNG():
def test(data):
if data.draw_bits(1):
data.start_example(11)
data.draw_bits(8)
data.stop_example()
runner = ConjectureRunner(
test,
settings=settings(max_examples=10,
database=InMemoryExampleDatabase()),
database_key=b"stuff",
)
runner.run()
assert len(runner.pareto_front) == 2
def test_pareto_front_contains_different_interesting_reasons():
with deterministic_PRNG():
def test(data):
data.mark_interesting(data.draw_bits(4))
runner = ConjectureRunner(
test,
settings=settings(
max_examples=5000,
database=InMemoryExampleDatabase(),
suppress_health_check=HealthCheck.all(),
),
database_key=b"stuff",
)
runner.run()
assert len(runner.pareto_front) == 2 ** 4
def test_will_reset_the_tree_as_it_goes(monkeypatch):
monkeypatch.setattr(engine_module, 'CACHE_RESET_FREQUENCY', 3)
def f(data):
data.draw_bits(8)
with deterministic_PRNG():
runner = ConjectureRunner(f, settings=settings(
database=None, suppress_health_check=HealthCheck.all(),
))
def step(n):
runner.test_function(ConjectureData.for_buffer([n]))
step(0)
step(1)
assert len(runner.tree[0]) > 1
step(2)
assert len(runner.tree[0]) == 1
def test_large_initial_write():
big = hbytes(b"\xff") * 512
def f(data):
data.write(big)
data.draw_bits(63)
with deterministic_PRNG():
runner = ConjectureRunner(
f,
settings=settings(
max_examples=5000,
buffer_size=1024,
database=None,
suppress_health_check=HealthCheck.all(),
),
)
runner.run()
assert runner.exit_reason == ExitReason.finished
def run(data):
with local_settings(self.settings):
with deterministic_PRNG():
with BuildContext(data, is_final=is_final):
args, kwargs = data.draw(self.search_strategy)
if expected_failure is not None:
text_repr[0] = arg_string(test, args, kwargs)
if print_example:
example = "%s(%s)" % (
test.__name__,
arg_string(test, args, kwargs),
)
report("Falsifying example: %s" % (example,))
elif current_verbosity() >= Verbosity.verbose:
report(
lambda: "Trying example: %s(%s)"
% (test.__name__, arg_string(test, args, kwargs))
)
return test(*args, **kwargs)
def test_does_not_shrink_multiple_bugs_when_told_not_to():
def test(data):
m = data.draw_bits(8)
n = data.draw_bits(8)
if m > 0:
data.mark_interesting(1)
if n > 5:
data.mark_interesting(2)
with deterministic_PRNG():
runner = ConjectureRunner(
test, settings=settings(TEST_SETTINGS, report_multiple_bugs=False)
)
runner.cached_test_function([255, 255])
runner.shrink_interesting_examples()
results = {d.buffer for d in runner.interesting_examples.values()}
assert len(results.intersection({bytes([0, 1]), bytes([1, 0])})) == 1
def test_optimiser_when_test_grows_buffer_to_overflow():
with deterministic_PRNG():
with buffer_size_limit(2):
def test(data):
m = data.draw_bits(8)
data.target_observations["m"] = m
if m > 100:
data.draw_bits(64)
data.mark_invalid()
runner = ConjectureRunner(test, settings=TEST_SETTINGS)
runner.cached_test_function(bytes(10))
try:
runner.optimise_targets()
except RunIsComplete:
pass
assert runner.best_observed_targets["m"] == 100
def test_discards_kill_branches():
starts = set()
with deterministic_PRNG():
def test(data):
assert runner.call_count <= 256
while True:
data.start_example(1)
b = data.draw_bits(8)
data.stop_example(b != 0)
if len(data.buffer) == 1:
s = bytes(data.buffer)
assert s not in starts
starts.add(s)
if b == 0:
break
runner = ConjectureRunner(test, settings=SMALL_COUNT_SETTINGS)
runner.run()
assert runner.call_count == 256
def test_database_contains_only_pareto_front():
with deterministic_PRNG():
def test(data):
data.target_observations["1"] = data.draw_bits(4)
data.draw_bits(64)
data.target_observations["2"] = data.draw_bits(8)
db = InMemoryExampleDatabase()
runner = ConjectureRunner(
test,
settings=settings(
max_examples=500,
database=db,
suppress_health_check=HealthCheck.all(),
),
database_key=b"stuff",
)
runner.run()
assert len(runner.pareto_front) <= 500
for v in runner.pareto_front:
assert v.status >= Status.VALID
assert len(db.data) == 1
(values, ) = db.data.values()
values = set(values)
assert len(values) == len(runner.pareto_front)
for data in runner.pareto_front:
assert data.buffer in values
assert data in runner.pareto_front
for k in values:
assert runner.cached_test_function(k) in runner.pareto_front
def test_does_not_trigger_health_check_on_simple_strategies(monkeypatch):
existing_draw_bits = ConjectureData.draw_bits
# We need to make drawing data artificially slow in order to trigger this
# effect. This isn't actually slow because time is fake in our CI, but
# we need it to pretend to be.
def draw_bits(self, n, forced=None):
time.sleep(0.001)
return existing_draw_bits(self, n, forced)
monkeypatch.setattr(ConjectureData, "draw_bits", draw_bits)
with deterministic_PRNG():
for _ in range(100):
# Setting max_examples=11 ensures we have enough examples for the
# health checks to finish running, but cuts the generation short
# after that point to allow this test to run in reasonable time.
@settings(database=None, max_examples=11, phases=[Phase.generate])
@given(st.binary())
def test(b):
pass
test()
def test_will_not_reset_the_tree_after_interesting_example(monkeypatch):
monkeypatch.setattr(engine_module, 'CACHE_RESET_FREQUENCY', 3)
def f(data):
if data.draw_bits(8) == 7:
data.mark_interesting()
with deterministic_PRNG():
runner = ConjectureRunner(f, settings=settings(
database=None, suppress_health_check=HealthCheck.all(),
))
def step(n):
runner.test_function(ConjectureData.for_buffer([n]))
step(0)
step(1)
assert len(runner.tree) > 1
step(7)
assert len(runner.tree) > 1
t = len(runner.tree)
runner.shrink_interesting_examples()
assert len(runner.tree) > t
def test_branch_ending_in_write():
seen = set()
def tf(data):
count = 0
while data.draw_bits(1):
count += 1
if count > 1:
data.draw_bits(1, forced=0)
b = hbytes(data.buffer)
assert b not in seen
seen.add(b)
with deterministic_PRNG():
runner = ConjectureRunner(tf, settings=TEST_SETTINGS)
for _ in hrange(100):
prefix = runner.generate_novel_prefix()
attempt = prefix + hbytes(2)
data = runner.cached_test_function(attempt)
assert data.status == Status.VALID
assert attempt.startswith(data.buffer)
def test_can_patch_up_examples():
with deterministic_PRNG():
def test(data):
data.start_example(42)
m = data.draw_bits(6)
data.target_observations["m"] = m
for _ in range(m):
data.draw_bits(1)
data.stop_example()
for i in range(4):
if i != data.draw_bits(8):
data.mark_invalid()
runner = ConjectureRunner(test, settings=TEST_SETTINGS)
d = runner.cached_test_function([0, 0, 1, 2, 3, 4])
assert d.status == Status.VALID
try:
runner.optimise_targets()
except RunIsComplete:
pass
assert runner.best_observed_targets["m"] == 63
def test_branch_ending_in_write():
seen = set()
def tf(data):
count = 0
while data.draw_bits(1):
count += 1
if count > 1:
data.draw_bits(1, forced=0)
b = hbytes(data.buffer)
assert b not in seen
seen.add(b)
with deterministic_PRNG():
runner = ConjectureRunner(tf, settings=TEST_SETTINGS)
for _ in hrange(100):
prefix = runner.generate_novel_prefix()
attempt = prefix + hbytes(2)
data = runner.cached_test_function(attempt)
assert data.status == Status.VALID
assert attempt.startswith(data.buffer)
def test_optimises_multiple_targets():
with deterministic_PRNG():
def test(data):
n = data.draw_bits(8)
m = data.draw_bits(8)
if n + m > 256:
data.mark_invalid()
data.target_observations["m"] = m
data.target_observations["n"] = n
data.target_observations["m + n"] = m + n
runner = ConjectureRunner(test, settings=TEST_SETTINGS)
runner.cached_test_function([200, 0])
runner.cached_test_function([0, 200])
try:
runner.optimise_targets()
except RunIsComplete:
pass
assert runner.best_observed_targets["m"] == 255
assert runner.best_observed_targets["n"] == 255
assert runner.best_observed_targets["m + n"] == 256
def run(data):
if not hasattr(data, "can_reproduce_example_from_repr"):
data.can_reproduce_example_from_repr = True
with local_settings(self.settings):
with BuildContext(data, is_final=is_final):
with deterministic_PRNG():
args, kwargs = data.draw(self.search_strategy)
if expected_failure is not None:
text_repr[0] = arg_string(test, args, kwargs)
if print_example:
example = "%s(%s)" % (
test.__name__,
arg_string(test, args, kwargs),
)
try:
ast.parse(example)
except SyntaxError:
data.can_reproduce_example_from_repr = False
report("Falsifying example: %s" % (example, ))
elif current_verbosity() >= Verbosity.verbose:
report(lambda: "Trying example: %s(%s)" % (
test.__name__, arg_string(test, args, kwargs)))
return test(*args, **kwargs)
def test_can_find_endpoints_of_a_range(lower, upper, score_up):
with deterministic_PRNG():
def test(data):
n = data.draw_bits(16)
if n < lower or n > upper:
data.mark_invalid()
if not score_up:
n = -n
data.target_observations["n"] = n
runner = ConjectureRunner(
test, settings=settings(TEST_SETTINGS, max_examples=1000)
)
runner.cached_test_function(int_to_bytes((lower + upper) // 2, 2))
try:
runner.optimise_targets()
except RunIsComplete:
pass
if score_up:
assert runner.best_observed_targets["n"] == upper
else:
assert runner.best_observed_targets["n"] == -lower
def run(data):
# Set up dynamic context needed by a single test run.
with local_settings(self.settings):
with deterministic_PRNG():
with BuildContext(data, is_final=is_final):
# Generate all arguments to the test function.
args, kwargs = data.draw(self.search_strategy)
if expected_failure is not None:
text_repr[0] = arg_string(test, args, kwargs)
if print_example or current_verbosity(
) >= Verbosity.verbose:
output = CUnicodeIO()
printer = RepresentationPrinter(output)
if print_example:
printer.text("Falsifying example:")
else:
printer.text("Trying example:")
if self.print_given_args:
printer.text(" ")
printer.text(test.__name__)
with printer.group(indent=4,
open="(",
close=""):
printer.break_()
for v in args:
printer.pretty(v)
# We add a comma unconditionally because
# generated arguments will always be
# kwargs, so there will always be more
# to come.
printer.text(",")
printer.breakable()
# We need to make sure to print these in the argument order for
# Python 2 and older versionf of Python 3.5. In modern versions
# this isn't an issue because kwargs is ordered.
arg_order = {
v: i
for i, v in enumerate(
getfullargspec(self.test).args)
}
for i, (k, v) in enumerate(
sorted(
kwargs.items(),
key=lambda t: (
arg_order.get(
t[0], float("inf")),
t[0],
),
)):
printer.text(k)
printer.text("=")
printer.pretty(v)
printer.text(",")
if i + 1 < len(kwargs):
printer.breakable()
printer.break_()
printer.text(")")
printer.flush()
report(output.getvalue())
return test(*args, **kwargs)
def find(
specifier, # type: SearchStrategy
condition, # type: Callable[[Any], bool]
settings=None, # type: Settings
random=None, # type: Any
database_key=None, # type: bytes
):
# type: (...) -> Any
"""Returns the minimal example from the given strategy ``specifier`` that
matches the predicate function ``condition``."""
if settings is None:
settings = Settings(max_examples=2000)
settings = Settings(settings, suppress_health_check=HealthCheck.all())
if database_key is None and settings.database is not None:
database_key = function_digest(condition)
if not isinstance(specifier, SearchStrategy):
raise InvalidArgument("Expected SearchStrategy but got %r of type %s" %
(specifier, type(specifier).__name__))
specifier.validate()
search = specifier
random = random or new_random()
successful_examples = [0]
last_data = [None]
last_repr = [None]
def template_condition(data):
with BuildContext(data):
try:
data.is_find = True
with deterministic_PRNG():
result = data.draw(search)
data.note(result)
success = condition(result)
except UnsatisfiedAssumption:
data.mark_invalid()
if success:
successful_examples[0] += 1
if settings.verbosity >= Verbosity.verbose:
if not successful_examples[0]:
report(u"Tried non-satisfying example %s" %
(nicerepr(result), ))
elif success:
if successful_examples[0] == 1:
last_repr[0] = nicerepr(result)
report(u"Found satisfying example %s" % (last_repr[0], ))
last_data[0] = data
elif (sort_key(hbytes(data.buffer)) < sort_key(
last_data[0].buffer)
) and nicerepr(result) != last_repr[0]:
last_repr[0] = nicerepr(result)
report(u"Shrunk example to %s" % (last_repr[0], ))
last_data[0] = data
if success and not data.frozen:
data.mark_interesting()
runner = ConjectureRunner(template_condition,
settings=settings,
random=random,
database_key=database_key)
runner.run()
note_engine_for_statistics(runner)
if runner.interesting_examples:
data = ConjectureData.for_buffer(
list(runner.interesting_examples.values())[0].buffer)
with BuildContext(data):
with deterministic_PRNG():
return data.draw(search)
if runner.valid_examples == 0 and (runner.exit_reason !=
ExitReason.finished):
raise Unsatisfiable("Unable to satisfy assumptions of %s." %
(get_pretty_function_description(condition), ))
raise NoSuchExample(get_pretty_function_description(condition))
def test_can_run_with_no_db():
with deterministic_PRNG():
with raises(AssertionError):
run_state_machine_as_test(DepthMachine, settings=Settings(database=None))
def find(
specifier, # type: SearchStrategy
condition, # type: Callable[[Any], bool]
settings=None, # type: Settings
random=None, # type: Any
database_key=None, # type: bytes
):
# type: (...) -> Any
"""Returns the minimal example from the given strategy ``specifier`` that
matches the predicate function ``condition``."""
if settings is None:
settings = Settings(max_examples=2000)
settings = Settings(settings, suppress_health_check=HealthCheck.all())
if database_key is None and settings.database is not None:
database_key = function_digest(condition)
if not isinstance(specifier, SearchStrategy):
raise InvalidArgument(
'Expected SearchStrategy but got %r of type %s' % (
specifier, type(specifier).__name__
))
specifier.validate()
search = specifier
random = random or new_random()
successful_examples = [0]
last_data = [None]
last_repr = [None]
def template_condition(data):
with BuildContext(data):
try:
data.is_find = True
with deterministic_PRNG():
result = data.draw(search)
data.note(result)
success = condition(result)
except UnsatisfiedAssumption:
data.mark_invalid()
if success:
successful_examples[0] += 1
if settings.verbosity >= Verbosity.verbose:
if not successful_examples[0]:
report(
u'Tried non-satisfying example %s' % (nicerepr(result),))
elif success:
if successful_examples[0] == 1:
last_repr[0] = nicerepr(result)
report(u'Found satisfying example %s' % (last_repr[0],))
last_data[0] = data
elif (
sort_key(hbytes(data.buffer)) <
sort_key(last_data[0].buffer)
) and nicerepr(result) != last_repr[0]:
last_repr[0] = nicerepr(result)
report(u'Shrunk example to %s' % (last_repr[0],))
last_data[0] = data
if success and not data.frozen:
data.mark_interesting()
start = benchmark_time()
runner = ConjectureRunner(
template_condition, settings=settings, random=random,
database_key=database_key,
)
runner.run()
note_engine_for_statistics(runner)
run_time = benchmark_time() - start
if runner.interesting_examples:
data = ConjectureData.for_buffer(
list(runner.interesting_examples.values())[0].buffer)
with BuildContext(data):
with deterministic_PRNG():
return data.draw(search)
if runner.valid_examples == 0 and (
runner.exit_reason != ExitReason.finished
):
if settings.timeout > 0 and run_time > settings.timeout:
raise Timeout(( # pragma: no cover
'Ran out of time before finding enough valid examples for '
'%s. Only %d valid examples found in %.2f seconds.'
) % (
get_pretty_function_description(condition),
runner.valid_examples, run_time))
else:
raise Unsatisfiable(
'Unable to satisfy assumptions of %s.' %
(get_pretty_function_description(condition),)
)
raise NoSuchExample(get_pretty_function_description(condition))
