def run(data): if not hasattr(data, 'can_reproduce_example_from_repr'): data.can_reproduce_example_from_repr = True with local_settings(self.settings): with BuildContext(data, is_final=is_final): with deterministic_PRNG(): args, kwargs = data.draw(self.search_strategy) if expected_failure is not None: text_repr[0] = arg_string(test, args, kwargs) if print_example: example = '%s(%s)' % ( test.__name__, arg_string(test, args, kwargs)) try: ast.parse(example) except SyntaxError: data.can_reproduce_example_from_repr = False report('Falsifying example: %s' % (example,)) elif current_verbosity() >= Verbosity.verbose: report( lambda: 'Trying example: %s(%s)' % ( test.__name__, arg_string(test, args, kwargs))) with deterministic_PRNG(): return test(*args, **kwargs)
def template_condition(data): with BuildContext(data): try: data.is_find = True with deterministic_PRNG(): result = data.draw(search) data.note(result) success = condition(result) except UnsatisfiedAssumption: data.mark_invalid() if success: successful_examples[0] += 1 if settings.verbosity >= Verbosity.verbose: if not successful_examples[0]: report( u'Tried non-satisfying example %s' % (nicerepr(result),)) elif success: if successful_examples[0] == 1: last_repr[0] = nicerepr(result) report(u'Found satisfying example %s' % (last_repr[0],)) last_data[0] = data elif ( sort_key(hbytes(data.buffer)) < sort_key(last_data[0].buffer) ) and nicerepr(result) != last_repr[0]: last_repr[0] = nicerepr(result) report(u'Shrunk example to %s' % (last_repr[0],)) last_data[0] = data if success and not data.frozen: data.mark_interesting()
def run_to_data(f): with deterministic_PRNG(): runner = ConjectureRunner(f, settings=TEST_SETTINGS) runner.run() assert runner.interesting_examples last_data, = runner.interesting_examples.values() return last_data
def test_exhaust_space(): with deterministic_PRNG(): runner = ConjectureRunner( lambda data: data.draw_bits(1), settings=TEST_SETTINGS ) runner.run() assert runner.tree.is_exhausted assert runner.valid_examples == 2
def test_find_does_not_pollute_state(): with deterministic_PRNG(): find(st.random_module(), lambda r: True) state_a = random.getstate() find(st.random_module(), lambda r: True) state_b = random.getstate() assert state_a != state_b
def run_to_buffer(f): with deterministic_PRNG(): runner = ConjectureRunner(f, settings=settings( max_examples=5000, buffer_size=1024, database=None, suppress_health_check=HealthCheck.all(), )) runner.run() assert runner.interesting_examples last_data, = runner.interesting_examples.values() return hbytes(last_data.buffer)
def accept(f): with deterministic_PRNG(): runner = ConjectureRunner(f, settings=settings( max_examples=5000, buffer_size=1024, database=None, suppress_health_check=HealthCheck.all(), )) runner.test_function(ConjectureData.for_buffer(start)) assert runner.interesting_examples last_data, = runner.interesting_examples.values() return runner.new_shrinker( last_data, lambda d: d.status == Status.INTERESTING )
def test_one_dead_branch(): with deterministic_PRNG(): seen = set() @run_to_buffer def x(data): i = data.draw_bytes(1)[0] if i > 0: data.mark_invalid() i = data.draw_bytes(1)[0] if len(seen) < 255: seen.add(i) elif i not in seen: data.mark_interesting()
def test_given_does_not_pollute_state(): with deterministic_PRNG(): @given(st.random_module()) def test(r): pass test() state_a = random.getstate() test() state_b = random.getstate() assert state_a != state_b
def test_large_initial_write(): big = hbytes(b'\xff') * 512 def f(data): data.write(big) data.draw_bits(63) with deterministic_PRNG(): runner = ConjectureRunner(f, settings=settings( max_examples=5000, buffer_size=1024, database=None, suppress_health_check=HealthCheck.all(), )) runner.run() assert runner.exit_reason == ExitReason.finished
def test_cached_test_function_returns_right_value(): count = [0] def tf(data): count[0] += 1 data.draw_bits(2) data.mark_interesting() with deterministic_PRNG(): runner = ConjectureRunner(tf, settings=TEST_SETTINGS) for _ in hrange(2): for b in (b"\0", b"\1"): d = runner.cached_test_function(b) assert d.status == Status.INTERESTING assert d.buffer == b assert count[0] == 2
def test_cached_test_function_does_not_reinvoke_on_prefix(): call_count = [0] def test_function(data): call_count[0] += 1 data.draw_bits(8) data.write(hbytes([7])) data.draw_bits(8) with deterministic_PRNG(): runner = ConjectureRunner(test_function, settings=TEST_SETTINGS) data = runner.cached_test_function(hbytes(3)) assert data.status == Status.VALID for n in [2, 1, 0]: prefix_data = runner.cached_test_function(hbytes(n)) assert prefix_data is Overrun assert call_count[0] == 1
def test_populates_the_pareto_front(): with deterministic_PRNG(): def test(data): data.target_observations[""] = data.draw_bits(4) runner = ConjectureRunner( test, settings=settings( max_examples=5000, database=InMemoryExampleDatabase(), suppress_health_check=HealthCheck.all(), ), database_key=b"stuff", ) runner.run() assert len(runner.pareto_front) == 2**4
def test_uses_tags_in_calculating_pareto_front(): with deterministic_PRNG(): def test(data): if data.draw_bits(1): data.start_example(11) data.draw_bits(8) data.stop_example() runner = ConjectureRunner( test, settings=settings(max_examples=10, database=InMemoryExampleDatabase()), database_key=b"stuff", ) runner.run() assert len(runner.pareto_front) == 2
def test_pareto_front_contains_different_interesting_reasons(): with deterministic_PRNG(): def test(data): data.mark_interesting(data.draw_bits(4)) runner = ConjectureRunner( test, settings=settings( max_examples=5000, database=InMemoryExampleDatabase(), suppress_health_check=HealthCheck.all(), ), database_key=b"stuff", ) runner.run() assert len(runner.pareto_front) == 2 ** 4
def test_will_reset_the_tree_as_it_goes(monkeypatch): monkeypatch.setattr(engine_module, 'CACHE_RESET_FREQUENCY', 3) def f(data): data.draw_bits(8) with deterministic_PRNG(): runner = ConjectureRunner(f, settings=settings( database=None, suppress_health_check=HealthCheck.all(), )) def step(n): runner.test_function(ConjectureData.for_buffer([n])) step(0) step(1) assert len(runner.tree[0]) > 1 step(2) assert len(runner.tree[0]) == 1
def test_large_initial_write(): big = hbytes(b"\xff") * 512 def f(data): data.write(big) data.draw_bits(63) with deterministic_PRNG(): runner = ConjectureRunner( f, settings=settings( max_examples=5000, buffer_size=1024, database=None, suppress_health_check=HealthCheck.all(), ), ) runner.run() assert runner.exit_reason == ExitReason.finished
def run(data): with local_settings(self.settings): with deterministic_PRNG(): with BuildContext(data, is_final=is_final): args, kwargs = data.draw(self.search_strategy) if expected_failure is not None: text_repr[0] = arg_string(test, args, kwargs) if print_example: example = "%s(%s)" % ( test.__name__, arg_string(test, args, kwargs), ) report("Falsifying example: %s" % (example,)) elif current_verbosity() >= Verbosity.verbose: report( lambda: "Trying example: %s(%s)" % (test.__name__, arg_string(test, args, kwargs)) ) return test(*args, **kwargs)
def test_does_not_shrink_multiple_bugs_when_told_not_to(): def test(data): m = data.draw_bits(8) n = data.draw_bits(8) if m > 0: data.mark_interesting(1) if n > 5: data.mark_interesting(2) with deterministic_PRNG(): runner = ConjectureRunner( test, settings=settings(TEST_SETTINGS, report_multiple_bugs=False) ) runner.cached_test_function([255, 255]) runner.shrink_interesting_examples() results = {d.buffer for d in runner.interesting_examples.values()} assert len(results.intersection({bytes([0, 1]), bytes([1, 0])})) == 1
def test_optimiser_when_test_grows_buffer_to_overflow(): with deterministic_PRNG(): with buffer_size_limit(2): def test(data): m = data.draw_bits(8) data.target_observations["m"] = m if m > 100: data.draw_bits(64) data.mark_invalid() runner = ConjectureRunner(test, settings=TEST_SETTINGS) runner.cached_test_function(bytes(10)) try: runner.optimise_targets() except RunIsComplete: pass assert runner.best_observed_targets["m"] == 100
def test_discards_kill_branches(): starts = set() with deterministic_PRNG(): def test(data): assert runner.call_count <= 256 while True: data.start_example(1) b = data.draw_bits(8) data.stop_example(b != 0) if len(data.buffer) == 1: s = bytes(data.buffer) assert s not in starts starts.add(s) if b == 0: break runner = ConjectureRunner(test, settings=SMALL_COUNT_SETTINGS) runner.run() assert runner.call_count == 256
def test_database_contains_only_pareto_front(): with deterministic_PRNG(): def test(data): data.target_observations["1"] = data.draw_bits(4) data.draw_bits(64) data.target_observations["2"] = data.draw_bits(8) db = InMemoryExampleDatabase() runner = ConjectureRunner( test, settings=settings( max_examples=500, database=db, suppress_health_check=HealthCheck.all(), ), database_key=b"stuff", ) runner.run() assert len(runner.pareto_front) <= 500 for v in runner.pareto_front: assert v.status >= Status.VALID assert len(db.data) == 1 (values, ) = db.data.values() values = set(values) assert len(values) == len(runner.pareto_front) for data in runner.pareto_front: assert data.buffer in values assert data in runner.pareto_front for k in values: assert runner.cached_test_function(k) in runner.pareto_front
def test_does_not_trigger_health_check_on_simple_strategies(monkeypatch): existing_draw_bits = ConjectureData.draw_bits # We need to make drawing data artificially slow in order to trigger this # effect. This isn't actually slow because time is fake in our CI, but # we need it to pretend to be. def draw_bits(self, n, forced=None): time.sleep(0.001) return existing_draw_bits(self, n, forced) monkeypatch.setattr(ConjectureData, "draw_bits", draw_bits) with deterministic_PRNG(): for _ in range(100): # Setting max_examples=11 ensures we have enough examples for the # health checks to finish running, but cuts the generation short # after that point to allow this test to run in reasonable time. @settings(database=None, max_examples=11, phases=[Phase.generate]) @given(st.binary()) def test(b): pass test()
def test_will_not_reset_the_tree_after_interesting_example(monkeypatch): monkeypatch.setattr(engine_module, 'CACHE_RESET_FREQUENCY', 3) def f(data): if data.draw_bits(8) == 7: data.mark_interesting() with deterministic_PRNG(): runner = ConjectureRunner(f, settings=settings( database=None, suppress_health_check=HealthCheck.all(), )) def step(n): runner.test_function(ConjectureData.for_buffer([n])) step(0) step(1) assert len(runner.tree) > 1 step(7) assert len(runner.tree) > 1 t = len(runner.tree) runner.shrink_interesting_examples() assert len(runner.tree) > t
def test_branch_ending_in_write(): seen = set() def tf(data): count = 0 while data.draw_bits(1): count += 1 if count > 1: data.draw_bits(1, forced=0) b = hbytes(data.buffer) assert b not in seen seen.add(b) with deterministic_PRNG(): runner = ConjectureRunner(tf, settings=TEST_SETTINGS) for _ in hrange(100): prefix = runner.generate_novel_prefix() attempt = prefix + hbytes(2) data = runner.cached_test_function(attempt) assert data.status == Status.VALID assert attempt.startswith(data.buffer)
def test_can_patch_up_examples(): with deterministic_PRNG(): def test(data): data.start_example(42) m = data.draw_bits(6) data.target_observations["m"] = m for _ in range(m): data.draw_bits(1) data.stop_example() for i in range(4): if i != data.draw_bits(8): data.mark_invalid() runner = ConjectureRunner(test, settings=TEST_SETTINGS) d = runner.cached_test_function([0, 0, 1, 2, 3, 4]) assert d.status == Status.VALID try: runner.optimise_targets() except RunIsComplete: pass assert runner.best_observed_targets["m"] == 63
def test_branch_ending_in_write(): seen = set() def tf(data): count = 0 while data.draw_bits(1): count += 1 if count > 1: data.draw_bits(1, forced=0) b = hbytes(data.buffer) assert b not in seen seen.add(b) with deterministic_PRNG(): runner = ConjectureRunner(tf, settings=TEST_SETTINGS) for _ in hrange(100): prefix = runner.generate_novel_prefix() attempt = prefix + hbytes(2) data = runner.cached_test_function(attempt) assert data.status == Status.VALID assert attempt.startswith(data.buffer)
def test_optimises_multiple_targets(): with deterministic_PRNG(): def test(data): n = data.draw_bits(8) m = data.draw_bits(8) if n + m > 256: data.mark_invalid() data.target_observations["m"] = m data.target_observations["n"] = n data.target_observations["m + n"] = m + n runner = ConjectureRunner(test, settings=TEST_SETTINGS) runner.cached_test_function([200, 0]) runner.cached_test_function([0, 200]) try: runner.optimise_targets() except RunIsComplete: pass assert runner.best_observed_targets["m"] == 255 assert runner.best_observed_targets["n"] == 255 assert runner.best_observed_targets["m + n"] == 256
def run(data): if not hasattr(data, "can_reproduce_example_from_repr"): data.can_reproduce_example_from_repr = True with local_settings(self.settings): with BuildContext(data, is_final=is_final): with deterministic_PRNG(): args, kwargs = data.draw(self.search_strategy) if expected_failure is not None: text_repr[0] = arg_string(test, args, kwargs) if print_example: example = "%s(%s)" % ( test.__name__, arg_string(test, args, kwargs), ) try: ast.parse(example) except SyntaxError: data.can_reproduce_example_from_repr = False report("Falsifying example: %s" % (example, )) elif current_verbosity() >= Verbosity.verbose: report(lambda: "Trying example: %s(%s)" % ( test.__name__, arg_string(test, args, kwargs))) return test(*args, **kwargs)
def test_can_find_endpoints_of_a_range(lower, upper, score_up): with deterministic_PRNG(): def test(data): n = data.draw_bits(16) if n < lower or n > upper: data.mark_invalid() if not score_up: n = -n data.target_observations["n"] = n runner = ConjectureRunner( test, settings=settings(TEST_SETTINGS, max_examples=1000) ) runner.cached_test_function(int_to_bytes((lower + upper) // 2, 2)) try: runner.optimise_targets() except RunIsComplete: pass if score_up: assert runner.best_observed_targets["n"] == upper else: assert runner.best_observed_targets["n"] == -lower
def run(data): # Set up dynamic context needed by a single test run. with local_settings(self.settings): with deterministic_PRNG(): with BuildContext(data, is_final=is_final): # Generate all arguments to the test function. args, kwargs = data.draw(self.search_strategy) if expected_failure is not None: text_repr[0] = arg_string(test, args, kwargs) if print_example or current_verbosity( ) >= Verbosity.verbose: output = CUnicodeIO() printer = RepresentationPrinter(output) if print_example: printer.text("Falsifying example:") else: printer.text("Trying example:") if self.print_given_args: printer.text(" ") printer.text(test.__name__) with printer.group(indent=4, open="(", close=""): printer.break_() for v in args: printer.pretty(v) # We add a comma unconditionally because # generated arguments will always be # kwargs, so there will always be more # to come. printer.text(",") printer.breakable() # We need to make sure to print these in the argument order for # Python 2 and older versionf of Python 3.5. In modern versions # this isn't an issue because kwargs is ordered. arg_order = { v: i for i, v in enumerate( getfullargspec(self.test).args) } for i, (k, v) in enumerate( sorted( kwargs.items(), key=lambda t: ( arg_order.get( t[0], float("inf")), t[0], ), )): printer.text(k) printer.text("=") printer.pretty(v) printer.text(",") if i + 1 < len(kwargs): printer.breakable() printer.break_() printer.text(")") printer.flush() report(output.getvalue()) return test(*args, **kwargs)
def find( specifier, # type: SearchStrategy condition, # type: Callable[[Any], bool] settings=None, # type: Settings random=None, # type: Any database_key=None, # type: bytes ): # type: (...) -> Any """Returns the minimal example from the given strategy ``specifier`` that matches the predicate function ``condition``.""" if settings is None: settings = Settings(max_examples=2000) settings = Settings(settings, suppress_health_check=HealthCheck.all()) if database_key is None and settings.database is not None: database_key = function_digest(condition) if not isinstance(specifier, SearchStrategy): raise InvalidArgument("Expected SearchStrategy but got %r of type %s" % (specifier, type(specifier).__name__)) specifier.validate() search = specifier random = random or new_random() successful_examples = [0] last_data = [None] last_repr = [None] def template_condition(data): with BuildContext(data): try: data.is_find = True with deterministic_PRNG(): result = data.draw(search) data.note(result) success = condition(result) except UnsatisfiedAssumption: data.mark_invalid() if success: successful_examples[0] += 1 if settings.verbosity >= Verbosity.verbose: if not successful_examples[0]: report(u"Tried non-satisfying example %s" % (nicerepr(result), )) elif success: if successful_examples[0] == 1: last_repr[0] = nicerepr(result) report(u"Found satisfying example %s" % (last_repr[0], )) last_data[0] = data elif (sort_key(hbytes(data.buffer)) < sort_key( last_data[0].buffer) ) and nicerepr(result) != last_repr[0]: last_repr[0] = nicerepr(result) report(u"Shrunk example to %s" % (last_repr[0], )) last_data[0] = data if success and not data.frozen: data.mark_interesting() runner = ConjectureRunner(template_condition, settings=settings, random=random, database_key=database_key) runner.run() note_engine_for_statistics(runner) if runner.interesting_examples: data = ConjectureData.for_buffer( list(runner.interesting_examples.values())[0].buffer) with BuildContext(data): with deterministic_PRNG(): return data.draw(search) if runner.valid_examples == 0 and (runner.exit_reason != ExitReason.finished): raise Unsatisfiable("Unable to satisfy assumptions of %s." % (get_pretty_function_description(condition), )) raise NoSuchExample(get_pretty_function_description(condition))
def test_can_run_with_no_db(): with deterministic_PRNG(): with raises(AssertionError): run_state_machine_as_test(DepthMachine, settings=Settings(database=None))
def find( specifier, # type: SearchStrategy condition, # type: Callable[[Any], bool] settings=None, # type: Settings random=None, # type: Any database_key=None, # type: bytes ): # type: (...) -> Any """Returns the minimal example from the given strategy ``specifier`` that matches the predicate function ``condition``.""" if settings is None: settings = Settings(max_examples=2000) settings = Settings(settings, suppress_health_check=HealthCheck.all()) if database_key is None and settings.database is not None: database_key = function_digest(condition) if not isinstance(specifier, SearchStrategy): raise InvalidArgument( 'Expected SearchStrategy but got %r of type %s' % ( specifier, type(specifier).__name__ )) specifier.validate() search = specifier random = random or new_random() successful_examples = [0] last_data = [None] last_repr = [None] def template_condition(data): with BuildContext(data): try: data.is_find = True with deterministic_PRNG(): result = data.draw(search) data.note(result) success = condition(result) except UnsatisfiedAssumption: data.mark_invalid() if success: successful_examples[0] += 1 if settings.verbosity >= Verbosity.verbose: if not successful_examples[0]: report( u'Tried non-satisfying example %s' % (nicerepr(result),)) elif success: if successful_examples[0] == 1: last_repr[0] = nicerepr(result) report(u'Found satisfying example %s' % (last_repr[0],)) last_data[0] = data elif ( sort_key(hbytes(data.buffer)) < sort_key(last_data[0].buffer) ) and nicerepr(result) != last_repr[0]: last_repr[0] = nicerepr(result) report(u'Shrunk example to %s' % (last_repr[0],)) last_data[0] = data if success and not data.frozen: data.mark_interesting() start = benchmark_time() runner = ConjectureRunner( template_condition, settings=settings, random=random, database_key=database_key, ) runner.run() note_engine_for_statistics(runner) run_time = benchmark_time() - start if runner.interesting_examples: data = ConjectureData.for_buffer( list(runner.interesting_examples.values())[0].buffer) with BuildContext(data): with deterministic_PRNG(): return data.draw(search) if runner.valid_examples == 0 and ( runner.exit_reason != ExitReason.finished ): if settings.timeout > 0 and run_time > settings.timeout: raise Timeout(( # pragma: no cover 'Ran out of time before finding enough valid examples for ' '%s. Only %d valid examples found in %.2f seconds.' ) % ( get_pretty_function_description(condition), runner.valid_examples, run_time)) else: raise Unsatisfiable( 'Unable to satisfy assumptions of %s.' % (get_pretty_function_description(condition),) ) raise NoSuchExample(get_pretty_function_description(condition))