class StateMachineTestCase(TestCase): settings = Settings(deadline=None, suppress_health_check=HealthCheck.all()) def runTest(self): run_state_machine_as_test(state_machine_class) runTest.is_hypothesis_test = True
def accept(test): test._hypothesis_internal_use_seed = seed current_settings = getattr(test, "_hypothesis_internal_use_settings", None) test._hypothesis_internal_use_settings = Settings(current_settings, database=None) return test
def run_state_machine_as_test(state_machine_factory, settings=None): """Run a state machine definition as a test, either silently doing nothing or printing a minimal breaking program and raising an exception. state_machine_factory is anything which returns an instance of GenericStateMachine when called with no arguments - it can be a class or a function. settings will be used to control the execution of the test. """ if settings is None: try: settings = state_machine_factory.TestCase.settings check_type(Settings, settings, "state_machine_factory.TestCase.settings") except AttributeError: settings = Settings(deadline=None, suppress_health_check=HealthCheck.all()) check_type(Settings, settings, "settings") @settings @given(st.data()) def run_state_machine(factory, data): machine = factory() check_type(GenericStateMachine, machine, "state_machine_factory()") data.conjecture_data.hypothesis_runner = machine n_steps = settings.stateful_step_count should_continue = cu.many(data.conjecture_data, min_size=1, max_size=n_steps, average_size=n_steps) print_steps = (current_build_context().is_final or current_verbosity() >= Verbosity.debug) try: if print_steps: machine.print_start() machine.check_invariants() while should_continue.more(): value = data.conjecture_data.draw(machine.steps()) if print_steps: machine.print_step(value) machine.execute_step(value) machine.check_invariants() finally: if print_steps: machine.print_end() machine.teardown() # Use a machine digest to identify stateful tests in the example database run_state_machine.hypothesis.inner_test._hypothesis_internal_add_digest = function_digest( state_machine_factory) # Copy some attributes so @seed and @reproduce_failure "just work" run_state_machine._hypothesis_internal_use_seed = getattr( state_machine_factory, "_hypothesis_internal_use_seed", None) run_state_machine._hypothesis_internal_use_reproduce_failure = getattr( state_machine_factory, "_hypothesis_internal_use_reproduce_failure", None) run_state_machine(state_machine_factory)
class StateMachineTestCase(TestCase): settings = Settings( min_satisfying_examples=1 ) def runTest(self): run_state_machine_as_test(state_machine_class)
def find( specifier: SearchStrategy[Ex], condition: Callable[[Any], bool], *, settings: Settings = None, random: Random = None, database_key: bytes = None ) -> Ex: """Returns the minimal example from the given strategy ``specifier`` that matches the predicate function ``condition``.""" if settings is None: settings = Settings(max_examples=2000) settings = Settings( settings, suppress_health_check=HealthCheck.all(), report_multiple_bugs=False ) if database_key is None and settings.database is not None: database_key = function_digest(condition) if not isinstance(specifier, SearchStrategy): raise InvalidArgument( "Expected SearchStrategy but got %r of type %s" % (specifier, type(specifier).__name__) ) specifier.validate() last = [] # type: List[Ex] @settings @given(specifier) def test(v): if condition(v): last[:] = [v] raise Found() if random is not None: test = seed(random.getrandbits(64))(test) test._hypothesis_internal_is_find = True test._hypothesis_internal_database_key = database_key try: test() except Found: return last[0] raise NoSuchExample(get_pretty_function_description(condition))
def find( specifier, # type: SearchStrategy condition, # type: Callable[[Any], bool] settings=None, # type: Settings random=None, # type: Any database_key=None, # type: bytes ): # type: (...) -> Any """Returns the minimal example from the given strategy ``specifier`` that matches the predicate function ``condition``.""" if settings is None: settings = Settings(max_examples=2000) settings = Settings(settings, suppress_health_check=HealthCheck.all()) if database_key is None and settings.database is not None: database_key = function_digest(condition) if not isinstance(specifier, SearchStrategy): raise InvalidArgument("Expected SearchStrategy but got %r of type %s" % (specifier, type(specifier).__name__)) specifier.validate() search = specifier random = random or new_random() successful_examples = [0] last_data = [None] last_repr = [None] def template_condition(data): with BuildContext(data): try: data.is_find = True with deterministic_PRNG(): result = data.draw(search) data.note(result) success = condition(result) except UnsatisfiedAssumption: data.mark_invalid() if success: successful_examples[0] += 1 if settings.verbosity >= Verbosity.verbose: if not successful_examples[0]: report(u"Tried non-satisfying example %s" % (nicerepr(result), )) elif success: if successful_examples[0] == 1: last_repr[0] = nicerepr(result) report(u"Found satisfying example %s" % (last_repr[0], )) last_data[0] = data elif (sort_key(hbytes(data.buffer)) < sort_key( last_data[0].buffer) ) and nicerepr(result) != last_repr[0]: last_repr[0] = nicerepr(result) report(u"Shrunk example to %s" % (last_repr[0], )) last_data[0] = data if success and not data.frozen: data.mark_interesting() runner = ConjectureRunner(template_condition, settings=settings, random=random, database_key=database_key) runner.run() note_engine_for_statistics(runner) if runner.interesting_examples: data = ConjectureData.for_buffer( list(runner.interesting_examples.values())[0].buffer) with BuildContext(data): with deterministic_PRNG(): return data.draw(search) if runner.valid_examples == 0 and (runner.exit_reason != ExitReason.finished): raise Unsatisfiable("Unable to satisfy assumptions of %s." % (get_pretty_function_description(condition), )) raise NoSuchExample(get_pretty_function_description(condition))
class StateMachineTestCase(TestCase): settings = Settings()
def run_state_machine_as_test(state_machine_factory, settings=None): """Run a state machine definition as a test, either silently doing nothing or printing a minimal breaking program and raising an exception. state_machine_factory is anything which returns an instance of RuleBasedStateMachine when called with no arguments - it can be a class or a function. settings will be used to control the execution of the test. """ if settings is None: try: settings = state_machine_factory.TestCase.settings check_type(Settings, settings, "state_machine_factory.TestCase.settings") except AttributeError: settings = Settings(deadline=None, suppress_health_check=HealthCheck.all()) check_type(Settings, settings, "settings") @settings @given(st.data()) def run_state_machine(factory, data): machine = factory() if not isinstance(machine, _GenericStateMachine): raise InvalidArgument( "Expected RuleBasedStateMachine but state_machine_factory() " "returned %r (type=%s)" % (machine, type(machine).__name__)) data.conjecture_data.hypothesis_runner = machine print_steps = (current_build_context().is_final or current_verbosity() >= Verbosity.debug) try: if print_steps: machine.print_start() machine.check_invariants() max_steps = settings.stateful_step_count steps_run = 0 cd = data.conjecture_data while True: # We basically always want to run the maximum number of steps, # but need to leave a small probability of terminating early # in order to allow for reducing the number of steps once we # find a failing test case, so we stop with probability of # 2 ** -16 during normal operation but force a stop when we've # generated enough steps. cd.start_example(STATE_MACHINE_RUN_LABEL) if steps_run == 0: cd.draw_bits(16, forced=1) elif steps_run >= max_steps: cd.draw_bits(16, forced=0) break else: # All we really care about is whether this value is zero # or non-zero, so if it's > 1 we discard it and insert a # replacement value after cd.start_example(SHOULD_CONTINUE_LABEL) should_continue_value = cd.draw_bits(16) if should_continue_value > 1: cd.stop_example(discard=True) cd.draw_bits(16, forced=int(bool(should_continue_value))) else: cd.stop_example() if should_continue_value == 0: break steps_run += 1 value = data.conjecture_data.draw(machine.steps()) # Assign 'result' here in case 'execute_step' fails below result = multiple() try: result = machine.execute_step(value) finally: if print_steps: # 'result' is only used if the step has target bundles. # If it does, and the result is a 'MultipleResult', # then 'print_step' prints a multi-variable assignment. machine.print_step(value, result) machine.check_invariants() data.conjecture_data.stop_example() finally: if print_steps: machine.print_end() machine.teardown() # Use a machine digest to identify stateful tests in the example database run_state_machine.hypothesis.inner_test._hypothesis_internal_add_digest = function_digest( state_machine_factory) # Copy some attributes so @seed and @reproduce_failure "just work" run_state_machine._hypothesis_internal_use_seed = getattr( state_machine_factory, "_hypothesis_internal_use_seed", None) run_state_machine._hypothesis_internal_use_reproduce_failure = getattr( state_machine_factory, "_hypothesis_internal_use_reproduce_failure", None) run_state_machine._hypothesis_internal_print_given_args = False run_state_machine(state_machine_factory)
class StateMachineTestCase(TestCase): settings = Settings(deadline=None, suppress_health_check=HealthCheck.all())
def find(specifier, condition, settings=None, random=None, database_key=None): """Returns the minimal example from the given strategy ``specifier`` that matches the predicate function ``condition``.""" settings = settings or Settings( max_examples=2000, min_satisfying_examples=0, max_shrinks=2000, ) settings = Settings(settings, perform_health_check=False) if database_key is None and settings.database is not None: database_key = function_digest(condition) if not isinstance(specifier, SearchStrategy): raise InvalidArgument( 'Expected SearchStrategy but got %r of type %s' % ( specifier, type(specifier).__name__ )) specifier.validate() search = specifier random = random or new_random() successful_examples = [0] last_data = [None] def template_condition(data): with BuildContext(data): try: data.is_find = True result = data.draw(search) data.note(result) success = condition(result) except UnsatisfiedAssumption: data.mark_invalid() if success: successful_examples[0] += 1 if settings.verbosity == Verbosity.verbose: if not successful_examples[0]: report(lambda: u'Trying example %s' % ( nicerepr(result), )) elif success: if successful_examples[0] == 1: report(lambda: u'Found satisfying example %s' % ( nicerepr(result), )) last_data[0] = data elif ( sort_key(hbytes(data.buffer)) < sort_key(last_data[0].buffer) ): report(lambda: u'Shrunk example to %s' % ( nicerepr(result), )) last_data[0] = data if success and not data.frozen: data.mark_interesting() start = time.time() runner = ConjectureRunner( template_condition, settings=settings, random=random, database_key=database_key, ) runner.run() note_engine_for_statistics(runner) run_time = time.time() - start if runner.interesting_examples: data = ConjectureData.for_buffer( list(runner.interesting_examples.values())[0].buffer) with BuildContext(data): return data.draw(search) if ( runner.valid_examples <= settings.min_satisfying_examples and runner.exit_reason != ExitReason.finished ): if settings.timeout > 0 and run_time > settings.timeout: raise Timeout(( 'Ran out of time before finding enough valid examples for ' '%s. Only %d valid examples found in %.2f seconds.' ) % ( get_pretty_function_description(condition), runner.valid_examples, run_time)) else: raise Unsatisfiable(( 'Unable to satisfy assumptions of ' '%s. Only %d examples considered satisfied assumptions' ) % ( get_pretty_function_description(condition), runner.valid_examples,)) raise NoSuchExample(get_pretty_function_description(condition))
def perform_health_checks(random, settings, test_runner, search_strategy): if not settings.perform_health_check: return if not Settings.default.perform_health_check: return health_check_random = Random(random.getrandbits(128)) # We "pre warm" the health check with one draw to give it some # time to calculate any cached data. This prevents the case # where the first draw of the health check takes ages because # of loading unicode data the first time. data = ConjectureData(max_length=settings.buffer_size, draw_bytes=lambda data, n, distribution: distribution(health_check_random, n)) with Settings(settings, verbosity=Verbosity.quiet): try: test_runner( data, reify_and_execute( search_strategy, lambda *args, **kwargs: None, )) except BaseException: pass count = 0 overruns = 0 filtered_draws = 0 start = time.time() while (count < 10 and time.time() < start + 1 and filtered_draws < 50 and overruns < 20): try: data = ConjectureData(max_length=settings.buffer_size, draw_bytes=lambda data, n, distribution: distribution(health_check_random, n)) with Settings(settings, verbosity=Verbosity.quiet): test_runner( data, reify_and_execute( search_strategy, lambda *args, **kwargs: None, )) count += 1 except UnsatisfiedAssumption: filtered_draws += 1 except StopTest: if data.status == Status.INVALID: filtered_draws += 1 else: assert data.status == Status.OVERRUN overruns += 1 except InvalidArgument: raise except Exception: escalate_hypothesis_internal_error() if (HealthCheck.exception_in_generation in settings.suppress_health_check): raise report(traceback.format_exc()) if test_runner is default_new_style_executor: fail_health_check( settings, 'An exception occurred during data ' 'generation in initial health check. ' 'This indicates a bug in the strategy. ' 'This could either be a Hypothesis bug or ' "an error in a function you've passed to " 'it to construct your data.', HealthCheck.exception_in_generation, ) else: fail_health_check( settings, 'An exception occurred during data ' 'generation in initial health check. ' 'This indicates a bug in the strategy. ' 'This could either be a Hypothesis bug or ' 'an error in a function you\'ve passed to ' 'it to construct your data. Additionally, ' 'you have a custom executor, which means ' 'that this could be your executor failing ' 'to handle a function which returns None. ', HealthCheck.exception_in_generation, ) if overruns >= 20 or (not count and overruns > 0): fail_health_check( settings, ('Examples routinely exceeded the max allowable size. ' '(%d examples overran while generating %d valid ones)' '. Generating examples this large will usually lead to' ' bad results. You should try setting average_size or ' 'max_size parameters on your collections and turning ' 'max_leaves down on recursive() calls.') % (overruns, count), HealthCheck.data_too_large) if filtered_draws >= 50 or (not count and filtered_draws > 0): fail_health_check( settings, ('It looks like your strategy is filtering out a lot ' 'of data. Health check found %d filtered examples but ' 'only %d good ones. This will make your tests much ' 'slower, and also will probably distort the data ' 'generation quite a lot. You should adapt your ' 'strategy to filter less. This can also be caused by ' 'a low max_leaves parameter in recursive() calls') % (filtered_draws, count), HealthCheck.filter_too_much) runtime = time.time() - start if runtime > 1.0 or count < 10: fail_health_check( settings, ('Data generation is extremely slow: Only produced ' '%d valid examples in %.2f seconds (%d invalid ones ' 'and %d exceeded maximum size). Try decreasing ' "size of the data you're generating (with e.g." 'average_size or max_leaves parameters).') % (count, runtime, filtered_draws, overruns), HealthCheck.too_slow, )
def find(specifier, condition, settings=None, random=None, storage=None): settings = settings or Settings( max_examples=2000, min_satisfying_examples=0, max_shrinks=2000, ) if not isinstance(specifier, SearchStrategy): raise InvalidArgument('Expected SearchStrategy but got %r of type %s' % (specifier, type(specifier).__name__)) search = specifier if storage is None and settings.database is not None: storage = settings.database.storage( 'find(%s)' % (binascii.hexlify(function_digest(condition)).decode('ascii'), )) random = random or new_random() successful_examples = [0] def template_condition(template): with BuildContext(): result = search.reify(template) success = condition(result) if success: successful_examples[0] += 1 if not successful_examples[0]: verbose_report(lambda: 'Trying example %s' % (repr(result), )) elif success: if successful_examples[0] == 1: verbose_report(lambda: 'Found satisfying example %s' % (repr(result), )) else: verbose_report(lambda: 'Shrunk example to %s' % (repr(result), )) return success template_condition.__name__ = condition.__name__ tracker = Tracker() try: template = best_satisfying_template( search, random, template_condition, settings, tracker=tracker, max_parameter_tries=2, storage=storage, ) with BuildContext(is_final=True, close_on_capture=False): return search.reify(template) except Timeout: raise except NoSuchExample: if search.template_upper_bound <= len(tracker): raise DefinitelyNoSuchExample( get_pretty_function_description(condition), search.template_upper_bound, ) raise NoSuchExample(get_pretty_function_description(condition))
def wrapped_test(*arguments, **kwargs): settings = wrapped_test._hypothesis_internal_use_settings if wrapped_test._hypothesis_internal_use_seed is not None: random = Random(wrapped_test._hypothesis_internal_use_seed) elif settings.derandomize: random = Random(function_digest(test)) else: random = new_random() import hypothesis.strategies as sd selfy = None arguments, kwargs = convert_positional_arguments( wrapped_test, arguments, kwargs) # If the test function is a method of some kind, the bound object # will be the first named argument if there are any, otherwise the # first vararg (if any). if argspec.args: selfy = kwargs.get(argspec.args[0]) elif arguments: selfy = arguments[0] test_runner = executor(selfy) for example in reversed( getattr(wrapped_test, 'hypothesis_explicit_examples', ())): if example.args: example_kwargs = dict( zip(original_argspec.args[-len(example.args):], example.args)) else: example_kwargs = example.kwargs example_kwargs.update(kwargs) # Note: Test may mutate arguments and we can't rerun explicit # examples, so we have to calculate the failure message at this # point rather than than later. message_on_failure = 'Falsifying example: %s(%s)' % ( test.__name__, arg_string(test, arguments, example_kwargs)) try: with BuildContext() as b: test_runner(lambda: test(*arguments, **example_kwargs)) except BaseException: report(message_on_failure) for n in b.notes: report(n) raise arguments = tuple(arguments) given_specifier = sd.tuples( sd.just(arguments), sd.fixed_dictionaries(generator_kwargs).map( lambda args: dict(args, **kwargs))) def fail_health_check(message): message += ( '\nSee http://hypothesis.readthedocs.org/en/latest/health' 'checks.html for more information about this.') if settings.strict: raise FailedHealthCheck(message) else: warnings.warn(FailedHealthCheck(message)) search_strategy = given_specifier search_strategy.validate() if settings.database: storage = settings.database.storage(fully_qualified_name(test)) else: storage = None start = time.time() warned_random = [False] perform_health_check = settings.perform_health_check if Settings.default is not None: perform_health_check &= Settings.default.perform_health_check if perform_health_check: initial_state = getglobalrandomstate() health_check_random = Random(random.getrandbits(128)) count = 0 bad_draws = 0 filtered_draws = 0 errors = 0 while (count < 10 and time.time() < start + 1 and filtered_draws < 50 and bad_draws < 50): try: with Settings(settings, verbosity=Verbosity.quiet): test_runner( reify_and_execute( search_strategy, search_strategy.draw_template( health_check_random, search_strategy.draw_parameter( health_check_random, )), lambda *args, **kwargs: None, )) count += 1 except BadTemplateDraw: bad_draws += 1 except UnsatisfiedAssumption: filtered_draws += 1 except Exception: if errors == 0: report(traceback.format_exc()) errors += 1 if test_runner is default_executor: fail_health_check( 'An exception occurred during data ' 'generation in initial health check. ' 'This indicates a bug in the strategy. ' 'This could either be a Hypothesis bug or ' "an error in a function yo've passed to " 'it to construct your data.') else: fail_health_check( 'An exception occurred during data ' 'generation in initial health check. ' 'This indicates a bug in the strategy. ' 'This could either be a Hypothesis bug or ' 'an error in a function you\'ve passed to ' 'it to construct your data. Additionally, ' 'you have a custom executor, which means ' 'that this could be your executor failing ' 'to handle a function which returns None. ') if filtered_draws >= 50: fail_health_check(( 'It looks like your strategy is filtering out a lot ' 'of data. Health check found %d filtered examples but ' 'only %d good ones. This will make your tests much ' 'slower, and also will probably distort the data ' 'generation quite a lot. You should adapt your ' 'strategy to filter less.') % (filtered_draws, count)) if bad_draws >= 50: fail_health_check( 'Hypothesis is struggling to generate examples. ' 'This is often a sign of a recursive strategy which ' 'fans out too broadly. If you\'re using recursive, ' 'try to reduce the size of the recursive step or ' 'increase the maximum permitted number of leaves.') runtime = time.time() - start if runtime > 1.0 or count < 10: fail_health_check( ('Data generation is extremely slow: Only produced ' '%d valid examples in %.2f seconds. Try decreasing ' "size of the data yo're generating (with e.g." 'average_size or max_leaves parameters).') % (count, runtime)) if getglobalrandomstate() != initial_state: warned_random[0] = True fail_health_check( 'Data generation depends on global random module. ' 'This makes results impossible to replay, which ' 'prevents Hypothesis from working correctly. ' 'If you want to use methods from random, use ' 'randoms() from hypothesis.strategies to get an ' 'instance of Random you can use. Alternatively, you ' 'can use the random_module() strategy to explicitly ' 'seed the random module.') last_exception = [None] repr_for_last_exception = [None] def is_template_example(xs): if perform_health_check and not warned_random[0]: initial_state = getglobalrandomstate() record_repr = [None] try: result = test_runner( reify_and_execute( search_strategy, xs, test, record_repr=record_repr, )) if result is not None and settings.perform_health_check: raise FailedHealthCheck( ('Tests run under @given should return None, but ' '%s returned %r instead.') % (test.__name__, result), settings) return False except (HypothesisDeprecationWarning, FailedHealthCheck, UnsatisfiedAssumption): raise except Exception: last_exception[0] = traceback.format_exc() repr_for_last_exception[0] = record_repr[0] verbose_report(last_exception[0]) return True finally: if (not warned_random[0] and perform_health_check and getglobalrandomstate() != initial_state): warned_random[0] = True fail_health_check( 'Your test used the global random module. ' 'This is unlikely to work correctly. You should ' 'consider using the randoms() strategy from ' 'hypothesis.strategies instead. Alternatively, ' 'you can use the random_module() strategy to ' 'explicitly seed the random module.') is_template_example.__name__ = test.__name__ is_template_example.__qualname__ = qualname(test) with settings: falsifying_template = None try: falsifying_template = best_satisfying_template( search_strategy, random, is_template_example, settings, storage, start_time=start, ) except NoSuchExample: return assert last_exception[0] is not None try: test_runner( reify_and_execute(search_strategy, falsifying_template, test, print_example=True, is_final=True)) except UnsatisfiedAssumption: report(traceback.format_exc()) raise Flaky( 'Unreliable assumption: An example which satisfied ' 'assumptions on the first run now fails it.') report( 'Failed to reproduce exception. Expected: \n' + last_exception[0], ) try: test_runner( reify_and_execute(search_strategy, falsifying_template, test_is_flaky( test, repr_for_last_exception[0]), print_example=True, is_final=True)) except UnsatisfiedAssumption: raise Flaky( 'Unreliable test data: Failed to reproduce a failure ' 'and then when it came to recreating the example in ' 'order to print the test data with a flaky result ' 'the example was filtered out (by e.g. a ' 'call to filter in your strategy) when we didn\'t ' 'expect it to be.')
def find(specifier, condition, settings=None, random=None, database_key=None): settings = settings or Settings( max_examples=2000, min_satisfying_examples=0, max_shrinks=2000, ) if database_key is None and settings.database is not None: database_key = function_digest(condition) if not isinstance(specifier, SearchStrategy): raise InvalidArgument( 'Expected SearchStrategy but got %r of type %s' % ( specifier, type(specifier).__name__ )) specifier.validate() search = specifier random = random or new_random() successful_examples = [0] last_data = [None] def template_condition(data): with BuildContext(data): try: data.is_find = True result = data.draw(search) data.note(result) success = condition(result) except UnsatisfiedAssumption: data.mark_invalid() if success: successful_examples[0] += 1 if settings.verbosity == Verbosity.verbose: if not successful_examples[0]: report(lambda: u'Trying example %s' % ( nicerepr(result), )) elif success: if successful_examples[0] == 1: report(lambda: u'Found satisfying example %s' % ( nicerepr(result), )) else: report(lambda: u'Shrunk example to %s' % ( nicerepr(result), )) last_data[0] = data if success and not data.frozen: data.mark_interesting() from hypothesis.internal.conjecture.engine import ConjectureRunner from hypothesis.internal.conjecture.data import ConjectureData, Status start = time.time() runner = ConjectureRunner( template_condition, settings=settings, random=random, database_key=database_key, ) runner.run() note_engine_for_statistics(runner) run_time = time.time() - start if runner.last_data.status == Status.INTERESTING: data = ConjectureData.for_buffer(runner.last_data.buffer) with BuildContext(data): return data.draw(search) if runner.valid_examples <= settings.min_satisfying_examples: if settings.timeout > 0 and run_time > settings.timeout: raise Timeout(( 'Ran out of time before finding enough valid examples for ' '%s. Only %d valid examples found in %.2f seconds.' ) % ( get_pretty_function_description(condition), runner.valid_examples, run_time)) else: raise Unsatisfiable(( 'Unable to satisfy assumptions of ' '%s. Only %d examples considered satisfied assumptions' ) % ( get_pretty_function_description(condition), runner.valid_examples,)) raise NoSuchExample(get_pretty_function_description(condition))
def wrapped_test(*arguments, **kwargs): settings = wrapped_test._hypothesis_internal_use_settings if wrapped_test._hypothesis_internal_use_seed is not None: random = Random( wrapped_test._hypothesis_internal_use_seed) elif settings.derandomize: random = Random(function_digest(test)) else: random = new_random() import hypothesis.strategies as sd selfy = None arguments, kwargs = convert_positional_arguments( wrapped_test, arguments, kwargs) # If the test function is a method of some kind, the bound object # will be the first named argument if there are any, otherwise the # first vararg (if any). if argspec.args: selfy = kwargs.get(argspec.args[0]) elif arguments: selfy = arguments[0] test_runner = new_style_executor(selfy) for example in reversed(getattr( wrapped_test, 'hypothesis_explicit_examples', () )): if example.args: if len(example.args) > len(original_argspec.args): raise InvalidArgument( 'example has too many arguments for test. ' 'Expected at most %d but got %d' % ( len(original_argspec.args), len(example.args))) example_kwargs = dict(zip( original_argspec.args[-len(example.args):], example.args )) else: example_kwargs = example.kwargs if Phase.explicit not in settings.phases: continue example_kwargs.update(kwargs) # Note: Test may mutate arguments and we can't rerun explicit # examples, so we have to calculate the failure message at this # point rather than than later. message_on_failure = 'Falsifying example: %s(%s)' % ( test.__name__, arg_string(test, arguments, example_kwargs) ) try: with BuildContext(None) as b: test_runner( None, lambda data: test(*arguments, **example_kwargs) ) except BaseException: traceback.print_exc() report(message_on_failure) for n in b.notes: report(n) raise if settings.max_examples <= 0: return arguments = tuple(arguments) given_specifier = sd.tuples( sd.just(arguments), sd.fixed_dictionaries(generator_kwargs).map( lambda args: dict(args, **kwargs) ) ) def fail_health_check(message, label): if label in settings.suppress_health_check: return message += ( '\nSee https://hypothesis.readthedocs.io/en/latest/health' 'checks.html for more information about this. ' ) message += ( 'If you want to disable just this health check, add %s ' 'to the suppress_health_check settings for this test.' ) % (label,) raise FailedHealthCheck(message) search_strategy = given_specifier if selfy is not None: search_strategy = WithRunner(search_strategy, selfy) search_strategy.validate() perform_health_check = settings.perform_health_check perform_health_check &= Settings.default.perform_health_check from hypothesis.internal.conjecture.data import ConjectureData, \ Status, StopTest if not ( Phase.reuse in settings.phases or Phase.generate in settings.phases ): return if perform_health_check: health_check_random = Random(random.getrandbits(128)) # We "pre warm" the health check with one draw to give it some # time to calculate any cached data. This prevents the case # where the first draw of the health check takes ages because # of loading unicode data the first time. data = ConjectureData( max_length=settings.buffer_size, draw_bytes=lambda data, n, distribution: distribution(health_check_random, n) ) with Settings(settings, verbosity=Verbosity.quiet): try: test_runner(data, reify_and_execute( search_strategy, lambda *args, **kwargs: None, )) except BaseException: pass count = 0 overruns = 0 filtered_draws = 0 start = time.time() while ( count < 10 and time.time() < start + 1 and filtered_draws < 50 and overruns < 20 ): try: data = ConjectureData( max_length=settings.buffer_size, draw_bytes=lambda data, n, distribution: distribution(health_check_random, n) ) with Settings(settings, verbosity=Verbosity.quiet): test_runner(data, reify_and_execute( search_strategy, lambda *args, **kwargs: None, )) count += 1 except UnsatisfiedAssumption: filtered_draws += 1 except StopTest: if data.status == Status.INVALID: filtered_draws += 1 else: assert data.status == Status.OVERRUN overruns += 1 except InvalidArgument: raise except Exception: if ( HealthCheck.exception_in_generation in settings.suppress_health_check ): raise report(traceback.format_exc()) if test_runner is default_new_style_executor: fail_health_check( 'An exception occurred during data ' 'generation in initial health check. ' 'This indicates a bug in the strategy. ' 'This could either be a Hypothesis bug or ' "an error in a function you've passed to " 'it to construct your data.', HealthCheck.exception_in_generation, ) else: fail_health_check( 'An exception occurred during data ' 'generation in initial health check. ' 'This indicates a bug in the strategy. ' 'This could either be a Hypothesis bug or ' 'an error in a function you\'ve passed to ' 'it to construct your data. Additionally, ' 'you have a custom executor, which means ' 'that this could be your executor failing ' 'to handle a function which returns None. ', HealthCheck.exception_in_generation, ) if overruns >= 20 or ( not count and overruns > 0 ): fail_health_check(( 'Examples routinely exceeded the max allowable size. ' '(%d examples overran while generating %d valid ones)' '. Generating examples this large will usually lead to' ' bad results. You should try setting average_size or ' 'max_size parameters on your collections and turning ' 'max_leaves down on recursive() calls.') % ( overruns, count ), HealthCheck.data_too_large) if filtered_draws >= 50 or ( not count and filtered_draws > 0 ): fail_health_check(( 'It looks like your strategy is filtering out a lot ' 'of data. Health check found %d filtered examples but ' 'only %d good ones. This will make your tests much ' 'slower, and also will probably distort the data ' 'generation quite a lot. You should adapt your ' 'strategy to filter less. This can also be caused by ' 'a low max_leaves parameter in recursive() calls') % ( filtered_draws, count ), HealthCheck.filter_too_much) runtime = time.time() - start if runtime > 1.0 or count < 10: fail_health_check(( 'Data generation is extremely slow: Only produced ' '%d valid examples in %.2f seconds (%d invalid ones ' 'and %d exceeded maximum size). Try decreasing ' "size of the data you're generating (with e.g." 'average_size or max_leaves parameters).' ) % (count, runtime, filtered_draws, overruns), HealthCheck.too_slow, ) last_exception = [None] repr_for_last_exception = [None] def evaluate_test_data(data): try: result = test_runner(data, reify_and_execute( search_strategy, test, )) if result is not None and settings.perform_health_check: fail_health_check(( 'Tests run under @given should return None, but ' '%s returned %r instead.' ) % (test.__name__, result), HealthCheck.return_value) return False except UnsatisfiedAssumption: data.mark_invalid() except ( HypothesisDeprecationWarning, FailedHealthCheck, StopTest, ): raise except Exception: last_exception[0] = traceback.format_exc() verbose_report(last_exception[0]) data.mark_interesting() from hypothesis.internal.conjecture.engine import ConjectureRunner falsifying_example = None database_key = str_to_bytes(fully_qualified_name(test)) start_time = time.time() runner = ConjectureRunner( evaluate_test_data, settings=settings, random=random, database_key=database_key, ) runner.run() note_engine_for_statistics(runner) run_time = time.time() - start_time timed_out = ( settings.timeout > 0 and run_time >= settings.timeout ) if runner.last_data is None: return if runner.last_data.status == Status.INTERESTING: falsifying_example = runner.last_data.buffer if settings.database is not None: settings.database.save( database_key, falsifying_example ) else: if runner.valid_examples < min( settings.min_satisfying_examples, settings.max_examples, ): if timed_out: raise Timeout(( 'Ran out of time before finding a satisfying ' 'example for ' '%s. Only found %d examples in ' + '%.2fs.' ) % ( get_pretty_function_description(test), runner.valid_examples, run_time )) else: raise Unsatisfiable(( 'Unable to satisfy assumptions of hypothesis ' '%s. Only %d examples considered ' 'satisfied assumptions' ) % ( get_pretty_function_description(test), runner.valid_examples,)) return assert last_exception[0] is not None try: with settings: test_runner( ConjectureData.for_buffer(falsifying_example), reify_and_execute( search_strategy, test, print_example=True, is_final=True )) except (UnsatisfiedAssumption, StopTest): report(traceback.format_exc()) raise Flaky( 'Unreliable assumption: An example which satisfied ' 'assumptions on the first run now fails it.' ) report( 'Failed to reproduce exception. Expected: \n' + last_exception[0], ) filter_message = ( 'Unreliable test data: Failed to reproduce a failure ' 'and then when it came to recreating the example in ' 'order to print the test data with a flaky result ' 'the example was filtered out (by e.g. a ' 'call to filter in your strategy) when we didn\'t ' 'expect it to be.' ) try: test_runner( ConjectureData.for_buffer(falsifying_example), reify_and_execute( search_strategy, test_is_flaky(test, repr_for_last_exception[0]), print_example=True, is_final=True )) except (UnsatisfiedAssumption, StopTest): raise Flaky(filter_message)
class StateMachineTestCase(TestCase): settings = Settings( min_satisfying_examples=1 )
def run_state_machine_as_test(state_machine_factory, *, settings=None): """Run a state machine definition as a test, either silently doing nothing or printing a minimal breaking program and raising an exception. state_machine_factory is anything which returns an instance of RuleBasedStateMachine when called with no arguments - it can be a class or a function. settings will be used to control the execution of the test. """ if settings is None: try: settings = state_machine_factory.TestCase.settings check_type(Settings, settings, "state_machine_factory.TestCase.settings") except AttributeError: settings = Settings(deadline=None, suppress_health_check=HealthCheck.all()) check_type(Settings, settings, "settings") @settings @given(st.data()) def run_state_machine(factory, data): cd = data.conjecture_data machine = factory() check_type(RuleBasedStateMachine, machine, "state_machine_factory()") cd.hypothesis_runner = machine print_steps = (current_build_context().is_final or current_verbosity() >= Verbosity.debug) try: if print_steps: report("state = %s()" % (machine.__class__.__name__, )) machine.check_invariants() max_steps = settings.stateful_step_count steps_run = 0 while True: # We basically always want to run the maximum number of steps, # but need to leave a small probability of terminating early # in order to allow for reducing the number of steps once we # find a failing test case, so we stop with probability of # 2 ** -16 during normal operation but force a stop when we've # generated enough steps. cd.start_example(STATE_MACHINE_RUN_LABEL) if steps_run == 0: cd.draw_bits(16, forced=1) elif steps_run >= max_steps: cd.draw_bits(16, forced=0) break else: # All we really care about is whether this value is zero # or non-zero, so if it's > 1 we discard it and insert a # replacement value after cd.start_example(SHOULD_CONTINUE_LABEL) should_continue_value = cd.draw_bits(16) if should_continue_value > 1: cd.stop_example(discard=True) cd.draw_bits(16, forced=int(bool(should_continue_value))) else: cd.stop_example() if should_continue_value == 0: break steps_run += 1 # Choose a rule to run, preferring an initialize rule if there are # any which have not been run yet. if machine._initialize_rules_to_run: init_rules = [ st.tuples(st.just(rule), st.fixed_dictionaries(rule.arguments)) for rule in machine._initialize_rules_to_run ] rule, data = cd.draw(st.one_of(init_rules)) machine._initialize_rules_to_run.remove(rule) else: rule, data = cd.draw(machine._rules_strategy) # Pretty-print the values this rule was called with *before* calling # _add_result_to_targets, to avoid printing arguments which are also # a return value using the variable name they are assigned to. # See https://github.com/HypothesisWorks/hypothesis/issues/2341 if print_steps: data_to_print = { k: machine._pretty_print(v) for k, v in data.items() } # Assign 'result' here in case executing the rule fails below result = multiple() try: data = dict(data) for k, v in list(data.items()): if isinstance(v, VarReference): data[k] = machine.names_to_values[v.name] result = rule.function(machine, **data) if rule.targets: if isinstance(result, MultipleResults): for single_result in result.values: machine._add_result_to_targets( rule.targets, single_result) else: machine._add_result_to_targets( rule.targets, result) finally: if print_steps: # 'result' is only used if the step has target bundles. # If it does, and the result is a 'MultipleResult', # then 'print_step' prints a multi-variable assignment. machine._print_step(rule, data_to_print, result) machine.check_invariants() cd.stop_example() finally: if print_steps: report("state.teardown()") machine.teardown() # Use a machine digest to identify stateful tests in the example database run_state_machine.hypothesis.inner_test._hypothesis_internal_add_digest = function_digest( state_machine_factory) # Copy some attributes so @seed and @reproduce_failure "just work" run_state_machine._hypothesis_internal_use_seed = getattr( state_machine_factory, "_hypothesis_internal_use_seed", None) run_state_machine._hypothesis_internal_use_reproduce_failure = getattr( state_machine_factory, "_hypothesis_internal_use_reproduce_failure", None) run_state_machine._hypothesis_internal_print_given_args = False run_state_machine(state_machine_factory)
def run_state_machine_as_test(state_machine_factory, settings=None): """Run a state machine definition as a test, either silently doing nothing or printing a minimal breaking program and raising an exception. state_machine_factory is anything which returns an instance of GenericStateMachine when called with no arguments - it can be a class or a function. settings will be used to control the execution of the test. """ if settings is None: try: settings = state_machine_factory.TestCase.settings check_type(Settings, settings, "state_machine_factory.TestCase.settings") except AttributeError: settings = Settings(deadline=None, suppress_health_check=HealthCheck.all()) check_type(Settings, settings, "settings") @settings @given(st.data()) def run_state_machine(factory, data): machine = factory() if isinstance(machine, GenericStateMachine) and not isinstance( machine, RuleBasedStateMachine): note_deprecation( "%s inherits from GenericStateMachine, which is deprecated. Use a " "RuleBasedStateMachine, or a test function with st.data(), instead." % (type(machine).__name__, ), since="2019-05-29", ) else: check_type(RuleBasedStateMachine, machine, "state_machine_factory()") data.conjecture_data.hypothesis_runner = machine n_steps = settings.stateful_step_count should_continue = cu.many(data.conjecture_data, min_size=1, max_size=n_steps, average_size=n_steps) print_steps = (current_build_context().is_final or current_verbosity() >= Verbosity.debug) try: if print_steps: machine.print_start() machine.check_invariants() while should_continue.more(): value = data.conjecture_data.draw(machine.steps()) # Assign 'result' here in case 'execute_step' fails below result = multiple() try: result = machine.execute_step(value) finally: if print_steps: # 'result' is only used if the step has target bundles. # If it does, and the result is a 'MultipleResult', # then 'print_step' prints a multi-variable assignment. machine.print_step(value, result) machine.check_invariants() finally: if print_steps: machine.print_end() machine.teardown() # Use a machine digest to identify stateful tests in the example database run_state_machine.hypothesis.inner_test._hypothesis_internal_add_digest = function_digest( state_machine_factory) # Copy some attributes so @seed and @reproduce_failure "just work" run_state_machine._hypothesis_internal_use_seed = getattr( state_machine_factory, "_hypothesis_internal_use_seed", None) run_state_machine._hypothesis_internal_use_reproduce_failure = getattr( state_machine_factory, "_hypothesis_internal_use_reproduce_failure", None) run_state_machine._hypothesis_internal_print_given_args = False run_state_machine(state_machine_factory)