def run_state_machine_as_test(state_machine_factory, settings=None):
"""Run a state machine definition as a test, either silently doing nothing
or printing a minimal breaking program and raising an exception.
state_machine_factory is anything which returns an instance of
GenericStateMachine when called with no arguments - it can be a class or a
function. settings will be used to control the execution of the test.
"""
if settings is None:
try:
settings = state_machine_factory.TestCase.settings
check_type(Settings, settings,
"state_machine_factory.TestCase.settings")
except AttributeError:
settings = Settings(deadline=None,
suppress_health_check=HealthCheck.all())
check_type(Settings, settings, "settings")
@settings
@given(st.data())
def run_state_machine(factory, data):
machine = factory()
check_type(GenericStateMachine, machine, "state_machine_factory()")
data.conjecture_data.hypothesis_runner = machine
n_steps = settings.stateful_step_count
should_continue = cu.many(data.conjecture_data,
min_size=1,
max_size=n_steps,
average_size=n_steps)
print_steps = (current_build_context().is_final
or current_verbosity() >= Verbosity.debug)
try:
if print_steps:
machine.print_start()
machine.check_invariants()
while should_continue.more():
value = data.conjecture_data.draw(machine.steps())
if print_steps:
machine.print_step(value)
machine.execute_step(value)
machine.check_invariants()
finally:
if print_steps:
machine.print_end()
machine.teardown()
# Use a machine digest to identify stateful tests in the example database
run_state_machine.hypothesis.inner_test._hypothesis_internal_add_digest = function_digest(
state_machine_factory)
# Copy some attributes so @seed and @reproduce_failure "just work"
run_state_machine._hypothesis_internal_use_seed = getattr(
state_machine_factory, "_hypothesis_internal_use_seed", None)
run_state_machine._hypothesis_internal_use_reproduce_failure = getattr(
state_machine_factory, "_hypothesis_internal_use_reproduce_failure",
None)
run_state_machine(state_machine_factory)
def test_digest_is_stable_across_process_runs():
# Hard coded as the only sensible way to check this doesn't change between
# process runs. There's nothing special about these values. If you update
# the code just update them to match.
digest = function_digest(test_digests_are_reasonably_unique)
print(repr(digest))
assert digest == b'\x8d\x07\xdb\xe1\xbeC\x92\xec-\xb4PWj\x0c%\x87'
def run_state_machine_as_test(state_machine_factory, settings=None):
"""Run a state machine definition as a test, either silently doing nothing
or printing a minimal breaking program and raising an exception.
state_machine_factory is anything which returns an instance of
GenericStateMachine when called with no arguments - it can be a class or a
function. settings will be used to control the execution of the test.
"""
if settings is None:
try:
settings = state_machine_factory.TestCase.settings
check_type(Settings, settings, "state_machine_factory.TestCase.settings")
except AttributeError:
settings = Settings(deadline=None, suppress_health_check=HealthCheck.all())
check_type(Settings, settings, "settings")
@settings
@given(st.data())
def run_state_machine(factory, data):
machine = factory()
check_type(GenericStateMachine, machine, "state_machine_factory()")
data.conjecture_data.hypothesis_runner = machine
n_steps = settings.stateful_step_count
should_continue = cu.many(
data.conjecture_data, min_size=1, max_size=n_steps, average_size=n_steps
)
print_steps = (
current_build_context().is_final or current_verbosity() >= Verbosity.debug
)
try:
if print_steps:
machine.print_start()
machine.check_invariants()
while should_continue.more():
value = data.conjecture_data.draw(machine.steps())
if print_steps:
machine.print_step(value)
machine.execute_step(value)
machine.check_invariants()
finally:
if print_steps:
machine.print_end()
machine.teardown()
# Use a machine digest to identify stateful tests in the example database
run_state_machine.hypothesis.inner_test._hypothesis_internal_add_digest = function_digest(
state_machine_factory
)
# Copy some attributes so @seed and @reproduce_failure "just work"
run_state_machine._hypothesis_internal_use_seed = getattr(
state_machine_factory, "_hypothesis_internal_use_seed", None
)
run_state_machine._hypothesis_internal_use_reproduce_failure = getattr(
state_machine_factory, "_hypothesis_internal_use_reproduce_failure", None
)
run_state_machine(state_machine_factory)
def find(specifier, condition, settings=None, random=None, storage=None):
settings = settings or Settings(
max_examples=2000,
min_satisfying_examples=0,
max_shrinks=2000,
)
search = strategy(specifier, settings)
if storage is None and settings.database is not None:
storage = settings.database.storage(
u'find(%s)' %
(binascii.hexlify(function_digest(condition)).decode(u'ascii'), ))
random = random or Random()
successful_examples = [0]
def template_condition(template):
with BuildContext():
result = search.reify(template)
success = condition(result)
if success:
successful_examples[0] += 1
if not successful_examples[0]:
verbose_report(lambda: u'Trying example %s' % (repr(result), ))
elif success:
if successful_examples[0] == 1:
verbose_report(lambda: u'Found satisfying example %s' %
(repr(result), ))
else:
verbose_report(lambda: u'Shrunk example to %s' %
(repr(result), ))
return success
template_condition.__name__ = condition.__name__
tracker = Tracker()
try:
template = best_satisfying_template(
search,
random,
template_condition,
settings,
tracker=tracker,
max_parameter_tries=2,
storage=storage,
)
with BuildContext():
return search.reify(template)
except Timeout:
raise
except NoSuchExample:
if search.template_upper_bound <= len(tracker):
raise DefinitelyNoSuchExample(
get_pretty_function_description(condition),
search.template_upper_bound,
)
raise NoSuchExample(get_pretty_function_description(condition))
def find(specifier, condition, settings=None, random=None, storage=None):
settings = settings or Settings(
max_examples=2000,
min_satisfying_examples=0,
max_shrinks=2000,
)
search = strategy(specifier, settings)
if storage is None and settings.database is not None:
storage = settings.database.storage(
'find(%s)' % (
binascii.hexlify(function_digest(condition)).decode('ascii'),
)
)
random = random or Random()
successful_examples = [0]
def template_condition(template):
result = search.reify(template)
success = condition(result)
if success:
successful_examples[0] += 1
if not successful_examples[0]:
verbose_report(lambda: 'Trying example %s' % (
repr(result),
))
elif success:
if successful_examples[0] == 1:
verbose_report(lambda: 'Found satisfying example %s' % (
repr(result),
))
else:
verbose_report(lambda: 'Shrunk example to %s' % (
repr(result),
))
return success
template_condition.__name__ = condition.__name__
tracker = Tracker()
try:
return search.reify(best_satisfying_template(
search, random, template_condition, settings,
tracker=tracker, max_parameter_tries=2,
storage=storage,
))
except Timeout:
raise
except NoSuchExample:
if search.template_upper_bound <= len(tracker):
raise DefinitelyNoSuchExample(
get_pretty_function_description(condition),
search.template_upper_bound,
)
raise NoSuchExample(get_pretty_function_description(condition))
def get_random_for_wrapped_test(test, wrapped_test):
settings = wrapped_test._hypothesis_internal_use_settings
if wrapped_test._hypothesis_internal_use_seed is not None:
return Random(wrapped_test._hypothesis_internal_use_seed)
elif settings.derandomize:
return Random(function_digest(test))
else:
return new_random()
def get_random_for_wrapped_test(test, wrapped_test):
settings = wrapped_test._hypothesis_internal_use_settings
wrapped_test._hypothesis_internal_use_generated_seed = None
if wrapped_test._hypothesis_internal_use_seed is not None:
return Random(wrapped_test._hypothesis_internal_use_seed)
elif settings.derandomize:
return Random(int_from_bytes(function_digest(test)))
elif global_force_seed is not None:
return Random(global_force_seed)
else:
seed = rnd_module.getrandbits(128)
wrapped_test._hypothesis_internal_use_generated_seed = seed
return Random(seed)
def get_random_for_wrapped_test(test, wrapped_test):
settings = wrapped_test._hypothesis_internal_use_settings
wrapped_test._hypothesis_internal_use_generated_seed = None
if wrapped_test._hypothesis_internal_use_seed is not None:
return Random(wrapped_test._hypothesis_internal_use_seed)
elif settings.derandomize:
return Random(int_from_bytes(function_digest(test)))
elif global_force_seed is not None:
return Random(global_force_seed)
else:
seed = rnd_module.getrandbits(128)
wrapped_test._hypothesis_internal_use_generated_seed = seed
return Random(seed)
def fuzz_one_input(
buffer: Union[bytes, bytearray, memoryview, BinaryIO]
) -> Optional[bytes]:
"""Run the test as a fuzz target, driven with the `buffer` of bytes.
Returns None if buffer invalid for the strategy, canonical pruned
bytes if the buffer was valid, and leaves raised exceptions alone.
Note: this feature is experimental and may change or be removed.
"""
if isinstance(buffer, io.IOBase):
buffer = buffer.read()
if isinstance(buffer, (bytearray, memoryview)):
buffer = bytes(buffer)
assert isinstance(buffer, bytes)
test = wrapped_test.hypothesis.inner_test
settings = wrapped_test._hypothesis_internal_use_settings
random = get_random_for_wrapped_test(test, wrapped_test)
_args, _kwargs, test_runner, search_strategy = process_arguments_to_given(
wrapped_test,
(),
{},
given_kwargs,
argspec,
settings,
)
assert not _args
assert not _kwargs
state = StateForActualGivenExecution(
test_runner,
search_strategy,
test,
settings,
random,
wrapped_test,
)
data = ConjectureData.for_buffer(buffer)
try:
state.execute_once(data)
except (StopTest, UnsatisfiedAssumption):
return None
except BaseException:
if settings.database is not None:
settings.database.save(function_digest(test),
bytes(data.buffer))
raise
return bytes(data.buffer)
def find(
specifier: SearchStrategy[Ex],
condition: Callable[[Any], bool],
*,
settings: Settings = None,
random: Random = None,
database_key: bytes = None
) -> Ex:
"""Returns the minimal example from the given strategy ``specifier`` that
matches the predicate function ``condition``."""
if settings is None:
settings = Settings(max_examples=2000)
settings = Settings(
settings, suppress_health_check=HealthCheck.all(), report_multiple_bugs=False
)
if database_key is None and settings.database is not None:
database_key = function_digest(condition)
if not isinstance(specifier, SearchStrategy):
raise InvalidArgument(
"Expected SearchStrategy but got %r of type %s"
% (specifier, type(specifier).__name__)
)
specifier.validate()
last = [] # type: List[Ex]
@settings
@given(specifier)
def test(v):
if condition(v):
last[:] = [v]
raise Found()
if random is not None:
test = seed(random.getrandbits(64))(test)
test._hypothesis_internal_is_find = True
test._hypothesis_internal_database_key = database_key
try:
test()
except Found:
return last[0]
raise NoSuchExample(get_pretty_function_description(condition))
def find(specifier, condition, settings=None, random=None, storage=None):
settings = settings or Settings(max_examples=2000, min_satisfying_examples=0, max_shrinks=2000)
from hypothesis.internal.strategymethod import strategy
search = strategy(specifier, settings)
if storage is None and settings.database is not None:
storage = settings.database.storage(
u"find(%s)" % (binascii.hexlify(function_digest(condition)).decode(u"ascii"),)
)
random = random or new_random()
successful_examples = [0]
def template_condition(template):
with BuildContext():
result = search.reify(template)
success = condition(result)
if success:
successful_examples[0] += 1
if not successful_examples[0]:
verbose_report(lambda: u"Trying example %s" % (repr(result),))
elif success:
if successful_examples[0] == 1:
verbose_report(lambda: u"Found satisfying example %s" % (repr(result),))
else:
verbose_report(lambda: u"Shrunk example to %s" % (repr(result),))
return success
template_condition.__name__ = condition.__name__
tracker = Tracker()
try:
template = best_satisfying_template(
search, random, template_condition, settings, tracker=tracker, max_parameter_tries=2, storage=storage
)
with BuildContext(is_final=True, close_on_capture=False):
return search.reify(template)
except Timeout:
raise
except NoSuchExample:
if search.template_upper_bound <= len(tracker):
raise DefinitelyNoSuchExample(get_pretty_function_description(condition), search.template_upper_bound)
raise NoSuchExample(get_pretty_function_description(condition))
def run_test_with_generator(test):
if settings.derandomize:
assert provided_random is None
random = Random(
function_digest(test)
)
else:
random = provided_random or Random()
original_argspec = inspect.getargspec(test)
if generator_arguments and original_argspec.varargs:
raise InvalidArgument(
'varargs are not supported with positional arguments to @given'
)
extra_kwargs = [
k for k in generator_kwargs if k not in original_argspec.args]
if extra_kwargs and not original_argspec.keywords:
raise InvalidArgument(
'%s() got an unexpected keyword argument %r' % (
test.__name__,
extra_kwargs[0]
))
if (
len(generator_arguments) > len(original_argspec.args)
):
raise InvalidArgument((
'Too many positional arguments for %s() (got %d but'
' expected at most %d') % (
test.__name__, len(generator_arguments),
len(original_argspec.args)))
arguments = original_argspec.args
specifiers = list(generator_arguments)
seen_kwarg = None
for a in arguments:
if a in generator_kwargs:
seen_kwarg = seen_kwarg or a
specifiers.append(generator_kwargs[a])
else:
if seen_kwarg is not None:
raise InvalidArgument((
'Argument %s comes after keyword %s which has been '
'specified, but does not itself have a '
'specification') % (
a, seen_kwarg
))
argspec = inspect.ArgSpec(
args=arguments,
keywords=original_argspec.keywords,
varargs=original_argspec.varargs,
defaults=tuple(map(HypothesisProvided, specifiers))
)
unused_kwargs = {}
for k in extra_kwargs:
unused_kwargs[k] = HypothesisProvided(generator_kwargs[k])
@copy_argspec(
test.__name__, argspec
)
def wrapped_test(*arguments, **kwargs):
selfy = None
arguments, kwargs = convert_positional_arguments(
wrapped_test, arguments, kwargs)
# Anything in unused_kwargs hasn't been injected through
# argspec.defaults, so we need to add them.
for k in unused_kwargs:
if k not in kwargs:
kwargs[k] = unused_kwargs[k]
# If the test function is a method of some kind, the bound object
# will be the first named argument if there are any, otherwise the
# first vararg (if any).
if argspec.args:
selfy = kwargs.get(argspec.args[0])
elif arguments:
selfy = arguments[0]
if isinstance(selfy, HypothesisProvided):
selfy = None
test_runner = executor(selfy)
for example in getattr(
wrapped_test, 'hypothesis_explicit_examples', ()
):
if example.args:
example_kwargs = dict(zip(
argspec.args[-len(example.args):], example.args
))
else:
example_kwargs = dict(example.kwargs)
for k, v in kwargs.items():
if not isinstance(v, HypothesisProvided):
example_kwargs[k] = v
# Note: Test may mutate arguments and we can't rerun explicit
# examples, so we have to calculate the failure message at this
# point rather than than later.
message_on_failure = 'Falsifying example: %s(%s)' % (
test.__name__, arg_string(test, arguments, example_kwargs)
)
try:
test_runner(
lambda: test(*arguments, **example_kwargs)
)
except BaseException:
report(message_on_failure)
raise
if not any(
isinstance(x, HypothesisProvided)
for xs in (arguments, kwargs.values())
for x in xs
):
# All arguments have been satisfied without needing to invoke
# hypothesis
test_runner(lambda: test(*arguments, **kwargs))
return
def convert_to_specifier(v):
if isinstance(v, HypothesisProvided):
return strategy(v.value, settings)
else:
return sd.just(v)
given_specifier = sd.tuples(
sd.tuples(*map(convert_to_specifier, arguments)),
sd.fixed_dictionaries(dict(
(k, convert_to_specifier(v)) for (k, v) in kwargs.items()))
)
search_strategy = strategy(given_specifier, settings)
if settings.database:
storage = settings.database.storage(
fully_qualified_name(test))
else:
storage = None
last_exception = [None]
repr_for_last_exception = [None]
def is_template_example(xs):
record_repr = [None]
try:
test_runner(reify_and_execute(
search_strategy, xs, test,
always_print=settings.max_shrinks <= 0,
record_repr=record_repr,
))
return False
except UnsatisfiedAssumption as e:
raise e
except Exception as e:
if settings.max_shrinks <= 0:
raise e
last_exception[0] = traceback.format_exc()
repr_for_last_exception[0] = record_repr[0]
verbose_report(last_exception[0])
return True
is_template_example.__name__ = test.__name__
is_template_example.__qualname__ = qualname(test)
falsifying_template = None
try:
falsifying_template = best_satisfying_template(
search_strategy, random, is_template_example,
settings, storage
)
except NoSuchExample:
return
assert last_exception[0] is not None
with settings:
test_runner(reify_and_execute(
search_strategy, falsifying_template, test,
print_example=True
))
report(
'Failed to reproduce exception. Expected: \n' +
last_exception[0],
)
test_runner(reify_and_execute(
search_strategy, falsifying_template,
test_is_flaky(test, repr_for_last_exception[0]),
print_example=True
))
wrapped_test.__name__ = test.__name__
wrapped_test.__doc__ = test.__doc__
wrapped_test.is_hypothesis_test = True
wrapped_test.hypothesis_explicit_examples = getattr(
test, 'hypothesis_explicit_examples', []
)
return wrapped_test
def run_state_machine_as_test(state_machine_factory, settings=None):
"""Run a state machine definition as a test, either silently doing nothing
or printing a minimal breaking program and raising an exception.
state_machine_factory is anything which returns an instance of
RuleBasedStateMachine when called with no arguments - it can be a class or a
function. settings will be used to control the execution of the test.
"""
if settings is None:
try:
settings = state_machine_factory.TestCase.settings
check_type(Settings, settings,
"state_machine_factory.TestCase.settings")
except AttributeError:
settings = Settings(deadline=None,
suppress_health_check=HealthCheck.all())
check_type(Settings, settings, "settings")
@settings
@given(st.data())
def run_state_machine(factory, data):
machine = factory()
if not isinstance(machine, _GenericStateMachine):
raise InvalidArgument(
"Expected RuleBasedStateMachine but state_machine_factory() "
"returned %r (type=%s)" % (machine, type(machine).__name__))
data.conjecture_data.hypothesis_runner = machine
print_steps = (current_build_context().is_final
or current_verbosity() >= Verbosity.debug)
try:
if print_steps:
machine.print_start()
machine.check_invariants()
max_steps = settings.stateful_step_count
steps_run = 0
cd = data.conjecture_data
while True:
# We basically always want to run the maximum number of steps,
# but need to leave a small probability of terminating early
# in order to allow for reducing the number of steps once we
# find a failing test case, so we stop with probability of
# 2 ** -16 during normal operation but force a stop when we've
# generated enough steps.
cd.start_example(STATE_MACHINE_RUN_LABEL)
if steps_run == 0:
cd.draw_bits(16, forced=1)
elif steps_run >= max_steps:
cd.draw_bits(16, forced=0)
break
else:
# All we really care about is whether this value is zero
# or non-zero, so if it's > 1 we discard it and insert a
# replacement value after
cd.start_example(SHOULD_CONTINUE_LABEL)
should_continue_value = cd.draw_bits(16)
if should_continue_value > 1:
cd.stop_example(discard=True)
cd.draw_bits(16,
forced=int(bool(should_continue_value)))
else:
cd.stop_example()
if should_continue_value == 0:
break
steps_run += 1
value = data.conjecture_data.draw(machine.steps())
# Assign 'result' here in case 'execute_step' fails below
result = multiple()
try:
result = machine.execute_step(value)
finally:
if print_steps:
# 'result' is only used if the step has target bundles.
# If it does, and the result is a 'MultipleResult',
# then 'print_step' prints a multi-variable assignment.
machine.print_step(value, result)
machine.check_invariants()
data.conjecture_data.stop_example()
finally:
if print_steps:
machine.print_end()
machine.teardown()
# Use a machine digest to identify stateful tests in the example database
run_state_machine.hypothesis.inner_test._hypothesis_internal_add_digest = function_digest(
state_machine_factory)
# Copy some attributes so @seed and @reproduce_failure "just work"
run_state_machine._hypothesis_internal_use_seed = getattr(
state_machine_factory, "_hypothesis_internal_use_seed", None)
run_state_machine._hypothesis_internal_use_reproduce_failure = getattr(
state_machine_factory, "_hypothesis_internal_use_reproduce_failure",
None)
run_state_machine._hypothesis_internal_print_given_args = False
run_state_machine(state_machine_factory)
def test_can_digest_a_function_with_no_name():
def foo(x, y):
pass
function_digest(partial(foo, 1))
def test_can_digest_a_built_in_function():
import math
assert function_digest(math.isnan) != function_digest(range)
def test_digests_are_reasonably_unique():
assert function_digest(test_simple_conversion) != function_digest(
test_does_not_error_on_confused_sources)
def wrapped_test(*arguments, **kwargs):
settings = wrapped_test._hypothesis_internal_use_settings
if wrapped_test._hypothesis_internal_use_seed is not None:
random = Random(wrapped_test._hypothesis_internal_use_seed)
elif settings.derandomize:
random = Random(function_digest(test))
else:
random = new_random()
import hypothesis.strategies as sd
selfy = None
arguments, kwargs = convert_positional_arguments(wrapped_test, arguments, kwargs)
# If the test function is a method of some kind, the bound object
# will be the first named argument if there are any, otherwise the
# first vararg (if any).
if argspec.args:
selfy = kwargs.get(argspec.args[0])
elif arguments:
selfy = arguments[0]
test_runner = new_style_executor(selfy)
for example in reversed(getattr(wrapped_test, "hypothesis_explicit_examples", ())):
if example.args:
if len(example.args) > len(original_argspec.args):
raise InvalidArgument(
"example has too many arguments for test. "
"Expected at most %d but got %d" % (len(original_argspec.args), len(example.args))
)
example_kwargs = dict(zip(original_argspec.args[-len(example.args) :], example.args))
else:
example_kwargs = example.kwargs
if Phase.explicit not in settings.phases:
continue
example_kwargs.update(kwargs)
# Note: Test may mutate arguments and we can't rerun explicit
# examples, so we have to calculate the failure message at this
# point rather than than later.
message_on_failure = "Falsifying example: %s(%s)" % (
test.__name__,
arg_string(test, arguments, example_kwargs),
)
try:
with BuildContext() as b:
test_runner(None, lambda data: test(*arguments, **example_kwargs))
except BaseException:
report(message_on_failure)
for n in b.notes:
report(n)
raise
if settings.max_examples <= 0:
return
arguments = tuple(arguments)
given_specifier = sd.tuples(
sd.just(arguments), sd.fixed_dictionaries(generator_kwargs).map(lambda args: dict(args, **kwargs))
)
def fail_health_check(message, label):
if label in settings.suppress_health_check:
return
message += (
"\nSee https://hypothesis.readthedocs.io/en/latest/health"
"checks.html for more information about this. "
)
message += (
"If you want to disable just this health check, add %s "
"to the suppress_health_check settings for this test."
) % (label,)
raise FailedHealthCheck(message)
search_strategy = given_specifier
if selfy is not None:
search_strategy = WithRunner(search_strategy, selfy)
search_strategy.validate()
perform_health_check = settings.perform_health_check
perform_health_check &= Settings.default.perform_health_check
from hypothesis.internal.conjecture.data import TestData, Status, StopTest
if not (Phase.reuse in settings.phases or Phase.generate in settings.phases):
return
if perform_health_check:
health_check_random = Random(random.getrandbits(128))
# We "pre warm" the health check with one draw to give it some
# time to calculate any cached data. This prevents the case
# where the first draw of the health check takes ages because
# of loading unicode data the first time.
data = TestData(
max_length=settings.buffer_size,
draw_bytes=lambda data, n, distribution: distribution(health_check_random, n),
)
with Settings(settings, verbosity=Verbosity.quiet):
try:
test_runner(data, reify_and_execute(search_strategy, lambda *args, **kwargs: None))
except BaseException:
pass
count = 0
overruns = 0
filtered_draws = 0
start = time.time()
while count < 10 and time.time() < start + 1 and filtered_draws < 50 and overruns < 20:
try:
data = TestData(
max_length=settings.buffer_size,
draw_bytes=lambda data, n, distribution: distribution(health_check_random, n),
)
with Settings(settings, verbosity=Verbosity.quiet):
test_runner(data, reify_and_execute(search_strategy, lambda *args, **kwargs: None))
count += 1
except UnsatisfiedAssumption:
filtered_draws += 1
except StopTest:
if data.status == Status.INVALID:
filtered_draws += 1
else:
assert data.status == Status.OVERRUN
overruns += 1
except InvalidArgument:
raise
except Exception:
if HealthCheck.exception_in_generation in settings.suppress_health_check:
raise
report(traceback.format_exc())
if test_runner is default_new_style_executor:
fail_health_check(
"An exception occurred during data "
"generation in initial health check. "
"This indicates a bug in the strategy. "
"This could either be a Hypothesis bug or "
"an error in a function you've passed to "
"it to construct your data.",
HealthCheck.exception_in_generation,
)
else:
fail_health_check(
"An exception occurred during data "
"generation in initial health check. "
"This indicates a bug in the strategy. "
"This could either be a Hypothesis bug or "
"an error in a function you've passed to "
"it to construct your data. Additionally, "
"you have a custom executor, which means "
"that this could be your executor failing "
"to handle a function which returns None. ",
HealthCheck.exception_in_generation,
)
if overruns >= 20 or (not count and overruns > 0):
fail_health_check(
(
"Examples routinely exceeded the max allowable size. "
"(%d examples overran while generating %d valid ones)"
". Generating examples this large will usually lead to"
" bad results. You should try setting average_size or "
"max_size parameters on your collections and turning "
"max_leaves down on recursive() calls."
)
% (overruns, count),
HealthCheck.data_too_large,
)
if filtered_draws >= 50 or (not count and filtered_draws > 0):
fail_health_check(
(
"It looks like your strategy is filtering out a lot "
"of data. Health check found %d filtered examples but "
"only %d good ones. This will make your tests much "
"slower, and also will probably distort the data "
"generation quite a lot. You should adapt your "
"strategy to filter less. This can also be caused by "
"a low max_leaves parameter in recursive() calls"
)
% (filtered_draws, count),
HealthCheck.filter_too_much,
)
runtime = time.time() - start
if runtime > 1.0 or count < 10:
fail_health_check(
(
"Data generation is extremely slow: Only produced "
"%d valid examples in %.2f seconds (%d invalid ones "
"and %d exceeded maximum size). Try decreasing "
"size of the data you're generating (with e.g."
"average_size or max_leaves parameters)."
)
% (count, runtime, filtered_draws, overruns),
HealthCheck.too_slow,
)
last_exception = [None]
repr_for_last_exception = [None]
def evaluate_test_data(data):
try:
result = test_runner(data, reify_and_execute(search_strategy, test))
if result is not None and settings.perform_health_check:
fail_health_check(
("Tests run under @given should return None, but " "%s returned %r instead.")
% (test.__name__, result),
HealthCheck.return_value,
)
return False
except UnsatisfiedAssumption:
data.mark_invalid()
except (HypothesisDeprecationWarning, FailedHealthCheck, StopTest):
raise
except Exception:
last_exception[0] = traceback.format_exc()
verbose_report(last_exception[0])
data.mark_interesting()
from hypothesis.internal.conjecture.engine import TestRunner
falsifying_example = None
database_key = str_to_bytes(fully_qualified_name(test))
start_time = time.time()
runner = TestRunner(evaluate_test_data, settings=settings, random=random, database_key=database_key)
runner.run()
run_time = time.time() - start_time
timed_out = settings.timeout > 0 and run_time >= settings.timeout
if runner.last_data is None:
return
if runner.last_data.status == Status.INTERESTING:
falsifying_example = runner.last_data.buffer
if settings.database is not None:
settings.database.save(database_key, falsifying_example)
else:
if runner.valid_examples < min(settings.min_satisfying_examples, settings.max_examples):
if timed_out:
raise Timeout(
(
"Ran out of time before finding a satisfying "
"example for "
"%s. Only found %d examples in " + "%.2fs."
)
% (get_pretty_function_description(test), runner.valid_examples, run_time)
)
else:
raise Unsatisfiable(
(
"Unable to satisfy assumptions of hypothesis "
"%s. Only %d examples considered "
"satisfied assumptions"
)
% (get_pretty_function_description(test), runner.valid_examples)
)
return
assert last_exception[0] is not None
try:
with settings:
test_runner(
TestData.for_buffer(falsifying_example),
reify_and_execute(search_strategy, test, print_example=True, is_final=True),
)
except (UnsatisfiedAssumption, StopTest):
report(traceback.format_exc())
raise Flaky(
"Unreliable assumption: An example which satisfied " "assumptions on the first run now fails it."
)
report("Failed to reproduce exception. Expected: \n" + last_exception[0])
filter_message = (
"Unreliable test data: Failed to reproduce a failure "
"and then when it came to recreating the example in "
"order to print the test data with a flaky result "
"the example was filtered out (by e.g. a "
"call to filter in your strategy) when we didn't "
"expect it to be."
)
try:
test_runner(
TestData.for_buffer(falsifying_example),
reify_and_execute(
search_strategy,
test_is_flaky(test, repr_for_last_exception[0]),
print_example=True,
is_final=True,
),
)
except (UnsatisfiedAssumption, StopTest):
raise Flaky(filter_message)
def test_can_digest_a_unicode_lambda():
function_digest(lambda x: '☃' in str(x))
def wrapped_test(*arguments, **kwargs):
settings = wrapped_test._hypothesis_internal_use_settings
if wrapped_test._hypothesis_internal_use_seed is not None:
random = Random(
wrapped_test._hypothesis_internal_use_seed)
elif settings.derandomize:
random = Random(function_digest(test))
else:
random = new_random()
import hypothesis.strategies as sd
selfy = None
arguments, kwargs = convert_positional_arguments(
wrapped_test, arguments, kwargs)
# If the test function is a method of some kind, the bound object
# will be the first named argument if there are any, otherwise the
# first vararg (if any).
if argspec.args:
selfy = kwargs.get(argspec.args[0])
elif arguments:
selfy = arguments[0]
test_runner = new_style_executor(selfy)
for example in reversed(getattr(
wrapped_test, 'hypothesis_explicit_examples', ()
)):
if example.args:
if len(example.args) > len(original_argspec.args):
raise InvalidArgument(
'example has too many arguments for test. '
'Expected at most %d but got %d' % (
len(original_argspec.args), len(example.args)))
example_kwargs = dict(zip(
original_argspec.args[-len(example.args):],
example.args
))
else:
example_kwargs = example.kwargs
example_kwargs.update(kwargs)
# Note: Test may mutate arguments and we can't rerun explicit
# examples, so we have to calculate the failure message at this
# point rather than than later.
message_on_failure = 'Falsifying example: %s(%s)' % (
test.__name__, arg_string(test, arguments, example_kwargs)
)
try:
with BuildContext() as b:
test_runner(
None,
lambda data: test(*arguments, **example_kwargs)
)
except BaseException:
report(message_on_failure)
for n in b.notes:
report(n)
raise
if settings.max_examples <= 0:
return
arguments = tuple(arguments)
given_specifier = sd.tuples(
sd.just(arguments),
sd.fixed_dictionaries(generator_kwargs).map(
lambda args: dict(args, **kwargs)
)
)
def fail_health_check(message):
message += (
'\nSee http://hypothesis.readthedocs.org/en/latest/health'
'checks.html for more information about this.'
)
raise FailedHealthCheck(message)
search_strategy = given_specifier
search_strategy.validate()
perform_health_check = settings.perform_health_check
perform_health_check &= Settings.default.perform_health_check
from hypothesis.internal.conjecture.data import TestData, Status, \
StopTest
if perform_health_check:
initial_state = getglobalrandomstate()
health_check_random = Random(random.getrandbits(128))
# We "pre warm" the health check with one draw to give it some
# time to calculate any cached data. This prevents the case
# where the first draw of the health check takes ages because
# of loading unicode data the first time.
data = TestData(
max_length=settings.buffer_size,
draw_bytes=lambda data, n, distribution:
distribution(health_check_random, n)
)
with Settings(settings, verbosity=Verbosity.quiet):
try:
test_runner(data, reify_and_execute(
search_strategy,
lambda *args, **kwargs: None,
))
except BaseException:
pass
count = 0
overruns = 0
filtered_draws = 0
start = time.time()
while (
count < 10 and time.time() < start + 1 and
filtered_draws < 50 and overruns < 20
):
try:
data = TestData(
max_length=settings.buffer_size,
draw_bytes=lambda data, n, distribution:
distribution(health_check_random, n)
)
with Settings(settings, verbosity=Verbosity.quiet):
test_runner(data, reify_and_execute(
search_strategy,
lambda *args, **kwargs: None,
))
count += 1
except UnsatisfiedAssumption:
filtered_draws += 1
except StopTest:
if data.status == Status.INVALID:
filtered_draws += 1
else:
assert data.status == Status.OVERRUN
overruns += 1
except Exception:
report(traceback.format_exc())
if test_runner is default_new_style_executor:
fail_health_check(
'An exception occurred during data '
'generation in initial health check. '
'This indicates a bug in the strategy. '
'This could either be a Hypothesis bug or '
"an error in a function yo've passed to "
'it to construct your data.'
)
else:
fail_health_check(
'An exception occurred during data '
'generation in initial health check. '
'This indicates a bug in the strategy. '
'This could either be a Hypothesis bug or '
'an error in a function you\'ve passed to '
'it to construct your data. Additionally, '
'you have a custom executor, which means '
'that this could be your executor failing '
'to handle a function which returns None. '
)
if overruns >= 20 or (
not count and overruns > 0
):
fail_health_check((
'Examples routinely exceeded the max allowable size. '
'(%d examples overran while generating %d valid ones)'
'. Generating examples this large will usually lead to'
' bad results. You should try setting average_size or '
'max_size parameters on your collections and turning '
'max_leaves down on recursive() calls.') % (
overruns, count
))
if filtered_draws >= 50 or (
not count and filtered_draws > 0
):
fail_health_check((
'It looks like your strategy is filtering out a lot '
'of data. Health check found %d filtered examples but '
'only %d good ones. This will make your tests much '
'slower, and also will probably distort the data '
'generation quite a lot. You should adapt your '
'strategy to filter less. This can also be caused by '
'a low max_leaves parameter in recursive() calls') % (
filtered_draws, count
))
runtime = time.time() - start
if runtime > 1.0 or count < 10:
fail_health_check((
'Data generation is extremely slow: Only produced '
'%d valid examples in %.2f seconds (%d invalid ones '
'and %d exceeded maximum size). Try decreasing '
"size of the data you're generating (with e.g."
'average_size or max_leaves parameters).'
) % (count, runtime, filtered_draws, overruns))
if getglobalrandomstate() != initial_state:
fail_health_check(
'Data generation depends on global random module. '
'This makes results impossible to replay, which '
'prevents Hypothesis from working correctly. '
'If you want to use methods from random, use '
'randoms() from hypothesis.strategies to get an '
'instance of Random you can use. Alternatively, you '
'can use the random_module() strategy to explicitly '
'seed the random module.'
)
last_exception = [None]
repr_for_last_exception = [None]
performed_random_check = [False]
def evaluate_test_data(data):
if perform_health_check and not performed_random_check[0]:
initial_state = getglobalrandomstate()
performed_random_check[0] = True
else:
initial_state = None
try:
result = test_runner(data, reify_and_execute(
search_strategy, test,
))
if result is not None and settings.perform_health_check:
raise FailedHealthCheck((
'Tests run under @given should return None, but '
'%s returned %r instead.'
) % (test.__name__, result), settings)
return False
except UnsatisfiedAssumption:
data.mark_invalid()
except (
HypothesisDeprecationWarning, FailedHealthCheck,
StopTest,
):
raise
except Exception:
last_exception[0] = traceback.format_exc()
verbose_report(last_exception[0])
data.mark_interesting()
finally:
if (
initial_state is not None and
getglobalrandomstate() != initial_state
):
fail_health_check(
'Your test used the global random module. '
'This is unlikely to work correctly. You should '
'consider using the randoms() strategy from '
'hypothesis.strategies instead. Alternatively, '
'you can use the random_module() strategy to '
'explicitly seed the random module.')
from hypothesis.internal.conjecture.engine import TestRunner
falsifying_example = None
database_key = str_to_bytes(fully_qualified_name(test))
start_time = time.time()
runner = TestRunner(
evaluate_test_data,
settings=settings, random=random,
database_key=database_key,
)
runner.run()
run_time = time.time() - start_time
timed_out = (
settings.timeout > 0 and
run_time >= settings.timeout
)
if runner.last_data.status == Status.INTERESTING:
falsifying_example = runner.last_data.buffer
if settings.database is not None:
settings.database.save(
database_key, falsifying_example
)
else:
if runner.valid_examples < min(
settings.min_satisfying_examples,
settings.max_examples,
):
if timed_out:
raise Timeout((
'Ran out of time before finding a satisfying '
'example for '
'%s. Only found %d examples in ' +
'%.2fs.'
) % (
get_pretty_function_description(test),
runner.valid_examples, run_time
))
else:
raise Unsatisfiable((
'Unable to satisfy assumptions of hypothesis '
'%s. Only %d examples considered '
'satisfied assumptions'
) % (
get_pretty_function_description(test),
runner.valid_examples,))
return
assert last_exception[0] is not None
try:
with settings:
test_runner(
TestData.for_buffer(falsifying_example),
reify_and_execute(
search_strategy, test,
print_example=True, is_final=True
))
except (UnsatisfiedAssumption, StopTest):
report(traceback.format_exc())
raise Flaky(
'Unreliable assumption: An example which satisfied '
'assumptions on the first run now fails it.'
)
report(
'Failed to reproduce exception. Expected: \n' +
last_exception[0],
)
filter_message = (
'Unreliable test data: Failed to reproduce a failure '
'and then when it came to recreating the example in '
'order to print the test data with a flaky result '
'the example was filtered out (by e.g. a '
'call to filter in your strategy) when we didn\'t '
'expect it to be.'
)
try:
test_runner(
TestData.for_buffer(falsifying_example),
reify_and_execute(
search_strategy,
test_is_flaky(test, repr_for_last_exception[0]),
print_example=True, is_final=True
))
except (UnsatisfiedAssumption, StopTest):
raise Flaky(filter_message)
def find(specifier, condition, settings=None, random=None, database_key=None):
settings = settings or Settings(
max_examples=2000,
min_satisfying_examples=0,
max_shrinks=2000,
)
if database_key is None and settings.database is not None:
database_key = function_digest(condition)
if not isinstance(specifier, SearchStrategy):
raise InvalidArgument(
'Expected SearchStrategy but got %r of type %s' % (
specifier, type(specifier).__name__
))
specifier.validate()
search = specifier
random = random or new_random()
successful_examples = [0]
last_data = [None]
def template_condition(data):
with BuildContext(data):
try:
data.is_find = True
result = data.draw(search)
data.note(result)
success = condition(result)
except UnsatisfiedAssumption:
data.mark_invalid()
if success:
successful_examples[0] += 1
if settings.verbosity == Verbosity.verbose:
if not successful_examples[0]:
report(lambda: u'Trying example %s' % (
nicerepr(result),
))
elif success:
if successful_examples[0] == 1:
report(lambda: u'Found satisfying example %s' % (
nicerepr(result),
))
else:
report(lambda: u'Shrunk example to %s' % (
nicerepr(result),
))
last_data[0] = data
if success and not data.frozen:
data.mark_interesting()
from hypothesis.internal.conjecture.engine import ConjectureRunner
from hypothesis.internal.conjecture.data import ConjectureData, Status
start = time.time()
runner = ConjectureRunner(
template_condition, settings=settings, random=random,
database_key=database_key,
)
runner.run()
note_engine_for_statistics(runner)
run_time = time.time() - start
if runner.last_data.status == Status.INTERESTING:
data = ConjectureData.for_buffer(runner.last_data.buffer)
with BuildContext(data):
return data.draw(search)
if runner.valid_examples <= settings.min_satisfying_examples:
if settings.timeout > 0 and run_time > settings.timeout:
raise Timeout((
'Ran out of time before finding enough valid examples for '
'%s. Only %d valid examples found in %.2f seconds.'
) % (
get_pretty_function_description(condition),
runner.valid_examples, run_time))
else:
raise Unsatisfiable((
'Unable to satisfy assumptions of '
'%s. Only %d examples considered satisfied assumptions'
) % (
get_pretty_function_description(condition),
runner.valid_examples,))
raise NoSuchExample(get_pretty_function_description(condition))
def wrapped_test(*arguments, **kwargs):
settings = wrapped_test._hypothesis_internal_use_settings
if wrapped_test._hypothesis_internal_use_seed is not None:
random = Random(
wrapped_test._hypothesis_internal_use_seed)
elif settings.derandomize:
random = Random(function_digest(test))
else:
random = new_random()
import hypothesis.strategies as sd
selfy = None
arguments, kwargs = convert_positional_arguments(
wrapped_test, arguments, kwargs)
# If the test function is a method of some kind, the bound object
# will be the first named argument if there are any, otherwise the
# first vararg (if any).
if argspec.args:
selfy = kwargs.get(argspec.args[0])
elif arguments:
selfy = arguments[0]
test_runner = new_style_executor(selfy)
for example in reversed(getattr(
wrapped_test, 'hypothesis_explicit_examples', ()
)):
if example.args:
if len(example.args) > len(original_argspec.args):
raise InvalidArgument(
'example has too many arguments for test. '
'Expected at most %d but got %d' % (
len(original_argspec.args), len(example.args)))
example_kwargs = dict(zip(
original_argspec.args[-len(example.args):],
example.args
))
else:
example_kwargs = example.kwargs
if Phase.explicit not in settings.phases:
continue
example_kwargs.update(kwargs)
# Note: Test may mutate arguments and we can't rerun explicit
# examples, so we have to calculate the failure message at this
# point rather than than later.
message_on_failure = 'Falsifying example: %s(%s)' % (
test.__name__, arg_string(test, arguments, example_kwargs)
)
try:
with BuildContext(None) as b:
test_runner(
None,
lambda data: test(*arguments, **example_kwargs)
)
except BaseException:
traceback.print_exc()
report(message_on_failure)
for n in b.notes:
report(n)
raise
if settings.max_examples <= 0:
return
arguments = tuple(arguments)
given_specifier = sd.tuples(
sd.just(arguments),
sd.fixed_dictionaries(generator_kwargs).map(
lambda args: dict(args, **kwargs)
)
)
def fail_health_check(message, label):
if label in settings.suppress_health_check:
return
message += (
'\nSee https://hypothesis.readthedocs.io/en/latest/health'
'checks.html for more information about this. '
)
message += (
'If you want to disable just this health check, add %s '
'to the suppress_health_check settings for this test.'
) % (label,)
raise FailedHealthCheck(message)
search_strategy = given_specifier
if selfy is not None:
search_strategy = WithRunner(search_strategy, selfy)
search_strategy.validate()
perform_health_check = settings.perform_health_check
perform_health_check &= Settings.default.perform_health_check
from hypothesis.internal.conjecture.data import ConjectureData, \
Status, StopTest
if not (
Phase.reuse in settings.phases or
Phase.generate in settings.phases
):
return
if perform_health_check:
health_check_random = Random(random.getrandbits(128))
# We "pre warm" the health check with one draw to give it some
# time to calculate any cached data. This prevents the case
# where the first draw of the health check takes ages because
# of loading unicode data the first time.
data = ConjectureData(
max_length=settings.buffer_size,
draw_bytes=lambda data, n, distribution:
distribution(health_check_random, n)
)
with Settings(settings, verbosity=Verbosity.quiet):
try:
test_runner(data, reify_and_execute(
search_strategy,
lambda *args, **kwargs: None,
))
except BaseException:
pass
count = 0
overruns = 0
filtered_draws = 0
start = time.time()
while (
count < 10 and time.time() < start + 1 and
filtered_draws < 50 and overruns < 20
):
try:
data = ConjectureData(
max_length=settings.buffer_size,
draw_bytes=lambda data, n, distribution:
distribution(health_check_random, n)
)
with Settings(settings, verbosity=Verbosity.quiet):
test_runner(data, reify_and_execute(
search_strategy,
lambda *args, **kwargs: None,
))
count += 1
except UnsatisfiedAssumption:
filtered_draws += 1
except StopTest:
if data.status == Status.INVALID:
filtered_draws += 1
else:
assert data.status == Status.OVERRUN
overruns += 1
except InvalidArgument:
raise
except Exception:
if (
HealthCheck.exception_in_generation in
settings.suppress_health_check
):
raise
report(traceback.format_exc())
if test_runner is default_new_style_executor:
fail_health_check(
'An exception occurred during data '
'generation in initial health check. '
'This indicates a bug in the strategy. '
'This could either be a Hypothesis bug or '
"an error in a function you've passed to "
'it to construct your data.',
HealthCheck.exception_in_generation,
)
else:
fail_health_check(
'An exception occurred during data '
'generation in initial health check. '
'This indicates a bug in the strategy. '
'This could either be a Hypothesis bug or '
'an error in a function you\'ve passed to '
'it to construct your data. Additionally, '
'you have a custom executor, which means '
'that this could be your executor failing '
'to handle a function which returns None. ',
HealthCheck.exception_in_generation,
)
if overruns >= 20 or (
not count and overruns > 0
):
fail_health_check((
'Examples routinely exceeded the max allowable size. '
'(%d examples overran while generating %d valid ones)'
'. Generating examples this large will usually lead to'
' bad results. You should try setting average_size or '
'max_size parameters on your collections and turning '
'max_leaves down on recursive() calls.') % (
overruns, count
), HealthCheck.data_too_large)
if filtered_draws >= 50 or (
not count and filtered_draws > 0
):
fail_health_check((
'It looks like your strategy is filtering out a lot '
'of data. Health check found %d filtered examples but '
'only %d good ones. This will make your tests much '
'slower, and also will probably distort the data '
'generation quite a lot. You should adapt your '
'strategy to filter less. This can also be caused by '
'a low max_leaves parameter in recursive() calls') % (
filtered_draws, count
), HealthCheck.filter_too_much)
runtime = time.time() - start
if runtime > 1.0 or count < 10:
fail_health_check((
'Data generation is extremely slow: Only produced '
'%d valid examples in %.2f seconds (%d invalid ones '
'and %d exceeded maximum size). Try decreasing '
"size of the data you're generating (with e.g."
'average_size or max_leaves parameters).'
) % (count, runtime, filtered_draws, overruns),
HealthCheck.too_slow,
)
last_exception = [None]
repr_for_last_exception = [None]
def evaluate_test_data(data):
try:
result = test_runner(data, reify_and_execute(
search_strategy, test,
))
if result is not None and settings.perform_health_check:
fail_health_check((
'Tests run under @given should return None, but '
'%s returned %r instead.'
) % (test.__name__, result), HealthCheck.return_value)
return False
except UnsatisfiedAssumption:
data.mark_invalid()
except (
HypothesisDeprecationWarning, FailedHealthCheck,
StopTest,
):
raise
except Exception:
last_exception[0] = traceback.format_exc()
verbose_report(last_exception[0])
data.mark_interesting()
from hypothesis.internal.conjecture.engine import ConjectureRunner
falsifying_example = None
database_key = str_to_bytes(fully_qualified_name(test))
start_time = time.time()
runner = ConjectureRunner(
evaluate_test_data,
settings=settings, random=random,
database_key=database_key,
)
runner.run()
note_engine_for_statistics(runner)
run_time = time.time() - start_time
timed_out = (
settings.timeout > 0 and
run_time >= settings.timeout
)
if runner.last_data is None:
return
if runner.last_data.status == Status.INTERESTING:
falsifying_example = runner.last_data.buffer
if settings.database is not None:
settings.database.save(
database_key, falsifying_example
)
else:
if runner.valid_examples < min(
settings.min_satisfying_examples,
settings.max_examples,
):
if timed_out:
raise Timeout((
'Ran out of time before finding a satisfying '
'example for '
'%s. Only found %d examples in ' +
'%.2fs.'
) % (
get_pretty_function_description(test),
runner.valid_examples, run_time
))
else:
raise Unsatisfiable((
'Unable to satisfy assumptions of hypothesis '
'%s. Only %d examples considered '
'satisfied assumptions'
) % (
get_pretty_function_description(test),
runner.valid_examples,))
return
assert last_exception[0] is not None
try:
with settings:
test_runner(
ConjectureData.for_buffer(falsifying_example),
reify_and_execute(
search_strategy, test,
print_example=True, is_final=True
))
except (UnsatisfiedAssumption, StopTest):
report(traceback.format_exc())
raise Flaky(
'Unreliable assumption: An example which satisfied '
'assumptions on the first run now fails it.'
)
report(
'Failed to reproduce exception. Expected: \n' +
last_exception[0],
)
filter_message = (
'Unreliable test data: Failed to reproduce a failure '
'and then when it came to recreating the example in '
'order to print the test data with a flaky result '
'the example was filtered out (by e.g. a '
'call to filter in your strategy) when we didn\'t '
'expect it to be.'
)
try:
test_runner(
ConjectureData.for_buffer(falsifying_example),
reify_and_execute(
search_strategy,
test_is_flaky(test, repr_for_last_exception[0]),
print_example=True, is_final=True
))
except (UnsatisfiedAssumption, StopTest):
raise Flaky(filter_message)
def run(self):
# Tell pytest to omit the body of this function from tracebacks
__tracebackhide__ = True
if global_force_seed is None:
database_key = function_digest(self.test)
else:
database_key = None
runner = ConjectureRunner(
self.evaluate_test_data,
settings=self.settings,
random=self.random,
database_key=database_key,
)
try:
runner.run()
finally:
self.used_examples_from_database = runner.used_examples_from_database
note_engine_for_statistics(runner)
self.used_examples_from_database = runner.used_examples_from_database
if runner.call_count == 0:
return
if runner.interesting_examples:
self.falsifying_examples = sorted(
[d for d in runner.interesting_examples.values()],
key=lambda d: sort_key(d.buffer),
reverse=True,
)
else:
if runner.valid_examples == 0:
raise Unsatisfiable(
"Unable to satisfy assumptions of hypothesis %s."
% (get_pretty_function_description(self.test),)
)
if not self.falsifying_examples:
return
elif not self.settings.report_multiple_bugs:
del self.falsifying_examples[:-1]
self.failed_normally = True
flaky = 0
for falsifying_example in self.falsifying_examples:
info = falsifying_example.extra_information
ran_example = ConjectureData.for_buffer(falsifying_example.buffer)
self.__was_flaky = False
assert info.__expected_exception is not None
try:
self.execute(
ran_example,
print_example=True,
is_final=True,
expected_failure=(
info.__expected_exception,
info.__expected_traceback,
),
)
except (UnsatisfiedAssumption, StopTest):
report(traceback.format_exc())
self.__flaky(
"Unreliable assumption: An example which satisfied "
"assumptions on the first run now fails it."
)
except BaseException as e:
if len(self.falsifying_examples) <= 1:
raise
tb = get_trimmed_traceback()
report("".join(traceback.format_exception(type(e), e, tb)))
finally: # pragma: no cover
# This section is in fact entirely covered by the tests in
# test_reproduce_failure, but it seems to trigger a lovely set
# of coverage bugs: The branches show up as uncovered (despite
# definitely being covered - you can add an assert False else
# branch to verify this and see it fail - and additionally the
# second branch still complains about lack of coverage even if
# you add a pragma: no cover to it!
# See https://bitbucket.org/ned/coveragepy/issues/623/
if self.settings.print_blob is not PrintSettings.NEVER:
failure_blob = encode_failure(falsifying_example.buffer)
# Have to use the example we actually ran, not the original
# falsifying example! Otherwise we won't catch problems
# where the repr of the generated example doesn't parse.
can_use_repr = ran_example.can_reproduce_example_from_repr
if self.settings.print_blob is PrintSettings.ALWAYS or (
self.settings.print_blob is PrintSettings.INFER
and self.settings.verbosity >= Verbosity.normal
and not can_use_repr
and len(failure_blob) < 200
):
report(
(
"\nYou can reproduce this example by temporarily "
"adding @reproduce_failure(%r, %r) as a decorator "
"on your test case"
)
% (__version__, failure_blob)
)
if self.__was_flaky:
flaky += 1
# If we only have one example then we should have raised an error or
# flaky prior to this point.
assert len(self.falsifying_examples) > 1
if flaky > 0:
raise Flaky(
(
"Hypothesis found %d distinct failures, but %d of them "
"exhibited some sort of flaky behaviour."
)
% (len(self.falsifying_examples), flaky)
)
else:
raise MultipleFailures(
("Hypothesis found %d distinct failures.")
% (len(self.falsifying_examples))
)
def test_digests_are_reasonably_unique():
assert (
function_digest(test_simple_conversion) !=
function_digest(test_does_not_error_on_confused_sources)
)
def run_test_with_generator(test):
if settings.derandomize:
assert provided_random is None
random = Random(function_digest(test))
else:
random = provided_random or Random()
original_argspec = inspect.getargspec(test)
if original_argspec.varargs:
raise InvalidArgument('varargs are not supported with @given')
extra_kwargs = [
k for k in generator_kwargs if k not in original_argspec.args
]
if extra_kwargs and not original_argspec.keywords:
raise InvalidArgument(
'%s() got an unexpected keyword argument %r' %
(test.__name__, extra_kwargs[0]))
if (len(generator_arguments) > len(original_argspec.args)):
raise InvalidArgument(
('Too many positional arguments for %s() (got %d but'
' expected at most %d') %
(test.__name__, len(generator_arguments),
len(original_argspec.args)))
arguments = original_argspec.args + sorted(extra_kwargs)
specifiers = list(generator_arguments)
seen_kwarg = None
for a in arguments:
if a in generator_kwargs:
seen_kwarg = seen_kwarg or a
specifiers.append(generator_kwargs[a])
else:
if seen_kwarg is not None:
raise InvalidArgument(
('Argument %s comes after keyword %s which has been '
'specified, but does not itself have a '
'specification') % (a, seen_kwarg))
argspec = inspect.ArgSpec(args=arguments,
keywords=original_argspec.keywords,
varargs=original_argspec.varargs,
defaults=tuple(
map(HypothesisProvided, specifiers)))
@copy_argspec(test.__name__, argspec)
def wrapped_test(*arguments, **kwargs):
selfy = None
# Because we converted all kwargs to given into real args and
# error if we have neither args nor kwargs, this should always
# be valid
assert argspec.args
selfy = kwargs.get(argspec.args[0])
if isinstance(selfy, HypothesisProvided):
selfy = None
test_runner = executor(selfy)
for example in getattr(wrapped_test,
'hypothesis_explicit_examples', ()):
if example.args:
example_kwargs = dict(
zip(argspec.args[-len(example.args):], example.args))
else:
example_kwargs = dict(example.kwargs)
for k, v in kwargs.items():
if not isinstance(v, HypothesisProvided):
example_kwargs[k] = v
test_runner(lambda: test(*arguments, **example_kwargs))
if not any(
isinstance(x, HypothesisProvided)
for xs in (arguments, kwargs.values()) for x in xs):
# All arguments have been satisfied without needing to invoke
# hypothesis
test_runner(lambda: test(*arguments, **kwargs))
return
def convert_to_specifier(v):
if isinstance(v, HypothesisProvided):
return v.value
else:
return just(v)
given_specifier = (tuple(map(convert_to_specifier, arguments)),
dict([(k, convert_to_specifier(v))
for (k, v) in kwargs.items()]))
search_strategy = strategy(given_specifier, settings)
if settings.database:
storage = settings.database.storage_for(
given_specifier, search_strategy)
else:
storage = None
def is_template_example(xs):
try:
test_runner(reify_and_execute(search_strategy, xs, test))
return False
except UnsatisfiedAssumption as e:
raise e
except Exception:
return True
is_template_example.__name__ = test.__name__
is_template_example.__qualname__ = getattr(test, '__qualname__',
test.__name__)
falsifying_template = None
try:
falsifying_template = best_satisfying_template(
search_strategy, random, is_template_example, settings,
storage)
except NoSuchExample:
return
test_runner(
reify_and_execute(search_strategy,
falsifying_template,
test,
print_example=True))
test_runner(
reify_and_execute(search_strategy,
falsifying_template,
test_is_flaky(test),
print_example=True))
wrapped_test.__name__ = test.__name__
wrapped_test.__doc__ = test.__doc__
wrapped_test.is_hypothesis_test = True
wrapped_test.hypothesis_explicit_examples = getattr(
test, 'hypothesis_explicit_examples', [])
return wrapped_test
def test_digest_returns_the_same_value_for_two_calls():
assert (
function_digest(test_simple_conversion) ==
function_digest(test_simple_conversion)
)
def find(specifier, condition, settings=None, random=None, storage=None):
settings = settings or Settings(
max_examples=2000,
min_satisfying_examples=0,
max_shrinks=2000,
)
if not isinstance(specifier, SearchStrategy):
raise InvalidArgument(
'Expected SearchStrategy but got %r of type %s' % (
specifier, type(specifier).__name__
))
search = specifier
if storage is None and settings.database is not None:
storage = settings.database.storage(
'find(%s)' % (
binascii.hexlify(function_digest(condition)).decode('ascii'),
)
)
random = random or new_random()
successful_examples = [0]
def template_condition(template):
with BuildContext():
result = search.reify(template)
success = condition(result)
if success:
successful_examples[0] += 1
if not successful_examples[0]:
verbose_report(lambda: 'Trying example %s' % (
repr(result),
))
elif success:
if successful_examples[0] == 1:
verbose_report(lambda: 'Found satisfying example %s' % (
repr(result),
))
else:
verbose_report(lambda: 'Shrunk example to %s' % (
repr(result),
))
return success
template_condition.__name__ = condition.__name__
tracker = Tracker()
try:
template = best_satisfying_template(
search, random, template_condition, settings,
tracker=tracker, max_parameter_tries=2,
storage=storage,
)
with BuildContext(is_final=True, close_on_capture=False):
return search.reify(template)
except Timeout:
raise
except NoSuchExample:
if search.template_upper_bound <= len(tracker):
raise DefinitelyNoSuchExample(
get_pretty_function_description(condition),
search.template_upper_bound,
)
raise NoSuchExample(get_pretty_function_description(condition))
def test_evalled_lambda_digests_are_stable_across_process_runs():
digest = function_digest(eval('lambda x: 42'))
print(repr(digest))
assert digest == b'\x86\x95\xf2\xab\x8a\x83\x91F|1\xbdi\x05\xdeC?'
def falsify(self, hypothesis, *argument_types, **kwargs): # pylint: disable=too-many-locals,too-many-branches
"""
Attempt to construct an example tuple x matching argument_types such
that hypothesis(*x) returns a falsey value
"""
teardown_example = kwargs.get('teardown_example') or (lambda x: None)
setup_example = kwargs.get('setup_example') or (lambda: None)
random = self.random
if random is None:
random = Random(function_digest(hypothesis))
build_context = BuildContext(random)
search_strategy = strategy(argument_types, self.settings)
storage = None
if self.database is not None:
storage = self.database.storage_for(argument_types)
def falsifies(args): # pylint: disable=missing-docstring
example = None
try:
try:
setup_example()
example = search_strategy.reify(args)
return not hypothesis(*example)
except UnsatisfiedAssumption:
return False
finally:
teardown_example(example)
track_seen = Tracker()
falsifying_examples = []
if storage:
for example in storage.fetch():
track_seen.track(example)
if falsifies(example):
falsifying_examples = [example]
break
satisfying_examples = 0
timed_out = False
max_examples = self.max_examples
min_satisfying_examples = self.min_satisfying_examples
parameter_source = ParameterSource(context=build_context,
strategy=search_strategy,
min_parameters=max(
2,
int(float(max_examples) / 10)))
start_time = time.time()
def time_to_call_it_a_day():
"""Have we exceeded our timeout?"""
if self.timeout <= 0:
return False
return time.time() >= start_time + self.timeout
for parameter in islice(parameter_source,
max_examples - len(track_seen)):
if len(track_seen) >= search_strategy.size_upper_bound:
break
if falsifying_examples:
break
if time_to_call_it_a_day():
break
args = search_strategy.produce_template(build_context, parameter)
if track_seen.track(args) > 1:
parameter_source.mark_bad()
continue
try:
setup_example()
a = None
try:
a = search_strategy.reify(args)
is_falsifying_example = not hypothesis(*a)
finally:
teardown_example(a)
except UnsatisfiedAssumption:
parameter_source.mark_bad()
continue
satisfying_examples += 1
if is_falsifying_example:
falsifying_examples.append(args)
run_time = time.time() - start_time
timed_out = self.timeout >= 0 and run_time >= self.timeout
if not falsifying_examples:
if (satisfying_examples
and len(track_seen) >= search_strategy.size_lower_bound):
raise Exhausted(hypothesis, satisfying_examples)
elif satisfying_examples < min_satisfying_examples:
if timed_out:
raise Timeout(hypothesis, satisfying_examples, run_time)
else:
raise Unsatisfiable(hypothesis, satisfying_examples,
run_time)
else:
raise Unfalsifiable(hypothesis)
for example in falsifying_examples:
if not falsifies(example):
raise Flaky(hypothesis, example)
best_example = falsifying_examples[0]
for simpler in search_strategy.simplify_such_that(
random,
best_example,
falsifies,
tracker=track_seen,
):
best_example = simpler
if time_to_call_it_a_day():
# We no cover in here because it's a bit sensitive to timing
# and tends to make tests flaky. There are tests that mean
# this is definitely covered most of the time.
break # pragma: no cover
if storage is not None:
storage.save(best_example)
setup_example()
return search_strategy.reify(best_example)
def find(specifier, condition, settings=None, random=None, database_key=None):
settings = settings or Settings(max_examples=2000, min_satisfying_examples=0, max_shrinks=2000)
if database_key is None and settings.database is not None:
database_key = function_digest(condition)
if not isinstance(specifier, SearchStrategy):
raise InvalidArgument("Expected SearchStrategy but got %r of type %s" % (specifier, type(specifier).__name__))
search = specifier
random = random or new_random()
successful_examples = [0]
last_data = [None]
def template_condition(data):
with BuildContext():
try:
data.is_find = True
result = data.draw(search)
data.note(result)
success = condition(result)
except UnsatisfiedAssumption:
data.mark_invalid()
if success:
successful_examples[0] += 1
if settings.verbosity == Verbosity.verbose:
if not successful_examples[0]:
report(lambda: u"Trying example %s" % (nicerepr(result),))
elif success:
if successful_examples[0] == 1:
report(lambda: u"Found satisfying example %s" % (nicerepr(result),))
else:
report(lambda: u"Shrunk example to %s" % (nicerepr(result),))
last_data[0] = data
if success and not data.frozen:
data.mark_interesting()
from hypothesis.internal.conjecture.engine import TestRunner
from hypothesis.internal.conjecture.data import TestData, Status
start = time.time()
runner = TestRunner(template_condition, settings=settings, random=random, database_key=database_key)
runner.run()
run_time = time.time() - start
if runner.last_data.status == Status.INTERESTING:
with BuildContext():
return TestData.for_buffer(runner.last_data.buffer).draw(search)
if runner.valid_examples <= settings.min_satisfying_examples:
if settings.timeout > 0 and run_time > settings.timeout:
raise Timeout(
(
"Ran out of time before finding enough valid examples for "
"%s. Only %d valid examples found in %.2f seconds."
)
% (get_pretty_function_description(condition), runner.valid_examples, run_time)
)
else:
raise Unsatisfiable(
("Unable to satisfy assumptions of " "%s. Only %d examples considered satisfied assumptions")
% (get_pretty_function_description(condition), runner.valid_examples)
)
raise NoSuchExample(get_pretty_function_description(condition))
def wrapped_test(*arguments, **kwargs):
if settings.derandomize:
random = Random(function_digest(test))
else:
random = provided_random or new_random()
import hypothesis.strategies as sd
from hypothesis.internal.strategymethod import strategy
selfy = None
arguments, kwargs = convert_positional_arguments(
wrapped_test, arguments, kwargs)
for arg in hypothesis_owned_arguments:
try:
value = kwargs[arg]
except KeyError:
continue
if not isinstance(value, HypothesisProvided):
note_deprecation(
'Passing in explicit values to override Hypothesis '
'provided values is deprecated and will no longer '
'work in Hypothesis 2.0. If you need to do this, '
'extract a common function and call that from a '
'Hypothesis based test.', settings
)
# Anything in unused_kwargs hasn't been injected through
# argspec.defaults, so we need to add them.
for k in unused_kwargs:
if k not in kwargs:
kwargs[k] = unused_kwargs[k]
# If the test function is a method of some kind, the bound object
# will be the first named argument if there are any, otherwise the
# first vararg (if any).
if argspec.args:
selfy = kwargs.get(argspec.args[0])
elif arguments:
selfy = arguments[0]
if isinstance(selfy, HypothesisProvided):
selfy = None
test_runner = executor(selfy)
for example in reversed(getattr(
wrapped_test, u'hypothesis_explicit_examples', ()
)):
if example.args:
example_kwargs = dict(zip(
argspec.args[-len(example.args):], example.args
))
else:
example_kwargs = dict(example.kwargs)
for k, v in kwargs.items():
if not isinstance(v, HypothesisProvided):
example_kwargs[k] = v
# Note: Test may mutate arguments and we can't rerun explicit
# examples, so we have to calculate the failure message at this
# point rather than than later.
message_on_failure = u'Falsifying example: %s(%s)' % (
test.__name__, arg_string(test, arguments, example_kwargs)
)
try:
with BuildContext() as b:
test_runner(
lambda: test(*arguments, **example_kwargs)
)
except BaseException:
report(message_on_failure)
for n in b.notes:
report(n)
raise
if not any(
isinstance(x, HypothesisProvided)
for xs in (arguments, kwargs.values())
for x in xs
):
# All arguments have been satisfied without needing to invoke
# hypothesis
test_runner(lambda: test(*arguments, **kwargs))
return
def convert_to_specifier(v):
if isinstance(v, HypothesisProvided):
return strategy(v.value, settings)
else:
return sd.just(v)
given_specifier = sd.tuples(
sd.tuples(*map(convert_to_specifier, arguments)),
sd.fixed_dictionaries(dict(
(k, convert_to_specifier(v)) for (k, v) in kwargs.items()))
)
def fail_health_check(message):
message += (
'\nSee http://hypothesis.readthedocs.org/en/latest/health'
'checks.html for more information about this.'
)
if settings.strict:
raise FailedHealthCheck(message)
else:
warnings.warn(FailedHealthCheck(message))
search_strategy = strategy(given_specifier, settings)
search_strategy.validate()
if settings.database:
storage = settings.database.storage(
fully_qualified_name(test))
else:
storage = None
start = time.time()
warned_random = [False]
perform_health_check = settings.perform_health_check
if Settings.default is not None:
perform_health_check &= Settings.default.perform_health_check
if perform_health_check:
initial_state = getglobalrandomstate()
health_check_random = Random(random.getrandbits(128))
count = 0
bad_draws = 0
filtered_draws = 0
errors = 0
while (
count < 10 and time.time() < start + 1 and
filtered_draws < 50 and bad_draws < 50
):
try:
with Settings(settings, verbosity=Verbosity.quiet):
test_runner(reify_and_execute(
search_strategy,
search_strategy.draw_template(
health_check_random,
search_strategy.draw_parameter(
health_check_random,
)),
lambda *args, **kwargs: None,
))
count += 1
except BadTemplateDraw:
bad_draws += 1
except UnsatisfiedAssumption:
filtered_draws += 1
except Exception:
if errors == 0:
report(traceback.format_exc())
errors += 1
if test_runner is default_executor:
fail_health_check(
'An exception occurred during data '
'generation in initial health check. '
'This indicates a bug in the strategy. '
'This could either be a Hypothesis bug or '
"an error in a function you've passed to "
'it to construct your data.'
)
else:
fail_health_check(
'An exception occurred during data '
'generation in initial health check. '
'This indicates a bug in the strategy. '
'This could either be a Hypothesis bug or '
'an error in a function you\'ve passed to '
'it to construct your data. Additionally, '
'you have a custom executor, which means '
'that this could be your executor failing '
'to handle a function which returns None. '
)
if filtered_draws >= 50:
fail_health_check((
'It looks like your strategy is filtering out a lot '
'of data. Health check found %d filtered examples but '
'only %d good ones. This will make your tests much '
'slower, and also will probably distort the data '
'generation quite a lot. You should adapt your '
'strategy to filter less.') % (
filtered_draws, count
))
if bad_draws >= 50:
fail_health_check(
'Hypothesis is struggling to generate examples. '
'This is often a sign of a recursive strategy which '
'fans out too broadly. If you\'re using recursive, '
'try to reduce the size of the recursive step or '
'increase the maximum permitted number of leaves.'
)
runtime = time.time() - start
if runtime > 1.0 or count < 10:
fail_health_check((
'Data generation is extremely slow: Only produced '
'%d valid examples in %.2f seconds. Try decreasing '
"size of the data you're generating (with e.g."
'average_size or max_leaves parameters).'
) % (count, runtime))
if getglobalrandomstate() != initial_state:
warned_random[0] = True
fail_health_check(
'Data generation depends on global random module. '
'This makes results impossible to replay, which '
'prevents Hypothesis from working correctly. '
'If you want to use methods from random, use '
'randoms() from hypothesis.strategies to get an '
'instance of Random you can use. Alternatively, you '
'can use the random_module() strategy to explicitly '
'seed the random module.'
)
last_exception = [None]
repr_for_last_exception = [None]
def is_template_example(xs):
if perform_health_check and not warned_random[0]:
initial_state = getglobalrandomstate()
record_repr = [None]
try:
result = test_runner(reify_and_execute(
search_strategy, xs, test,
record_repr=record_repr,
))
if result is not None:
note_deprecation((
'Tests run under @given should return None, but '
'%s returned %r instead.'
'In Hypothesis 2.0 this will become an error.'
) % (test.__name__, result), settings)
return False
except HypothesisDeprecationWarning:
raise
except UnsatisfiedAssumption as e:
raise e
except Exception as e:
last_exception[0] = traceback.format_exc()
repr_for_last_exception[0] = record_repr[0]
verbose_report(last_exception[0])
return True
finally:
if (
not warned_random[0] and
perform_health_check and
getglobalrandomstate() != initial_state
):
warned_random[0] = True
fail_health_check(
'Your test used the global random module. '
'This is unlikely to work correctly. You should '
'consider using the randoms() strategy from '
'hypothesis.strategies instead. Alternatively, '
'you can use the random_module() strategy to '
'explicitly seed the random module.'
)
is_template_example.__name__ = test.__name__
is_template_example.__qualname__ = qualname(test)
falsifying_template = None
try:
falsifying_template = best_satisfying_template(
search_strategy, random, is_template_example,
settings, storage, start_time=start,
)
except NoSuchExample:
return
assert last_exception[0] is not None
with settings:
test_runner(reify_and_execute(
search_strategy, falsifying_template, test,
print_example=True, is_final=True
))
report(
u'Failed to reproduce exception. Expected: \n' +
last_exception[0],
)
test_runner(reify_and_execute(
search_strategy, falsifying_template,
test_is_flaky(test, repr_for_last_exception[0]),
print_example=True, is_final=True
))
def test_digest_returns_the_same_value_for_two_calls():
assert function_digest(test_simple_conversion) == function_digest(
test_simple_conversion)
def wrapped_test(*arguments, **kwargs):
settings = wrapped_test._hypothesis_internal_use_settings
if wrapped_test._hypothesis_internal_use_seed is not None:
random = Random(wrapped_test._hypothesis_internal_use_seed)
elif settings.derandomize:
random = Random(function_digest(test))
else:
random = new_random()
import hypothesis.strategies as sd
selfy = None
arguments, kwargs = convert_positional_arguments(
wrapped_test, arguments, kwargs)
# If the test function is a method of some kind, the bound object
# will be the first named argument if there are any, otherwise the
# first vararg (if any).
if argspec.args:
selfy = kwargs.get(argspec.args[0])
elif arguments:
selfy = arguments[0]
test_runner = executor(selfy)
for example in reversed(
getattr(wrapped_test, 'hypothesis_explicit_examples', ())):
if example.args:
example_kwargs = dict(
zip(original_argspec.args[-len(example.args):],
example.args))
else:
example_kwargs = example.kwargs
example_kwargs.update(kwargs)
# Note: Test may mutate arguments and we can't rerun explicit
# examples, so we have to calculate the failure message at this
# point rather than than later.
message_on_failure = 'Falsifying example: %s(%s)' % (
test.__name__, arg_string(test, arguments, example_kwargs))
try:
with BuildContext() as b:
test_runner(lambda: test(*arguments, **example_kwargs))
except BaseException:
report(message_on_failure)
for n in b.notes:
report(n)
raise
arguments = tuple(arguments)
given_specifier = sd.tuples(
sd.just(arguments),
sd.fixed_dictionaries(generator_kwargs).map(
lambda args: dict(args, **kwargs)))
def fail_health_check(message):
message += (
'\nSee http://hypothesis.readthedocs.org/en/latest/health'
'checks.html for more information about this.')
if settings.strict:
raise FailedHealthCheck(message)
else:
warnings.warn(FailedHealthCheck(message))
search_strategy = given_specifier
search_strategy.validate()
if settings.database:
storage = settings.database.storage(fully_qualified_name(test))
else:
storage = None
start = time.time()
warned_random = [False]
perform_health_check = settings.perform_health_check
if Settings.default is not None:
perform_health_check &= Settings.default.perform_health_check
if perform_health_check:
initial_state = getglobalrandomstate()
health_check_random = Random(random.getrandbits(128))
count = 0
bad_draws = 0
filtered_draws = 0
errors = 0
while (count < 10 and time.time() < start + 1
and filtered_draws < 50 and bad_draws < 50):
try:
with Settings(settings, verbosity=Verbosity.quiet):
test_runner(
reify_and_execute(
search_strategy,
search_strategy.draw_template(
health_check_random,
search_strategy.draw_parameter(
health_check_random, )),
lambda *args, **kwargs: None,
))
count += 1
except BadTemplateDraw:
bad_draws += 1
except UnsatisfiedAssumption:
filtered_draws += 1
except Exception:
if errors == 0:
report(traceback.format_exc())
errors += 1
if test_runner is default_executor:
fail_health_check(
'An exception occurred during data '
'generation in initial health check. '
'This indicates a bug in the strategy. '
'This could either be a Hypothesis bug or '
"an error in a function yo've passed to "
'it to construct your data.')
else:
fail_health_check(
'An exception occurred during data '
'generation in initial health check. '
'This indicates a bug in the strategy. '
'This could either be a Hypothesis bug or '
'an error in a function you\'ve passed to '
'it to construct your data. Additionally, '
'you have a custom executor, which means '
'that this could be your executor failing '
'to handle a function which returns None. ')
if filtered_draws >= 50:
fail_health_check((
'It looks like your strategy is filtering out a lot '
'of data. Health check found %d filtered examples but '
'only %d good ones. This will make your tests much '
'slower, and also will probably distort the data '
'generation quite a lot. You should adapt your '
'strategy to filter less.') % (filtered_draws, count))
if bad_draws >= 50:
fail_health_check(
'Hypothesis is struggling to generate examples. '
'This is often a sign of a recursive strategy which '
'fans out too broadly. If you\'re using recursive, '
'try to reduce the size of the recursive step or '
'increase the maximum permitted number of leaves.')
runtime = time.time() - start
if runtime > 1.0 or count < 10:
fail_health_check(
('Data generation is extremely slow: Only produced '
'%d valid examples in %.2f seconds. Try decreasing '
"size of the data yo're generating (with e.g."
'average_size or max_leaves parameters).') %
(count, runtime))
if getglobalrandomstate() != initial_state:
warned_random[0] = True
fail_health_check(
'Data generation depends on global random module. '
'This makes results impossible to replay, which '
'prevents Hypothesis from working correctly. '
'If you want to use methods from random, use '
'randoms() from hypothesis.strategies to get an '
'instance of Random you can use. Alternatively, you '
'can use the random_module() strategy to explicitly '
'seed the random module.')
last_exception = [None]
repr_for_last_exception = [None]
def is_template_example(xs):
if perform_health_check and not warned_random[0]:
initial_state = getglobalrandomstate()
record_repr = [None]
try:
result = test_runner(
reify_and_execute(
search_strategy,
xs,
test,
record_repr=record_repr,
))
if result is not None and settings.perform_health_check:
raise FailedHealthCheck(
('Tests run under @given should return None, but '
'%s returned %r instead.') %
(test.__name__, result), settings)
return False
except (HypothesisDeprecationWarning, FailedHealthCheck,
UnsatisfiedAssumption):
raise
except Exception:
last_exception[0] = traceback.format_exc()
repr_for_last_exception[0] = record_repr[0]
verbose_report(last_exception[0])
return True
finally:
if (not warned_random[0] and perform_health_check
and getglobalrandomstate() != initial_state):
warned_random[0] = True
fail_health_check(
'Your test used the global random module. '
'This is unlikely to work correctly. You should '
'consider using the randoms() strategy from '
'hypothesis.strategies instead. Alternatively, '
'you can use the random_module() strategy to '
'explicitly seed the random module.')
is_template_example.__name__ = test.__name__
is_template_example.__qualname__ = qualname(test)
with settings:
falsifying_template = None
try:
falsifying_template = best_satisfying_template(
search_strategy,
random,
is_template_example,
settings,
storage,
start_time=start,
)
except NoSuchExample:
return
assert last_exception[0] is not None
try:
test_runner(
reify_and_execute(search_strategy,
falsifying_template,
test,
print_example=True,
is_final=True))
except UnsatisfiedAssumption:
report(traceback.format_exc())
raise Flaky(
'Unreliable assumption: An example which satisfied '
'assumptions on the first run now fails it.')
report(
'Failed to reproduce exception. Expected: \n' +
last_exception[0], )
try:
test_runner(
reify_and_execute(search_strategy,
falsifying_template,
test_is_flaky(
test,
repr_for_last_exception[0]),
print_example=True,
is_final=True))
except UnsatisfiedAssumption:
raise Flaky(
'Unreliable test data: Failed to reproduce a failure '
'and then when it came to recreating the example in '
'order to print the test data with a flaky result '
'the example was filtered out (by e.g. a '
'call to filter in your strategy) when we didn\'t '
'expect it to be.')
def test_can_digest_a_unicode_lambda():
function_digest(lambda x: "☃" in str(x))
def run_test_with_generator(test):
if settings.derandomize:
assert provided_random is None
random = Random(function_digest(test))
else:
random = provided_random or Random()
original_argspec = inspect.getargspec(test)
if generator_arguments and original_argspec.varargs:
raise InvalidArgument("varargs are not supported with positional arguments to @given")
extra_kwargs = [k for k in generator_kwargs if k not in original_argspec.args]
if extra_kwargs and not original_argspec.keywords:
raise InvalidArgument("%s() got an unexpected keyword argument %r" % (test.__name__, extra_kwargs[0]))
if len(generator_arguments) > len(original_argspec.args):
raise InvalidArgument(
("Too many positional arguments for %s() (got %d but" " expected at most %d")
% (test.__name__, len(generator_arguments), len(original_argspec.args))
)
arguments = original_argspec.args + sorted(extra_kwargs)
specifiers = list(generator_arguments)
seen_kwarg = None
for a in arguments:
if a in generator_kwargs:
seen_kwarg = seen_kwarg or a
specifiers.append(generator_kwargs[a])
else:
if seen_kwarg is not None:
raise InvalidArgument(
(
"Argument %s comes after keyword %s which has been "
"specified, but does not itself have a "
"specification"
)
% (a, seen_kwarg)
)
argspec = inspect.ArgSpec(
args=arguments,
keywords=original_argspec.keywords,
varargs=original_argspec.varargs,
defaults=tuple(map(HypothesisProvided, specifiers)),
)
@copy_argspec(test.__name__, argspec)
def wrapped_test(*arguments, **kwargs):
selfy = None
arguments, kwargs = convert_positional_arguments(wrapped_test, arguments, kwargs)
# Because we converted all kwargs to given into real args and
# error if we have neither args nor kwargs, this should always
# be valid
assert argspec.args
selfy = kwargs.get(argspec.args[0])
if isinstance(selfy, HypothesisProvided):
selfy = None
test_runner = executor(selfy)
for example in getattr(wrapped_test, "hypothesis_explicit_examples", ()):
if example.args:
example_kwargs = dict(zip(argspec.args[-len(example.args) :], example.args))
else:
example_kwargs = dict(example.kwargs)
for k, v in kwargs.items():
if not isinstance(v, HypothesisProvided):
example_kwargs[k] = v
# Note: Test may mutate arguments and we can't rerun explicit
# examples, so we have to calculate the failure message at this
# point rather than than later.
message_on_failure = "Falsifying example: %s(%s)" % (
test.__name__,
arg_string(test, arguments, example_kwargs),
)
try:
test_runner(lambda: test(*arguments, **example_kwargs))
except BaseException:
report(message_on_failure)
raise
if not any(isinstance(x, HypothesisProvided) for xs in (arguments, kwargs.values()) for x in xs):
# All arguments have been satisfied without needing to invoke
# hypothesis
test_runner(lambda: test(*arguments, **kwargs))
return
def convert_to_specifier(v):
if isinstance(v, HypothesisProvided):
return strategy(v.value, settings)
else:
return sd.just(v)
given_specifier = sd.tuples(
sd.tuples(*map(convert_to_specifier, arguments)),
sd.fixed_dictionaries(dict((k, convert_to_specifier(v)) for (k, v) in kwargs.items())),
)
search_strategy = strategy(given_specifier, settings)
if settings.database:
storage = settings.database.storage(fully_qualified_name(test))
else:
storage = None
last_exception = [None]
repr_for_last_exception = [None]
def is_template_example(xs):
record_repr = [None]
try:
test_runner(
reify_and_execute(
search_strategy, xs, test, always_print=settings.max_shrinks <= 0, record_repr=record_repr
)
)
return False
except UnsatisfiedAssumption as e:
raise e
except Exception as e:
if settings.max_shrinks <= 0:
raise e
last_exception[0] = traceback.format_exc()
repr_for_last_exception[0] = record_repr[0]
verbose_report(last_exception[0])
return True
is_template_example.__name__ = test.__name__
is_template_example.__qualname__ = qualname(test)
falsifying_template = None
try:
falsifying_template = best_satisfying_template(
search_strategy, random, is_template_example, settings, storage
)
except NoSuchExample:
return
assert last_exception[0] is not None
with settings:
test_runner(reify_and_execute(search_strategy, falsifying_template, test, print_example=True))
report("Failed to reproduce exception. Expected: \n" + last_exception[0])
test_runner(
reify_and_execute(
search_strategy,
falsifying_template,
test_is_flaky(test, repr_for_last_exception[0]),
print_example=True,
)
)
wrapped_test.__name__ = test.__name__
wrapped_test.__doc__ = test.__doc__
wrapped_test.is_hypothesis_test = True
wrapped_test.hypothesis_explicit_examples = getattr(test, "hypothesis_explicit_examples", [])
return wrapped_test
def find(
specifier, # type: SearchStrategy
condition, # type: Callable[[Any], bool]
settings=None, # type: Settings
random=None, # type: Any
database_key=None, # type: bytes
):
# type: (...) -> Any
"""Returns the minimal example from the given strategy ``specifier`` that
matches the predicate function ``condition``."""
if settings is None:
settings = Settings(max_examples=2000)
settings = Settings(settings, suppress_health_check=HealthCheck.all())
if database_key is None and settings.database is not None:
database_key = function_digest(condition)
if not isinstance(specifier, SearchStrategy):
raise InvalidArgument(
'Expected SearchStrategy but got %r of type %s' % (
specifier, type(specifier).__name__
))
specifier.validate()
search = specifier
random = random or new_random()
successful_examples = [0]
last_data = [None]
last_repr = [None]
def template_condition(data):
with BuildContext(data):
try:
data.is_find = True
with deterministic_PRNG():
result = data.draw(search)
data.note(result)
success = condition(result)
except UnsatisfiedAssumption:
data.mark_invalid()
if success:
successful_examples[0] += 1
if settings.verbosity >= Verbosity.verbose:
if not successful_examples[0]:
report(
u'Tried non-satisfying example %s' % (nicerepr(result),))
elif success:
if successful_examples[0] == 1:
last_repr[0] = nicerepr(result)
report(u'Found satisfying example %s' % (last_repr[0],))
last_data[0] = data
elif (
sort_key(hbytes(data.buffer)) <
sort_key(last_data[0].buffer)
) and nicerepr(result) != last_repr[0]:
last_repr[0] = nicerepr(result)
report(u'Shrunk example to %s' % (last_repr[0],))
last_data[0] = data
if success and not data.frozen:
data.mark_interesting()
start = benchmark_time()
runner = ConjectureRunner(
template_condition, settings=settings, random=random,
database_key=database_key,
)
runner.run()
note_engine_for_statistics(runner)
run_time = benchmark_time() - start
if runner.interesting_examples:
data = ConjectureData.for_buffer(
list(runner.interesting_examples.values())[0].buffer)
with BuildContext(data):
with deterministic_PRNG():
return data.draw(search)
if runner.valid_examples == 0 and (
runner.exit_reason != ExitReason.finished
):
if settings.timeout > 0 and run_time > settings.timeout:
raise Timeout(( # pragma: no cover
'Ran out of time before finding enough valid examples for '
'%s. Only %d valid examples found in %.2f seconds.'
) % (
get_pretty_function_description(condition),
runner.valid_examples, run_time))
else:
raise Unsatisfiable(
'Unable to satisfy assumptions of %s.' %
(get_pretty_function_description(condition),)
)
raise NoSuchExample(get_pretty_function_description(condition))
def run(self):
# Tell pytest to omit the body of this function from tracebacks
__tracebackhide__ = True
if global_force_seed is None:
database_key = function_digest(self.test)
else:
database_key = None
runner = ConjectureRunner(
self.evaluate_test_data,
settings=self.settings,
random=self.random,
database_key=database_key,
)
try:
runner.run()
finally:
self.used_examples_from_database = runner.used_examples_from_database
note_engine_for_statistics(runner)
self.used_examples_from_database = runner.used_examples_from_database
if runner.call_count == 0:
return
if runner.interesting_examples:
self.falsifying_examples = sorted(
[d for d in runner.interesting_examples.values()],
key=lambda d: sort_key(d.buffer),
reverse=True,
)
else:
if runner.valid_examples == 0:
raise Unsatisfiable(
"Unable to satisfy assumptions of hypothesis %s." %
(get_pretty_function_description(self.test), ))
if not self.falsifying_examples:
return
self.failed_normally = True
flaky = 0
for falsifying_example in self.falsifying_examples:
ran_example = ConjectureData.for_buffer(falsifying_example.buffer)
self.__was_flaky = False
assert falsifying_example.__expected_exception is not None
try:
self.execute(
ran_example,
print_example=True,
is_final=True,
expected_failure=(
falsifying_example.__expected_exception,
falsifying_example.__expected_traceback,
),
)
except (UnsatisfiedAssumption, StopTest):
report(traceback.format_exc())
self.__flaky(
"Unreliable assumption: An example which satisfied "
"assumptions on the first run now fails it.")
except BaseException as e:
if len(self.falsifying_examples) <= 1:
raise
tb = get_trimmed_traceback()
report("".join(traceback.format_exception(type(e), e, tb)))
finally: # pragma: no cover
# This section is in fact entirely covered by the tests in
# test_reproduce_failure, but it seems to trigger a lovely set
# of coverage bugs: The branches show up as uncovered (despite
# definitely being covered - you can add an assert False else
# branch to verify this and see it fail - and additionally the
# second branch still complains about lack of coverage even if
# you add a pragma: no cover to it!
# See https://bitbucket.org/ned/coveragepy/issues/623/
if self.settings.print_blob is not PrintSettings.NEVER:
failure_blob = encode_failure(falsifying_example.buffer)
# Have to use the example we actually ran, not the original
# falsifying example! Otherwise we won't catch problems
# where the repr of the generated example doesn't parse.
can_use_repr = ran_example.can_reproduce_example_from_repr
if self.settings.print_blob is PrintSettings.ALWAYS or (
self.settings.print_blob is PrintSettings.INFER
and self.settings.verbosity >= Verbosity.normal
and not can_use_repr and len(failure_blob) < 200):
report((
"\nYou can reproduce this example by temporarily "
"adding @reproduce_failure(%r, %r) as a decorator "
"on your test case") % (__version__, failure_blob))
if self.__was_flaky:
flaky += 1
# If we only have one example then we should have raised an error or
# flaky prior to this point.
assert len(self.falsifying_examples) > 1
if flaky > 0:
raise Flaky(
("Hypothesis found %d distinct failures, but %d of them "
"exhibited some sort of flaky behaviour.") %
(len(self.falsifying_examples), flaky))
else:
raise MultipleFailures(("Hypothesis found %d distinct failures.") %
(len(self.falsifying_examples)))
def run_test_with_generator(test):
if settings.derandomize:
assert provided_random is None
random = Random(
function_digest(test)
)
else:
random = provided_random or Random()
original_argspec = inspect.getargspec(test)
if original_argspec.varargs:
raise InvalidArgument(
'varargs are not supported with @given'
)
extra_kwargs = [
k for k in generator_kwargs if k not in original_argspec.args]
if extra_kwargs and not original_argspec.keywords:
raise InvalidArgument(
'%s() got an unexpected keyword argument %r' % (
test.__name__,
extra_kwargs[0]
))
if (
len(generator_arguments) > len(original_argspec.args)
):
raise InvalidArgument((
'Too many positional arguments for %s() (got %d but'
' expected at most %d') % (
test.__name__, len(generator_arguments),
len(original_argspec.args)))
arguments = original_argspec.args + sorted(extra_kwargs)
specifiers = list(generator_arguments)
seen_kwarg = None
for a in arguments:
if a in generator_kwargs:
seen_kwarg = seen_kwarg or a
specifiers.append(generator_kwargs[a])
else:
if seen_kwarg is not None:
raise InvalidArgument((
'Argument %s comes after keyword %s which has been '
'specified, but does not itself have a '
'specification') % (
a, seen_kwarg
))
argspec = inspect.ArgSpec(
args=arguments,
keywords=original_argspec.keywords,
varargs=original_argspec.varargs,
defaults=tuple(map(HypothesisProvided, specifiers))
)
@copy_argspec(
test.__name__, argspec
)
def wrapped_test(*arguments, **kwargs):
selfy = None
# Because we converted all kwargs to given into real args and
# error if we have neither args nor kwargs, this should always
# be valid
assert argspec.args
selfy = kwargs.get(argspec.args[0])
if isinstance(selfy, HypothesisProvided):
selfy = None
test_runner = executor(selfy)
for example in getattr(
wrapped_test, 'hypothesis_explicit_examples', ()
):
if example.args:
example_kwargs = dict(zip(
argspec.args[-len(example.args):], example.args
))
else:
example_kwargs = dict(example.kwargs)
for k, v in kwargs.items():
if not isinstance(v, HypothesisProvided):
example_kwargs[k] = v
test_runner(
lambda: test(*arguments, **example_kwargs)
)
if not any(
isinstance(x, HypothesisProvided)
for xs in (arguments, kwargs.values())
for x in xs
):
# All arguments have been satisfied without needing to invoke
# hypothesis
test_runner(lambda: test(*arguments, **kwargs))
return
def convert_to_specifier(v):
if isinstance(v, HypothesisProvided):
return v.value
else:
return just(v)
given_specifier = (
tuple(map(convert_to_specifier, arguments)),
dict([(k, convert_to_specifier(v)) for (k, v) in kwargs.items()])
)
search_strategy = strategy(given_specifier, settings)
if settings.database:
storage = settings.database.storage_for(
given_specifier, search_strategy)
else:
storage = None
def is_template_example(xs):
try:
test_runner(reify_and_execute(search_strategy, xs, test))
return False
except UnsatisfiedAssumption as e:
raise e
except Exception:
return True
is_template_example.__name__ = test.__name__
is_template_example.__qualname__ = getattr(
test, '__qualname__', test.__name__)
falsifying_template = None
try:
falsifying_template = best_satisfying_template(
search_strategy, random, is_template_example,
settings, storage
)
except NoSuchExample:
return
test_runner(reify_and_execute(
search_strategy, falsifying_template, test,
print_example=True
))
test_runner(reify_and_execute(
search_strategy, falsifying_template, test_is_flaky(test),
print_example=True
))
wrapped_test.__name__ = test.__name__
wrapped_test.__doc__ = test.__doc__
wrapped_test.is_hypothesis_test = True
wrapped_test.hypothesis_explicit_examples = getattr(
test, 'hypothesis_explicit_examples', []
)
return wrapped_test
def find(specifier, condition, settings=None, random=None, database_key=None):
"""Returns the minimal example from the given strategy ``specifier`` that
matches the predicate function ``condition``."""
settings = settings or Settings(
max_examples=2000,
min_satisfying_examples=0,
max_shrinks=2000,
)
if database_key is None and settings.database is not None:
database_key = function_digest(condition)
if not isinstance(specifier, SearchStrategy):
raise InvalidArgument(
'Expected SearchStrategy but got %r of type %s' % (
specifier, type(specifier).__name__
))
specifier.validate()
search = specifier
random = random or new_random()
successful_examples = [0]
last_data = [None]
def template_condition(data):
with BuildContext(data):
try:
data.is_find = True
result = data.draw(search)
data.note(result)
success = condition(result)
except UnsatisfiedAssumption:
data.mark_invalid()
if success:
successful_examples[0] += 1
if settings.verbosity == Verbosity.verbose:
if not successful_examples[0]:
report(lambda: u'Trying example %s' % (
nicerepr(result),
))
elif success:
if successful_examples[0] == 1:
report(lambda: u'Found satisfying example %s' % (
nicerepr(result),
))
else:
report(lambda: u'Shrunk example to %s' % (
nicerepr(result),
))
last_data[0] = data
if success and not data.frozen:
data.mark_interesting()
start = time.time()
runner = ConjectureRunner(
template_condition, settings=settings, random=random,
database_key=database_key,
)
runner.run()
note_engine_for_statistics(runner)
run_time = time.time() - start
if runner.last_data.status == Status.INTERESTING:
data = ConjectureData.for_buffer(runner.last_data.buffer)
with BuildContext(data):
return data.draw(search)
if (
runner.valid_examples <= settings.min_satisfying_examples and
runner.exit_reason != ExitReason.finished
):
if settings.timeout > 0 and run_time > settings.timeout:
raise Timeout((
'Ran out of time before finding enough valid examples for '
'%s. Only %d valid examples found in %.2f seconds.'
) % (
get_pretty_function_description(condition),
runner.valid_examples, run_time))
else:
raise Unsatisfiable((
'Unable to satisfy assumptions of '
'%s. Only %d examples considered satisfied assumptions'
) % (
get_pretty_function_description(condition),
runner.valid_examples,))
raise NoSuchExample(get_pretty_function_description(condition))
def find(
specifier, # type: SearchStrategy
condition, # type: Callable[[Any], bool]
settings=None, # type: Settings
random=None, # type: Any
database_key=None, # type: bytes
):
# type: (...) -> Any
"""Returns the minimal example from the given strategy ``specifier`` that
matches the predicate function ``condition``."""
if settings is None:
settings = Settings(max_examples=2000)
settings = Settings(settings, suppress_health_check=HealthCheck.all())
if database_key is None and settings.database is not None:
database_key = function_digest(condition)
if not isinstance(specifier, SearchStrategy):
raise InvalidArgument("Expected SearchStrategy but got %r of type %s" %
(specifier, type(specifier).__name__))
specifier.validate()
search = specifier
random = random or new_random()
successful_examples = [0]
last_data = [None]
last_repr = [None]
def template_condition(data):
with BuildContext(data):
try:
data.is_find = True
with deterministic_PRNG():
result = data.draw(search)
data.note(result)
success = condition(result)
except UnsatisfiedAssumption:
data.mark_invalid()
if success:
successful_examples[0] += 1
if settings.verbosity >= Verbosity.verbose:
if not successful_examples[0]:
report(u"Tried non-satisfying example %s" %
(nicerepr(result), ))
elif success:
if successful_examples[0] == 1:
last_repr[0] = nicerepr(result)
report(u"Found satisfying example %s" % (last_repr[0], ))
last_data[0] = data
elif (sort_key(hbytes(data.buffer)) < sort_key(
last_data[0].buffer)
) and nicerepr(result) != last_repr[0]:
last_repr[0] = nicerepr(result)
report(u"Shrunk example to %s" % (last_repr[0], ))
last_data[0] = data
if success and not data.frozen:
data.mark_interesting()
runner = ConjectureRunner(template_condition,
settings=settings,
random=random,
database_key=database_key)
runner.run()
note_engine_for_statistics(runner)
if runner.interesting_examples:
data = ConjectureData.for_buffer(
list(runner.interesting_examples.values())[0].buffer)
with BuildContext(data):
with deterministic_PRNG():
return data.draw(search)
if runner.valid_examples == 0 and (runner.exit_reason !=
ExitReason.finished):
raise Unsatisfiable("Unable to satisfy assumptions of %s." %
(get_pretty_function_description(condition), ))
raise NoSuchExample(get_pretty_function_description(condition))
def run_state_machine_as_test(state_machine_factory, settings=None):
"""Run a state machine definition as a test, either silently doing nothing
or printing a minimal breaking program and raising an exception.
state_machine_factory is anything which returns an instance of
GenericStateMachine when called with no arguments - it can be a class or a
function. settings will be used to control the execution of the test.
"""
if settings is None:
try:
settings = state_machine_factory.TestCase.settings
check_type(Settings, settings,
"state_machine_factory.TestCase.settings")
except AttributeError:
settings = Settings(deadline=None,
suppress_health_check=HealthCheck.all())
check_type(Settings, settings, "settings")
@settings
@given(st.data())
def run_state_machine(factory, data):
machine = factory()
if isinstance(machine, GenericStateMachine) and not isinstance(
machine, RuleBasedStateMachine):
note_deprecation(
"%s inherits from GenericStateMachine, which is deprecated. Use a "
"RuleBasedStateMachine, or a test function with st.data(), instead."
% (type(machine).__name__, ),
since="2019-05-29",
)
else:
check_type(RuleBasedStateMachine, machine,
"state_machine_factory()")
data.conjecture_data.hypothesis_runner = machine
n_steps = settings.stateful_step_count
should_continue = cu.many(data.conjecture_data,
min_size=1,
max_size=n_steps,
average_size=n_steps)
print_steps = (current_build_context().is_final
or current_verbosity() >= Verbosity.debug)
try:
if print_steps:
machine.print_start()
machine.check_invariants()
while should_continue.more():
value = data.conjecture_data.draw(machine.steps())
# Assign 'result' here in case 'execute_step' fails below
result = multiple()
try:
result = machine.execute_step(value)
finally:
if print_steps:
# 'result' is only used if the step has target bundles.
# If it does, and the result is a 'MultipleResult',
# then 'print_step' prints a multi-variable assignment.
machine.print_step(value, result)
machine.check_invariants()
finally:
if print_steps:
machine.print_end()
machine.teardown()
# Use a machine digest to identify stateful tests in the example database
run_state_machine.hypothesis.inner_test._hypothesis_internal_add_digest = function_digest(
state_machine_factory)
# Copy some attributes so @seed and @reproduce_failure "just work"
run_state_machine._hypothesis_internal_use_seed = getattr(
state_machine_factory, "_hypothesis_internal_use_seed", None)
run_state_machine._hypothesis_internal_use_reproduce_failure = getattr(
state_machine_factory, "_hypothesis_internal_use_reproduce_failure",
None)
run_state_machine._hypothesis_internal_print_given_args = False
run_state_machine(state_machine_factory)
def run_engine(self):
"""Run the test function many times, on database input and generated
input, using the Conjecture engine.
"""
# Tell pytest to omit the body of this function from tracebacks
__tracebackhide__ = True
try:
database_key = self.wrapped_test._hypothesis_internal_database_key
except AttributeError:
if global_force_seed is None:
database_key = function_digest(self.test)
else:
database_key = None
runner = ConjectureRunner(
self._execute_once_for_engine,
settings=self.settings,
random=self.random,
database_key=database_key,
)
# Use the Conjecture engine to run the test function many times
# on different inputs.
runner.run()
note_engine_for_statistics(runner)
if runner.call_count == 0:
return
if runner.interesting_examples:
self.falsifying_examples = sorted(
runner.interesting_examples.values(),
key=lambda d: sort_key(d.buffer),
reverse=True,
)
else:
if runner.valid_examples == 0:
raise Unsatisfiable(
"Unable to satisfy assumptions of hypothesis %s." %
(get_pretty_function_description(self.test), ))
if not self.falsifying_examples:
return
elif not self.settings.report_multiple_bugs:
# Pretend that we only found one failure, by discarding the others.
del self.falsifying_examples[:-1]
# The engine found one or more failures, so we need to reproduce and
# report them.
self.failed_normally = True
flaky = 0
for falsifying_example in self.falsifying_examples:
info = falsifying_example.extra_information
ran_example = ConjectureData.for_buffer(falsifying_example.buffer)
self.__was_flaky = False
assert info.__expected_exception is not None
try:
self.execute_once(
ran_example,
print_example=not self.is_find,
is_final=True,
expected_failure=(
info.__expected_exception,
info.__expected_traceback,
),
)
except (UnsatisfiedAssumption, StopTest):
report(traceback.format_exc())
self.__flaky(
"Unreliable assumption: An example which satisfied "
"assumptions on the first run now fails it.")
except BaseException as e:
if len(self.falsifying_examples) <= 1:
# There is only one failure, so we can report it by raising
# it directly.
raise
# We are reporting multiple failures, so we need to manually
# print each exception's stack trace and information.
tb = get_trimmed_traceback()
report("".join(traceback.format_exception(type(e), e, tb)))
finally: # pragma: no cover
# Mostly useful for ``find`` and ensuring that objects that
# hold on to a reference to ``data`` know that it's now been
# finished and they shouldn't attempt to draw more data from
# it.
ran_example.freeze()
# This section is in fact entirely covered by the tests in
# test_reproduce_failure, but it seems to trigger a lovely set
# of coverage bugs: The branches show up as uncovered (despite
# definitely being covered - you can add an assert False else
# branch to verify this and see it fail - and additionally the
# second branch still complains about lack of coverage even if
# you add a pragma: no cover to it!
# See https://bitbucket.org/ned/coveragepy/issues/623/
if self.settings.print_blob:
report(("\nYou can reproduce this example by temporarily "
"adding @reproduce_failure(%r, %r) as a decorator "
"on your test case") %
(__version__,
encode_failure(falsifying_example.buffer)))
if self.__was_flaky:
flaky += 1
# If we only have one example then we should have raised an error or
# flaky prior to this point.
assert len(self.falsifying_examples) > 1
if flaky > 0:
raise Flaky(
("Hypothesis found %d distinct failures, but %d of them "
"exhibited some sort of flaky behaviour.") %
(len(self.falsifying_examples), flaky))
else:
raise MultipleFailures(("Hypothesis found %d distinct failures.") %
(len(self.falsifying_examples)))
def run_test_with_generator(test):
if settings.derandomize:
assert provided_random is None
random = Random(function_digest(test))
else:
random = provided_random or Random()
original_argspec = getargspec(test)
if generator_arguments and original_argspec.varargs:
raise InvalidArgument(
u'varargs are not supported with positional arguments to '
u'@given'
)
extra_kwargs = [
k for k in generator_kwargs if k not in original_argspec.args]
if extra_kwargs and not original_argspec.keywords:
raise InvalidArgument(
u'%s() got an unexpected keyword argument %r' % (
test.__name__,
extra_kwargs[0]
))
if (
len(generator_arguments) > len(original_argspec.args)
):
raise InvalidArgument((
u'Too many positional arguments for %s() (got %d but'
u' expected at most %d') % (
test.__name__, len(generator_arguments),
len(original_argspec.args)))
arguments = original_argspec.args
specifiers = list(generator_arguments)
seen_kwarg = None
for a in arguments:
if isinstance(a, list): # pragma: no cover
raise InvalidArgument((
u'Cannot decorate function %s() because it has '
u'destructuring arguments') % (
test.__name__,
))
if a in generator_kwargs:
seen_kwarg = seen_kwarg or a
specifiers.append(generator_kwargs[a])
else:
if seen_kwarg is not None:
raise InvalidArgument((
u'Argument %s comes after keyword %s which has been '
u'specified, but does not itself have a '
u'specification') % (
a, seen_kwarg
))
argspec = inspect.ArgSpec(
args=arguments,
keywords=original_argspec.keywords,
varargs=original_argspec.varargs,
defaults=tuple(map(HypothesisProvided, specifiers))
)
unused_kwargs = {}
for k in extra_kwargs:
unused_kwargs[k] = HypothesisProvided(generator_kwargs[k])
@impersonate(test)
@copy_argspec(
test.__name__, argspec
)
def wrapped_test(*arguments, **kwargs):
import hypothesis.strategies as sd
from hypothesis.internal.strategymethod import strategy
selfy = None
arguments, kwargs = convert_positional_arguments(
wrapped_test, arguments, kwargs)
# Anything in unused_kwargs hasn't been injected through
# argspec.defaults, so we need to add them.
for k in unused_kwargs:
if k not in kwargs:
kwargs[k] = unused_kwargs[k]
# If the test function is a method of some kind, the bound object
# will be the first named argument if there are any, otherwise the
# first vararg (if any).
if argspec.args:
selfy = kwargs.get(argspec.args[0])
elif arguments:
selfy = arguments[0]
if isinstance(selfy, HypothesisProvided):
selfy = None
test_runner = executor(selfy)
for example in getattr(
wrapped_test, u'hypothesis_explicit_examples', ()
):
if example.args:
example_kwargs = dict(zip(
argspec.args[-len(example.args):], example.args
))
else:
example_kwargs = dict(example.kwargs)
for k, v in kwargs.items():
if not isinstance(v, HypothesisProvided):
example_kwargs[k] = v
# Note: Test may mutate arguments and we can't rerun explicit
# examples, so we have to calculate the failure message at this
# point rather than than later.
message_on_failure = u'Falsifying example: %s(%s)' % (
test.__name__, arg_string(test, arguments, example_kwargs)
)
try:
test_runner(
lambda: test(*arguments, **example_kwargs)
)
except BaseException:
report(message_on_failure)
raise
if not any(
isinstance(x, HypothesisProvided)
for xs in (arguments, kwargs.values())
for x in xs
):
# All arguments have been satisfied without needing to invoke
# hypothesis
test_runner(lambda: test(*arguments, **kwargs))
return
def convert_to_specifier(v):
if isinstance(v, HypothesisProvided):
return strategy(v.value, settings)
else:
return sd.just(v)
given_specifier = sd.tuples(
sd.tuples(*map(convert_to_specifier, arguments)),
sd.fixed_dictionaries(dict(
(k, convert_to_specifier(v)) for (k, v) in kwargs.items()))
)
search_strategy = strategy(given_specifier, settings)
if settings.database:
storage = settings.database.storage(
fully_qualified_name(test))
else:
storage = None
last_exception = [None]
repr_for_last_exception = [None]
def is_template_example(xs):
record_repr = [None]
try:
test_runner(reify_and_execute(
search_strategy, xs, test,
always_print=settings.max_shrinks <= 0,
record_repr=record_repr,
))
return False
except UnsatisfiedAssumption as e:
raise e
except Exception as e:
if settings.max_shrinks <= 0:
raise e
last_exception[0] = traceback.format_exc()
repr_for_last_exception[0] = record_repr[0]
verbose_report(last_exception[0])
return True
is_template_example.__name__ = test.__name__
is_template_example.__qualname__ = qualname(test)
falsifying_template = None
try:
falsifying_template = best_satisfying_template(
search_strategy, random, is_template_example,
settings, storage
)
except NoSuchExample:
return
assert last_exception[0] is not None
with settings:
test_runner(reify_and_execute(
search_strategy, falsifying_template, test,
print_example=True, is_final=True
))
report(
u'Failed to reproduce exception. Expected: \n' +
last_exception[0],
)
test_runner(reify_and_execute(
search_strategy, falsifying_template,
test_is_flaky(test, repr_for_last_exception[0]),
print_example=True, is_final=True
))
for attr in dir(test):
if attr[0] != '_' and not hasattr(wrapped_test, attr):
setattr(wrapped_test, attr, getattr(test, attr))
wrapped_test.is_hypothesis_test = True
return wrapped_test
def test_digests_are_reasonably_unique():
assert (
function_digest(test_simple_conversion) !=
function_digest(test_does_not_error_on_dynamically_defined_functions)
)
def find(specifier, condition, settings=None, random=None, database_key=None):
"""Returns the minimal example from the given strategy ``specifier`` that
matches the predicate function ``condition``."""
settings = settings or Settings(
max_examples=2000,
min_satisfying_examples=0,
max_shrinks=2000,
)
settings = Settings(settings, perform_health_check=False)
if database_key is None and settings.database is not None:
database_key = function_digest(condition)
if not isinstance(specifier, SearchStrategy):
raise InvalidArgument(
'Expected SearchStrategy but got %r of type %s' % (
specifier, type(specifier).__name__
))
specifier.validate()
search = specifier
random = random or new_random()
successful_examples = [0]
last_data = [None]
def template_condition(data):
with BuildContext(data):
try:
data.is_find = True
result = data.draw(search)
data.note(result)
success = condition(result)
except UnsatisfiedAssumption:
data.mark_invalid()
if success:
successful_examples[0] += 1
if settings.verbosity == Verbosity.verbose:
if not successful_examples[0]:
report(lambda: u'Trying example %s' % (
nicerepr(result),
))
elif success:
if successful_examples[0] == 1:
report(lambda: u'Found satisfying example %s' % (
nicerepr(result),
))
last_data[0] = data
elif (
sort_key(hbytes(data.buffer)) <
sort_key(last_data[0].buffer)
):
report(lambda: u'Shrunk example to %s' % (
nicerepr(result),
))
last_data[0] = data
if success and not data.frozen:
data.mark_interesting()
start = time.time()
runner = ConjectureRunner(
template_condition, settings=settings, random=random,
database_key=database_key,
)
runner.run()
note_engine_for_statistics(runner)
run_time = time.time() - start
if runner.interesting_examples:
data = ConjectureData.for_buffer(
list(runner.interesting_examples.values())[0].buffer)
with BuildContext(data):
return data.draw(search)
if (
runner.valid_examples <= settings.min_satisfying_examples and
runner.exit_reason != ExitReason.finished
):
if settings.timeout > 0 and run_time > settings.timeout:
raise Timeout((
'Ran out of time before finding enough valid examples for '
'%s. Only %d valid examples found in %.2f seconds.'
) % (
get_pretty_function_description(condition),
runner.valid_examples, run_time))
else:
raise Unsatisfiable((
'Unable to satisfy assumptions of '
'%s. Only %d examples considered satisfied assumptions'
) % (
get_pretty_function_description(condition),
runner.valid_examples,))
raise NoSuchExample(get_pretty_function_description(condition))
def test_can_digest_a_built_in_function():
import math
assert function_digest(math.isnan) != function_digest(range)
def falsify(
self, hypothesis,
*argument_types,
**kwargs
): # pylint: disable=too-many-locals,too-many-branches
"""
Attempt to construct an example tuple x matching argument_types such
that hypothesis(*x) returns a falsey value
"""
teardown_example = kwargs.get('teardown_example') or (lambda x: None)
setup_example = kwargs.get('setup_example') or (lambda: None)
random = self.random
if random is None:
random = Random(
function_digest(hypothesis)
)
build_context = BuildContext(random)
search_strategy = strategy(argument_types, self.settings)
storage = None
if self.database is not None:
storage = self.database.storage_for(argument_types)
def falsifies(args): # pylint: disable=missing-docstring
example = None
try:
try:
setup_example()
example = search_strategy.reify(args)
return not hypothesis(*example)
except UnsatisfiedAssumption:
return False
finally:
teardown_example(example)
track_seen = Tracker()
falsifying_examples = []
if storage:
for example in storage.fetch():
track_seen.track(example)
if falsifies(example):
falsifying_examples = [example]
break
satisfying_examples = 0
timed_out = False
max_examples = self.max_examples
min_satisfying_examples = self.min_satisfying_examples
parameter_source = ParameterSource(
context=build_context, strategy=search_strategy,
min_parameters=max(2, int(float(max_examples) / 10))
)
start_time = time.time()
def time_to_call_it_a_day():
"""Have we exceeded our timeout?"""
if self.timeout <= 0:
return False
return time.time() >= start_time + self.timeout
for parameter in islice(
parameter_source, max_examples - len(track_seen)
):
if len(track_seen) >= search_strategy.size_upper_bound:
break
if falsifying_examples:
break
if time_to_call_it_a_day():
break
args = search_strategy.produce_template(
build_context, parameter
)
if track_seen.track(args) > 1:
parameter_source.mark_bad()
continue
try:
setup_example()
a = None
try:
a = search_strategy.reify(args)
is_falsifying_example = not hypothesis(*a)
finally:
teardown_example(a)
except UnsatisfiedAssumption:
parameter_source.mark_bad()
continue
satisfying_examples += 1
if is_falsifying_example:
falsifying_examples.append(args)
run_time = time.time() - start_time
timed_out = self.timeout >= 0 and run_time >= self.timeout
if not falsifying_examples:
if (
satisfying_examples and
len(track_seen) >= search_strategy.size_lower_bound
):
raise Exhausted(
hypothesis, satisfying_examples)
elif satisfying_examples < min_satisfying_examples:
if timed_out:
raise Timeout(hypothesis, satisfying_examples, run_time)
else:
raise Unsatisfiable(
hypothesis, satisfying_examples, run_time)
else:
raise Unfalsifiable(hypothesis)
for example in falsifying_examples:
if not falsifies(example):
raise Flaky(hypothesis, example)
best_example = falsifying_examples[0]
for simpler in search_strategy.simplify_such_that(
random,
best_example, falsifies,
tracker=track_seen,
):
best_example = simpler
if time_to_call_it_a_day():
# We no cover in here because it's a bit sensitive to timing
# and tends to make tests flaky. There are tests that mean
# this is definitely covered most of the time.
break # pragma: no cover
if storage is not None:
storage.save(best_example)
setup_example()
return search_strategy.reify(best_example)
def test_can_digest_a_function_with_no_name():
def foo(x, y):
pass
function_digest(partial(foo, 1))
def test_lambda_digests_are_stable_across_process_runs():
digest = function_digest(lambda x: 42)
print(repr(digest))
assert digest == b'\x96=\xed6\x07\x1d\xb6:\x1a\x8b\x8eA\xd0\x19\xd7\x9c'
def run_state_machine_as_test(state_machine_factory, *, settings=None):
"""Run a state machine definition as a test, either silently doing nothing
or printing a minimal breaking program and raising an exception.
state_machine_factory is anything which returns an instance of
RuleBasedStateMachine when called with no arguments - it can be a class or a
function. settings will be used to control the execution of the test.
"""
if settings is None:
try:
settings = state_machine_factory.TestCase.settings
check_type(Settings, settings,
"state_machine_factory.TestCase.settings")
except AttributeError:
settings = Settings(deadline=None,
suppress_health_check=HealthCheck.all())
check_type(Settings, settings, "settings")
@settings
@given(st.data())
def run_state_machine(factory, data):
cd = data.conjecture_data
machine = factory()
check_type(RuleBasedStateMachine, machine, "state_machine_factory()")
cd.hypothesis_runner = machine
print_steps = (current_build_context().is_final
or current_verbosity() >= Verbosity.debug)
try:
if print_steps:
report("state = %s()" % (machine.__class__.__name__, ))
machine.check_invariants()
max_steps = settings.stateful_step_count
steps_run = 0
while True:
# We basically always want to run the maximum number of steps,
# but need to leave a small probability of terminating early
# in order to allow for reducing the number of steps once we
# find a failing test case, so we stop with probability of
# 2 ** -16 during normal operation but force a stop when we've
# generated enough steps.
cd.start_example(STATE_MACHINE_RUN_LABEL)
if steps_run == 0:
cd.draw_bits(16, forced=1)
elif steps_run >= max_steps:
cd.draw_bits(16, forced=0)
break
else:
# All we really care about is whether this value is zero
# or non-zero, so if it's > 1 we discard it and insert a
# replacement value after
cd.start_example(SHOULD_CONTINUE_LABEL)
should_continue_value = cd.draw_bits(16)
if should_continue_value > 1:
cd.stop_example(discard=True)
cd.draw_bits(16,
forced=int(bool(should_continue_value)))
else:
cd.stop_example()
if should_continue_value == 0:
break
steps_run += 1
# Choose a rule to run, preferring an initialize rule if there are
# any which have not been run yet.
if machine._initialize_rules_to_run:
init_rules = [
st.tuples(st.just(rule),
st.fixed_dictionaries(rule.arguments))
for rule in machine._initialize_rules_to_run
]
rule, data = cd.draw(st.one_of(init_rules))
machine._initialize_rules_to_run.remove(rule)
else:
rule, data = cd.draw(machine._rules_strategy)
# Pretty-print the values this rule was called with *before* calling
# _add_result_to_targets, to avoid printing arguments which are also
# a return value using the variable name they are assigned to.
# See https://github.com/HypothesisWorks/hypothesis/issues/2341
if print_steps:
data_to_print = {
k: machine._pretty_print(v)
for k, v in data.items()
}
# Assign 'result' here in case executing the rule fails below
result = multiple()
try:
data = dict(data)
for k, v in list(data.items()):
if isinstance(v, VarReference):
data[k] = machine.names_to_values[v.name]
result = rule.function(machine, **data)
if rule.targets:
if isinstance(result, MultipleResults):
for single_result in result.values:
machine._add_result_to_targets(
rule.targets, single_result)
else:
machine._add_result_to_targets(
rule.targets, result)
finally:
if print_steps:
# 'result' is only used if the step has target bundles.
# If it does, and the result is a 'MultipleResult',
# then 'print_step' prints a multi-variable assignment.
machine._print_step(rule, data_to_print, result)
machine.check_invariants()
cd.stop_example()
finally:
if print_steps:
report("state.teardown()")
machine.teardown()
# Use a machine digest to identify stateful tests in the example database
run_state_machine.hypothesis.inner_test._hypothesis_internal_add_digest = function_digest(
state_machine_factory)
# Copy some attributes so @seed and @reproduce_failure "just work"
run_state_machine._hypothesis_internal_use_seed = getattr(
state_machine_factory, "_hypothesis_internal_use_seed", None)
run_state_machine._hypothesis_internal_use_reproduce_failure = getattr(
state_machine_factory, "_hypothesis_internal_use_reproduce_failure",
None)
run_state_machine._hypothesis_internal_print_given_args = False
run_state_machine(state_machine_factory)
def test_digests_are_reasonably_unique():
assert (
function_digest(test_simple_conversion) !=
function_digest(test_does_not_error_on_dynamically_defined_functions))
