def __init__(self, specifier, settings):
self.specifier = specifier
self.leaf_strategy = strategy(specifier.leaf_values, settings)
self.branch_key_strategy = strategy(specifier.branch_keys, settings)
self.branch_label_strategy = strategy(specifier.branch_labels,
settings)
self.child_strategy = (lists(tuples(self.branch_key_strategy, self)))
def __init__(self, specifier, settings):
self.specifier = specifier
self.leaf_strategy = strategy(specifier.leaf_values, settings)
self.branch_key_strategy = strategy(
specifier.branch_keys, settings)
self.branch_label_strategy = strategy(
specifier.branch_labels, settings)
self.child_strategy = (
lists(tuples(self.branch_key_strategy, self)))
def define_rule(cls, targets, function, arguments):
converted_arguments = {}
for k, v in arguments.items():
if not isinstance(v, Bundle):
v = strategy(v)
converted_arguments[k] = v
if cls in cls._rules_per_class:
target = cls._rules_per_class[cls]
else:
target = cls._base_rules_per_class.setdefault(cls, [])
return target.append(
Rule(targets, function, converted_arguments)
)
def find(specifier, condition, settings=None, random=None, storage=None):
settings = settings or Settings(max_examples=2000, min_satisfying_examples=0, max_shrinks=2000)
from hypothesis.internal.strategymethod import strategy
search = strategy(specifier, settings)
if storage is None and settings.database is not None:
storage = settings.database.storage(
u"find(%s)" % (binascii.hexlify(function_digest(condition)).decode(u"ascii"),)
)
random = random or new_random()
successful_examples = [0]
def template_condition(template):
with BuildContext():
result = search.reify(template)
success = condition(result)
if success:
successful_examples[0] += 1
if not successful_examples[0]:
verbose_report(lambda: u"Trying example %s" % (repr(result),))
elif success:
if successful_examples[0] == 1:
verbose_report(lambda: u"Found satisfying example %s" % (repr(result),))
else:
verbose_report(lambda: u"Shrunk example to %s" % (repr(result),))
return success
template_condition.__name__ = condition.__name__
tracker = Tracker()
try:
template = best_satisfying_template(
search, random, template_condition, settings, tracker=tracker, max_parameter_tries=2, storage=storage
)
with BuildContext(is_final=True, close_on_capture=False):
return search.reify(template)
except Timeout:
raise
except NoSuchExample:
if search.template_upper_bound <= len(tracker):
raise DefinitelyNoSuchExample(get_pretty_function_description(condition), search.template_upper_bound)
raise NoSuchExample(get_pretty_function_description(condition))
def pack(self, source_and_morpher):
source, morpher = source_and_morpher
return morpher.become(strategy(self.expand(source)))
def wrapped_test(*arguments, **kwargs):
if settings.derandomize:
random = Random(function_digest(test))
else:
random = provided_random or new_random()
import hypothesis.strategies as sd
from hypothesis.internal.strategymethod import strategy
selfy = None
arguments, kwargs = convert_positional_arguments(
wrapped_test, arguments, kwargs)
for arg in hypothesis_owned_arguments:
try:
value = kwargs[arg]
except KeyError:
continue
if not isinstance(value, HypothesisProvided):
note_deprecation(
'Passing in explicit values to override Hypothesis '
'provided values is deprecated and will no longer '
'work in Hypothesis 2.0. If you need to do this, '
'extract a common function and call that from a '
'Hypothesis based test.', settings
)
# Anything in unused_kwargs hasn't been injected through
# argspec.defaults, so we need to add them.
for k in unused_kwargs:
if k not in kwargs:
kwargs[k] = unused_kwargs[k]
# If the test function is a method of some kind, the bound object
# will be the first named argument if there are any, otherwise the
# first vararg (if any).
if argspec.args:
selfy = kwargs.get(argspec.args[0])
elif arguments:
selfy = arguments[0]
if isinstance(selfy, HypothesisProvided):
selfy = None
test_runner = executor(selfy)
for example in reversed(getattr(
wrapped_test, u'hypothesis_explicit_examples', ()
)):
if example.args:
example_kwargs = dict(zip(
argspec.args[-len(example.args):], example.args
))
else:
example_kwargs = dict(example.kwargs)
for k, v in kwargs.items():
if not isinstance(v, HypothesisProvided):
example_kwargs[k] = v
# Note: Test may mutate arguments and we can't rerun explicit
# examples, so we have to calculate the failure message at this
# point rather than than later.
message_on_failure = u'Falsifying example: %s(%s)' % (
test.__name__, arg_string(test, arguments, example_kwargs)
)
try:
with BuildContext() as b:
test_runner(
lambda: test(*arguments, **example_kwargs)
)
except BaseException:
report(message_on_failure)
for n in b.notes:
report(n)
raise
if not any(
isinstance(x, HypothesisProvided)
for xs in (arguments, kwargs.values())
for x in xs
):
# All arguments have been satisfied without needing to invoke
# hypothesis
test_runner(lambda: test(*arguments, **kwargs))
return
def convert_to_specifier(v):
if isinstance(v, HypothesisProvided):
return strategy(v.value, settings)
else:
return sd.just(v)
given_specifier = sd.tuples(
sd.tuples(*map(convert_to_specifier, arguments)),
sd.fixed_dictionaries(dict(
(k, convert_to_specifier(v)) for (k, v) in kwargs.items()))
)
def fail_health_check(message):
message += (
'\nSee http://hypothesis.readthedocs.org/en/latest/health'
'checks.html for more information about this.'
)
if settings.strict:
raise FailedHealthCheck(message)
else:
warnings.warn(FailedHealthCheck(message))
search_strategy = strategy(given_specifier, settings)
search_strategy.validate()
if settings.database:
storage = settings.database.storage(
fully_qualified_name(test))
else:
storage = None
start = time.time()
warned_random = [False]
perform_health_check = settings.perform_health_check
if Settings.default is not None:
perform_health_check &= Settings.default.perform_health_check
if perform_health_check:
initial_state = getglobalrandomstate()
health_check_random = Random(random.getrandbits(128))
count = 0
bad_draws = 0
filtered_draws = 0
errors = 0
while (
count < 10 and time.time() < start + 1 and
filtered_draws < 50 and bad_draws < 50
):
try:
with Settings(settings, verbosity=Verbosity.quiet):
test_runner(reify_and_execute(
search_strategy,
search_strategy.draw_template(
health_check_random,
search_strategy.draw_parameter(
health_check_random,
)),
lambda *args, **kwargs: None,
))
count += 1
except BadTemplateDraw:
bad_draws += 1
except UnsatisfiedAssumption:
filtered_draws += 1
except Exception:
if errors == 0:
report(traceback.format_exc())
errors += 1
if test_runner is default_executor:
fail_health_check(
'An exception occurred during data '
'generation in initial health check. '
'This indicates a bug in the strategy. '
'This could either be a Hypothesis bug or '
"an error in a function you've passed to "
'it to construct your data.'
)
else:
fail_health_check(
'An exception occurred during data '
'generation in initial health check. '
'This indicates a bug in the strategy. '
'This could either be a Hypothesis bug or '
'an error in a function you\'ve passed to '
'it to construct your data. Additionally, '
'you have a custom executor, which means '
'that this could be your executor failing '
'to handle a function which returns None. '
)
if filtered_draws >= 50:
fail_health_check((
'It looks like your strategy is filtering out a lot '
'of data. Health check found %d filtered examples but '
'only %d good ones. This will make your tests much '
'slower, and also will probably distort the data '
'generation quite a lot. You should adapt your '
'strategy to filter less.') % (
filtered_draws, count
))
if bad_draws >= 50:
fail_health_check(
'Hypothesis is struggling to generate examples. '
'This is often a sign of a recursive strategy which '
'fans out too broadly. If you\'re using recursive, '
'try to reduce the size of the recursive step or '
'increase the maximum permitted number of leaves.'
)
runtime = time.time() - start
if runtime > 1.0 or count < 10:
fail_health_check((
'Data generation is extremely slow: Only produced '
'%d valid examples in %.2f seconds. Try decreasing '
"size of the data you're generating (with e.g."
'average_size or max_leaves parameters).'
) % (count, runtime))
if getglobalrandomstate() != initial_state:
warned_random[0] = True
fail_health_check(
'Data generation depends on global random module. '
'This makes results impossible to replay, which '
'prevents Hypothesis from working correctly. '
'If you want to use methods from random, use '
'randoms() from hypothesis.strategies to get an '
'instance of Random you can use. Alternatively, you '
'can use the random_module() strategy to explicitly '
'seed the random module.'
)
last_exception = [None]
repr_for_last_exception = [None]
def is_template_example(xs):
if perform_health_check and not warned_random[0]:
initial_state = getglobalrandomstate()
record_repr = [None]
try:
result = test_runner(reify_and_execute(
search_strategy, xs, test,
record_repr=record_repr,
))
if result is not None:
note_deprecation((
'Tests run under @given should return None, but '
'%s returned %r instead.'
'In Hypothesis 2.0 this will become an error.'
) % (test.__name__, result), settings)
return False
except HypothesisDeprecationWarning:
raise
except UnsatisfiedAssumption as e:
raise e
except Exception as e:
last_exception[0] = traceback.format_exc()
repr_for_last_exception[0] = record_repr[0]
verbose_report(last_exception[0])
return True
finally:
if (
not warned_random[0] and
perform_health_check and
getglobalrandomstate() != initial_state
):
warned_random[0] = True
fail_health_check(
'Your test used the global random module. '
'This is unlikely to work correctly. You should '
'consider using the randoms() strategy from '
'hypothesis.strategies instead. Alternatively, '
'you can use the random_module() strategy to '
'explicitly seed the random module.'
)
is_template_example.__name__ = test.__name__
is_template_example.__qualname__ = qualname(test)
falsifying_template = None
try:
falsifying_template = best_satisfying_template(
search_strategy, random, is_template_example,
settings, storage, start_time=start,
)
except NoSuchExample:
return
assert last_exception[0] is not None
with settings:
test_runner(reify_and_execute(
search_strategy, falsifying_template, test,
print_example=True, is_final=True
))
report(
u'Failed to reproduce exception. Expected: \n' +
last_exception[0],
)
test_runner(reify_and_execute(
search_strategy, falsifying_template,
test_is_flaky(test, repr_for_last_exception[0]),
print_example=True, is_final=True
))
def test_can_give_list_of_examples(self):
strategy(lists(strat)).example()
def wrapped_test(*arguments, **kwargs):
import hypothesis.strategies as sd
from hypothesis.internal.strategymethod import strategy
selfy = None
arguments, kwargs = convert_positional_arguments(
wrapped_test, arguments, kwargs)
# Anything in unused_kwargs hasn't been injected through
# argspec.defaults, so we need to add them.
for k in unused_kwargs:
if k not in kwargs:
kwargs[k] = unused_kwargs[k]
# If the test function is a method of some kind, the bound object
# will be the first named argument if there are any, otherwise the
# first vararg (if any).
if argspec.args:
selfy = kwargs.get(argspec.args[0])
elif arguments:
selfy = arguments[0]
if isinstance(selfy, HypothesisProvided):
selfy = None
test_runner = executor(selfy)
for example in getattr(
wrapped_test, u'hypothesis_explicit_examples', ()
):
if example.args:
example_kwargs = dict(zip(
argspec.args[-len(example.args):], example.args
))
else:
example_kwargs = dict(example.kwargs)
for k, v in kwargs.items():
if not isinstance(v, HypothesisProvided):
example_kwargs[k] = v
# Note: Test may mutate arguments and we can't rerun explicit
# examples, so we have to calculate the failure message at this
# point rather than than later.
message_on_failure = u'Falsifying example: %s(%s)' % (
test.__name__, arg_string(test, arguments, example_kwargs)
)
try:
test_runner(
lambda: test(*arguments, **example_kwargs)
)
except BaseException:
report(message_on_failure)
raise
if not any(
isinstance(x, HypothesisProvided)
for xs in (arguments, kwargs.values())
for x in xs
):
# All arguments have been satisfied without needing to invoke
# hypothesis
test_runner(lambda: test(*arguments, **kwargs))
return
def convert_to_specifier(v):
if isinstance(v, HypothesisProvided):
return strategy(v.value, settings)
else:
return sd.just(v)
given_specifier = sd.tuples(
sd.tuples(*map(convert_to_specifier, arguments)),
sd.fixed_dictionaries(dict(
(k, convert_to_specifier(v)) for (k, v) in kwargs.items()))
)
search_strategy = strategy(given_specifier, settings)
if settings.database:
storage = settings.database.storage(
fully_qualified_name(test))
else:
storage = None
last_exception = [None]
repr_for_last_exception = [None]
def is_template_example(xs):
record_repr = [None]
try:
test_runner(reify_and_execute(
search_strategy, xs, test,
always_print=settings.max_shrinks <= 0,
record_repr=record_repr,
))
return False
except UnsatisfiedAssumption as e:
raise e
except Exception as e:
if settings.max_shrinks <= 0:
raise e
last_exception[0] = traceback.format_exc()
repr_for_last_exception[0] = record_repr[0]
verbose_report(last_exception[0])
return True
is_template_example.__name__ = test.__name__
is_template_example.__qualname__ = qualname(test)
falsifying_template = None
try:
falsifying_template = best_satisfying_template(
search_strategy, random, is_template_example,
settings, storage
)
except NoSuchExample:
return
assert last_exception[0] is not None
with settings:
test_runner(reify_and_execute(
search_strategy, falsifying_template, test,
print_example=True, is_final=True
))
report(
u'Failed to reproduce exception. Expected: \n' +
last_exception[0],
)
test_runner(reify_and_execute(
search_strategy, falsifying_template,
test_is_flaky(test, repr_for_last_exception[0]),
print_example=True, is_final=True
))
def templates_for_strategy(specifier, settings):
return TemplatesStrategy(strategy(specifier.base, settings))
def wrapped_test(*arguments, **kwargs):
if settings.derandomize:
random = Random(function_digest(test))
else:
random = provided_random or new_random()
import hypothesis.strategies as sd
from hypothesis.internal.strategymethod import strategy
selfy = None
arguments, kwargs = convert_positional_arguments(
wrapped_test, arguments, kwargs)
for arg in hypothesis_owned_arguments:
try:
value = kwargs[arg]
except KeyError:
continue
if not isinstance(value, HypothesisProvided):
note_deprecation(
'Passing in explicit values to override Hypothesis '
'provided values is deprecated and will no longer '
'work in Hypothesis 2.0. If you need to do this, '
'extract a common function and call that from a '
'Hypothesis based test.', settings)
# Anything in unused_kwargs hasn't been injected through
# argspec.defaults, so we need to add them.
for k in unused_kwargs:
if k not in kwargs:
kwargs[k] = unused_kwargs[k]
# If the test function is a method of some kind, the bound object
# will be the first named argument if there are any, otherwise the
# first vararg (if any).
if argspec.args:
selfy = kwargs.get(argspec.args[0])
elif arguments:
selfy = arguments[0]
if isinstance(selfy, HypothesisProvided):
selfy = None
test_runner = executor(selfy)
for example in reversed(
getattr(wrapped_test, u'hypothesis_explicit_examples',
())):
if example.args:
example_kwargs = dict(
zip(argspec.args[-len(example.args):], example.args))
else:
example_kwargs = dict(example.kwargs)
for k, v in kwargs.items():
if not isinstance(v, HypothesisProvided):
example_kwargs[k] = v
# Note: Test may mutate arguments and we can't rerun explicit
# examples, so we have to calculate the failure message at this
# point rather than than later.
message_on_failure = u'Falsifying example: %s(%s)' % (
test.__name__, arg_string(test, arguments, example_kwargs))
try:
with BuildContext() as b:
test_runner(lambda: test(*arguments, **example_kwargs))
except BaseException:
report(message_on_failure)
for n in b.notes:
report(n)
raise
if not any(
isinstance(x, HypothesisProvided)
for xs in (arguments, kwargs.values()) for x in xs):
# All arguments have been satisfied without needing to invoke
# hypothesis
test_runner(lambda: test(*arguments, **kwargs))
return
def convert_to_specifier(v):
if isinstance(v, HypothesisProvided):
return strategy(v.value, settings)
else:
return sd.just(v)
given_specifier = sd.tuples(
sd.tuples(*map(convert_to_specifier, arguments)),
sd.fixed_dictionaries(
dict((k, convert_to_specifier(v))
for (k, v) in kwargs.items())))
def fail_health_check(message):
message += (
'\nSee http://hypothesis.readthedocs.org/en/latest/health'
'checks.html for more information about this.')
if settings.strict:
raise FailedHealthCheck(message)
else:
warnings.warn(FailedHealthCheck(message))
search_strategy = strategy(given_specifier, settings)
search_strategy.validate()
if settings.database:
storage = settings.database.storage(fully_qualified_name(test))
else:
storage = None
start = time.time()
warned_random = [False]
perform_health_check = settings.perform_health_check
if Settings.default is not None:
perform_health_check &= Settings.default.perform_health_check
if perform_health_check:
initial_state = getglobalrandomstate()
health_check_random = Random(random.getrandbits(128))
count = 0
bad_draws = 0
filtered_draws = 0
errors = 0
while (count < 10 and time.time() < start + 1
and filtered_draws < 50 and bad_draws < 50):
try:
with Settings(settings, verbosity=Verbosity.quiet):
test_runner(
reify_and_execute(
search_strategy,
search_strategy.draw_template(
health_check_random,
search_strategy.draw_parameter(
health_check_random, )),
lambda *args, **kwargs: None,
))
count += 1
except BadTemplateDraw:
bad_draws += 1
except UnsatisfiedAssumption:
filtered_draws += 1
except Exception:
if errors == 0:
report(traceback.format_exc())
errors += 1
if test_runner is default_executor:
fail_health_check(
'An exception occurred during data '
'generation in initial health check. '
'This indicates a bug in the strategy. '
'This could either be a Hypothesis bug or '
"an error in a function you've passed to "
'it to construct your data.')
else:
fail_health_check(
'An exception occurred during data '
'generation in initial health check. '
'This indicates a bug in the strategy. '
'This could either be a Hypothesis bug or '
'an error in a function you\'ve passed to '
'it to construct your data. Additionally, '
'you have a custom executor, which means '
'that this could be your executor failing '
'to handle a function which returns None. ')
if filtered_draws >= 50:
fail_health_check((
'It looks like your strategy is filtering out a lot '
'of data. Health check found %d filtered examples but '
'only %d good ones. This will make your tests much '
'slower, and also will probably distort the data '
'generation quite a lot. You should adapt your '
'strategy to filter less.') % (filtered_draws, count))
if bad_draws >= 50:
fail_health_check(
'Hypothesis is struggling to generate examples. '
'This is often a sign of a recursive strategy which '
'fans out too broadly. If you\'re using recursive, '
'try to reduce the size of the recursive step or '
'increase the maximum permitted number of leaves.')
runtime = time.time() - start
if runtime > 1.0 or count < 10:
fail_health_check(
('Data generation is extremely slow: Only produced '
'%d valid examples in %.2f seconds. Try decreasing '
"size of the data you're generating (with e.g."
'average_size or max_leaves parameters).') %
(count, runtime))
if getglobalrandomstate() != initial_state:
warned_random[0] = True
fail_health_check(
'Data generation depends on global random module. '
'This makes results impossible to replay, which '
'prevents Hypothesis from working correctly. '
'If you want to use methods from random, use '
'randoms() from hypothesis.strategies to get an '
'instance of Random you can use. Alternatively, you '
'can use the random_module() strategy to explicitly '
'seed the random module.')
last_exception = [None]
repr_for_last_exception = [None]
def is_template_example(xs):
if perform_health_check and not warned_random[0]:
initial_state = getglobalrandomstate()
record_repr = [None]
try:
result = test_runner(
reify_and_execute(
search_strategy,
xs,
test,
record_repr=record_repr,
))
if result is not None:
note_deprecation(
('Tests run under @given should return None, but '
'%s returned %r instead.'
'In Hypothesis 2.0 this will become an error.') %
(test.__name__, result), settings)
return False
except HypothesisDeprecationWarning:
raise
except UnsatisfiedAssumption as e:
raise e
except Exception as e:
last_exception[0] = traceback.format_exc()
repr_for_last_exception[0] = record_repr[0]
verbose_report(last_exception[0])
return True
finally:
if (not warned_random[0] and perform_health_check
and getglobalrandomstate() != initial_state):
warned_random[0] = True
fail_health_check(
'Your test used the global random module. '
'This is unlikely to work correctly. You should '
'consider using the randoms() strategy from '
'hypothesis.strategies instead. Alternatively, '
'you can use the random_module() strategy to '
'explicitly seed the random module.')
is_template_example.__name__ = test.__name__
is_template_example.__qualname__ = qualname(test)
falsifying_template = None
try:
falsifying_template = best_satisfying_template(
search_strategy,
random,
is_template_example,
settings,
storage,
start_time=start,
)
except NoSuchExample:
return
assert last_exception[0] is not None
with settings:
test_runner(
reify_and_execute(search_strategy,
falsifying_template,
test,
print_example=True,
is_final=True))
report(
u'Failed to reproduce exception. Expected: \n' +
last_exception[0], )
test_runner(
reify_and_execute(search_strategy,
falsifying_template,
test_is_flaky(
test, repr_for_last_exception[0]),
print_example=True,
is_final=True))
def pack(self, source_and_morpher):
source, morpher = source_and_morpher
return morpher.become(strategy(self.expand(source)))
def templates_for(strat):
return TemplatesStrategy(strategy(strat))
def templates_for_strategy(specifier, settings):
return TemplatesStrategy(strategy(specifier.base, settings))
def strategy_test_suite(
specifier,
max_examples=10,
random=None,
):
settings = Settings(
database=None,
max_examples=max_examples,
max_iterations=max_examples * 2,
min_satisfying_examples=2,
average_list_length=2.0,
)
random = random or Random()
strat = strategy(specifier, settings)
specifier_test = given(templates_for(specifier),
randoms(),
settings=settings)
class ValidationSuite(TestCase):
def __repr__(self):
return u'strategy_test_suite(%s)' % (repr(specifier), )
@given(specifier, settings=settings)
def test_does_not_error(self, value):
pass
def test_can_give_example(self):
strat.example()
def test_can_give_list_of_examples(self):
strategy(lists(strat)).example()
def test_will_give_unsatisfiable_if_all_rejected(self):
@given(specifier, settings=settings)
def nope(x):
assume(False)
self.assertRaises(Unsatisfiable, nope)
def test_will_find_a_constant_failure(self):
@given(specifier, settings=settings)
def nope(x):
raise Rejected()
self.assertRaises(Rejected, nope)
def test_will_find_a_failure_from_the_database(self):
db = ExampleDatabase()
@given(specifier,
settings=Settings(settings, max_examples=10, database=db))
def nope(x):
raise Rejected()
try:
for i in hrange(3):
self.assertRaises(Rejected, nope) # pragma: no cover
finally:
db.close()
@given(templates_for(specifier),
templates_for(specifier),
settings=settings)
def test_simplicity_is_asymmetric(self, x, y):
assert not (strat.strictly_simpler(x, y)
and strat.strictly_simpler(y, x))
@given(integers(), settings=settings)
def test_templates_generated_from_same_random_are_equal(self, i):
try:
t1 = strat.draw_and_produce(Random(i))
t2 = strat.draw_and_produce(Random(i))
except BadTemplateDraw:
assume(False)
if t1 is not t2:
assert t1 == t2
assert hash(t1) == hash(t2)
@given(integers(), settings=settings)
def test_templates_generated_from_same_random_are_equal_after_reify(
self, i):
try:
t1 = strat.draw_and_produce(Random(i))
t2 = strat.draw_and_produce(Random(i))
except BadTemplateDraw:
assume(False)
if t1 is not t2:
with BuildContext():
strat.reify(t1)
with BuildContext():
strat.reify(t2)
assert t1 == t2
assert hash(t1) == hash(t2)
@given(randoms(), settings=settings)
def test_will_handle_a_really_weird_failure(self, rnd):
db = ExampleDatabase()
@given(specifier,
settings=Settings(
settings,
database=db,
max_examples=max_examples,
min_satisfying_examples=2,
average_list_length=2.0,
),
random=rnd)
def nope(x):
s = hashlib.sha1(repr(x).encode(u'utf-8')).digest()
assert Random(s).randint(0, 1) == Random(s).randint(0, 1)
if Random(s).randint(0, 1):
raise Rejected(u'%r with digest %r' % (x, s))
try:
try:
nope()
except Rejected:
pass
try:
nope()
except Rejected:
pass
finally:
db.close()
@specifier_test
def test_is_basic(self, value, rnd):
def is_basic(v):
if v is None or isinstance(v, text_type):
return True
if isinstance(v, integer_types):
return not (abs(v) >> 64)
if isinstance(v, list):
return all(is_basic(w) for w in v)
return False
supposedly_basic = strat.to_basic(value)
self.assertTrue(is_basic(supposedly_basic), repr(supposedly_basic))
@specifier_test
def test_only_raises_bad_data_in_from_basic(self, value, rnd):
basic = strat.to_basic(value)
messed_basic = mutate_basic(basic, rnd)
try:
strat.from_basic(messed_basic)
except BadData:
pass
@specifier_test
def test_can_round_trip_through_the_database(self, template, rnd):
empty_db = ExampleDatabase(backend=SQLiteBackend(u':memory:'), )
try:
storage = empty_db.storage(u'round trip')
storage.save(template, strat)
values = list(storage.fetch(strat))
assert len(values) == 1
assert strat.to_basic(template) == strat.to_basic(values[0])
finally:
empty_db.close()
@specifier_test
def test_template_is_hashable(self, template, rnd):
hash(template)
# It can be easy to forget to convert a list...
hash(strat.from_basic(strat.to_basic(template)))
@specifier_test
def test_can_minimize_to_empty(self, template, rnd):
simplest = template
tracker = Tracker()
while True:
for t in strat.full_simplify(rnd, simplest):
if tracker.track(t) == 1:
simplest = t
break
else:
break
assert list(strat.full_simplify(rnd, simplest)) == []
@specifier_test
def test_full_simplify_completes(self, template, rnd):
# Cut off at 100 for the occasional case where we get
# really very large templates which have too many simplifies.
for x in islice(strat.full_simplify(rnd, template), 100):
pass
@specifier_test
def test_does_not_increase_complexity(self, template, rnd):
for s in islice(strat.full_simplify(rnd, template), 100):
assert not strat.strictly_simpler(template, s)
@given(randoms(), settings=Settings(settings, max_examples=100))
def test_can_create_templates(self, random):
parameter = strat.draw_parameter(random)
try:
strat.draw_template(random, parameter)
except BadTemplateDraw:
assume(False)
return ValidationSuite
def wrapped_test(*arguments, **kwargs):
import hypothesis.strategies as sd
from hypothesis.internal.strategymethod import strategy
selfy = None
arguments, kwargs = convert_positional_arguments(
wrapped_test, arguments, kwargs)
# Anything in unused_kwargs hasn't been injected through
# argspec.defaults, so we need to add them.
for k in unused_kwargs:
if k not in kwargs:
kwargs[k] = unused_kwargs[k]
# If the test function is a method of some kind, the bound object
# will be the first named argument if there are any, otherwise the
# first vararg (if any).
if argspec.args:
selfy = kwargs.get(argspec.args[0])
elif arguments:
selfy = arguments[0]
if isinstance(selfy, HypothesisProvided):
selfy = None
test_runner = executor(selfy)
for example in getattr(wrapped_test,
u'hypothesis_explicit_examples', ()):
if example.args:
example_kwargs = dict(
zip(argspec.args[-len(example.args):], example.args))
else:
example_kwargs = dict(example.kwargs)
for k, v in kwargs.items():
if not isinstance(v, HypothesisProvided):
example_kwargs[k] = v
# Note: Test may mutate arguments and we can't rerun explicit
# examples, so we have to calculate the failure message at this
# point rather than than later.
message_on_failure = u'Falsifying example: %s(%s)' % (
test.__name__, arg_string(test, arguments, example_kwargs))
try:
test_runner(lambda: test(*arguments, **example_kwargs))
except BaseException:
report(message_on_failure)
raise
if not any(
isinstance(x, HypothesisProvided)
for xs in (arguments, kwargs.values()) for x in xs):
# All arguments have been satisfied without needing to invoke
# hypothesis
test_runner(lambda: test(*arguments, **kwargs))
return
def convert_to_specifier(v):
if isinstance(v, HypothesisProvided):
return strategy(v.value, settings)
else:
return sd.just(v)
given_specifier = sd.tuples(
sd.tuples(*map(convert_to_specifier, arguments)),
sd.fixed_dictionaries(
dict((k, convert_to_specifier(v))
for (k, v) in kwargs.items())))
search_strategy = strategy(given_specifier, settings)
if settings.database:
storage = settings.database.storage(fully_qualified_name(test))
else:
storage = None
last_exception = [None]
repr_for_last_exception = [None]
def is_template_example(xs):
record_repr = [None]
try:
test_runner(
reify_and_execute(
search_strategy,
xs,
test,
always_print=settings.max_shrinks <= 0,
record_repr=record_repr,
))
return False
except UnsatisfiedAssumption as e:
raise e
except Exception as e:
if settings.max_shrinks <= 0:
raise e
last_exception[0] = traceback.format_exc()
repr_for_last_exception[0] = record_repr[0]
verbose_report(last_exception[0])
return True
is_template_example.__name__ = test.__name__
is_template_example.__qualname__ = qualname(test)
falsifying_template = None
try:
falsifying_template = best_satisfying_template(
search_strategy, random, is_template_example, settings,
storage)
except NoSuchExample:
return
assert last_exception[0] is not None
with settings:
test_runner(
reify_and_execute(search_strategy,
falsifying_template,
test,
print_example=True,
is_final=True))
report(
u'Failed to reproduce exception. Expected: \n' +
last_exception[0], )
test_runner(
reify_and_execute(search_strategy,
falsifying_template,
test_is_flaky(
test, repr_for_last_exception[0]),
print_example=True,
is_final=True))
def test_can_give_list_of_examples(self):
strategy(lists(strat)).example()
def convert_to_specifier(v):
if isinstance(v, HypothesisProvided):
return strategy(v.value, settings)
else:
return sd.just(v)
def templates_for(strat):
return TemplatesStrategy(strategy(strat))
def find(specifier, condition, settings=None, random=None, storage=None):
settings = settings or Settings(
max_examples=2000,
min_satisfying_examples=0,
max_shrinks=2000,
)
from hypothesis.internal.strategymethod import strategy
search = strategy(specifier, settings)
if storage is None and settings.database is not None:
storage = settings.database.storage(
u'find(%s)' %
(binascii.hexlify(function_digest(condition)).decode(u'ascii'), ))
random = random or Random()
successful_examples = [0]
def template_condition(template):
with BuildContext():
result = search.reify(template)
success = condition(result)
if success:
successful_examples[0] += 1
if not successful_examples[0]:
verbose_report(lambda: u'Trying example %s' % (repr(result), ))
elif success:
if successful_examples[0] == 1:
verbose_report(lambda: u'Found satisfying example %s' %
(repr(result), ))
else:
verbose_report(lambda: u'Shrunk example to %s' %
(repr(result), ))
return success
template_condition.__name__ = condition.__name__
tracker = Tracker()
try:
template = best_satisfying_template(
search,
random,
template_condition,
settings,
tracker=tracker,
max_parameter_tries=2,
storage=storage,
)
with BuildContext(is_final=True, close_on_capture=False):
return search.reify(template)
except Timeout:
raise
except NoSuchExample:
if search.template_upper_bound <= len(tracker):
raise DefinitelyNoSuchExample(
get_pretty_function_description(condition),
search.template_upper_bound,
)
raise NoSuchExample(get_pretty_function_description(condition))
def convert_to_specifier(v):
if isinstance(v, HypothesisProvided):
return strategy(v.value, settings)
else:
return sd.just(v)
def strategy_test_suite(
specifier,
max_examples=10, random=None,
):
settings = Settings(
database=None,
max_examples=max_examples,
max_iterations=max_examples * 2,
min_satisfying_examples=2,
average_list_length=2.0,
)
random = random or Random()
strat = strategy(specifier, settings)
specifier_test = given(
templates_for(specifier), randoms(), settings=settings
)
class ValidationSuite(TestCase):
def __repr__(self):
return u'strategy_test_suite(%s)' % (
repr(specifier),
)
@given(specifier, settings=settings)
def test_does_not_error(self, value):
pass
def test_can_give_example(self):
strat.example()
def test_can_give_list_of_examples(self):
strategy(lists(strat)).example()
def test_will_give_unsatisfiable_if_all_rejected(self):
@given(specifier, settings=settings)
def nope(x):
assume(False)
self.assertRaises(Unsatisfiable, nope)
def test_will_find_a_constant_failure(self):
@given(specifier, settings=settings)
def nope(x):
raise Rejected()
self.assertRaises(Rejected, nope)
def test_will_find_a_failure_from_the_database(self):
db = ExampleDatabase()
@given(specifier, settings=Settings(
settings, max_examples=10, database=db))
def nope(x):
raise Rejected()
try:
for i in hrange(3):
self.assertRaises(Rejected, nope) # pragma: no cover
finally:
db.close()
@given(
templates_for(specifier), templates_for(specifier),
settings=settings
)
def test_simplicity_is_asymmetric(self, x, y):
assert not (
strat.strictly_simpler(x, y) and
strat.strictly_simpler(y, x)
)
@given(integers(), settings=settings)
def test_templates_generated_from_same_random_are_equal(self, i):
try:
t1 = strat.draw_and_produce(Random(i))
t2 = strat.draw_and_produce(Random(i))
except BadTemplateDraw:
assume(False)
if t1 is not t2:
assert t1 == t2
assert hash(t1) == hash(t2)
@given(integers(), settings=settings)
def test_templates_generated_from_same_random_are_equal_after_reify(
self, i
):
try:
t1 = strat.draw_and_produce(Random(i))
t2 = strat.draw_and_produce(Random(i))
except BadTemplateDraw:
assume(False)
if t1 is not t2:
with BuildContext():
strat.reify(t1)
with BuildContext():
strat.reify(t2)
assert t1 == t2
assert hash(t1) == hash(t2)
@given(randoms(), settings=settings)
def test_will_handle_a_really_weird_failure(self, rnd):
db = ExampleDatabase()
@given(
specifier,
settings=Settings(
settings,
database=db,
max_examples=max_examples,
min_satisfying_examples=2,
average_list_length=2.0,
), random=rnd
)
def nope(x):
s = hashlib.sha1(repr(x).encode(u'utf-8')).digest()
assert Random(s).randint(0, 1) == Random(s).randint(0, 1)
if Random(s).randint(0, 1):
raise Rejected(u'%r with digest %r' % (
x, s
))
try:
try:
nope()
except Rejected:
pass
try:
nope()
except Rejected:
pass
finally:
db.close()
@specifier_test
def test_is_basic(self, value, rnd):
def is_basic(v):
if v is None or isinstance(v, text_type):
return True
if isinstance(v, integer_types):
return not (abs(v) >> 64)
if isinstance(v, list):
return all(is_basic(w) for w in v)
return False
supposedly_basic = strat.to_basic(value)
self.assertTrue(is_basic(supposedly_basic), repr(supposedly_basic))
@specifier_test
def test_only_raises_bad_data_in_from_basic(self, value, rnd):
basic = strat.to_basic(value)
messed_basic = mutate_basic(basic, rnd)
try:
strat.from_basic(messed_basic)
except BadData:
pass
@specifier_test
def test_can_round_trip_through_the_database(self, template, rnd):
empty_db = ExampleDatabase(
backend=SQLiteBackend(u':memory:'),
)
try:
storage = empty_db.storage(u'round trip')
storage.save(template, strat)
values = list(storage.fetch(strat))
assert len(values) == 1
assert strat.to_basic(template) == strat.to_basic(values[0])
finally:
empty_db.close()
@specifier_test
def test_template_is_hashable(self, template, rnd):
hash(template)
# It can be easy to forget to convert a list...
hash(strat.from_basic(strat.to_basic(template)))
@specifier_test
def test_can_minimize_to_empty(self, template, rnd):
simplest = template
tracker = Tracker()
while True:
for t in strat.full_simplify(rnd, simplest):
if tracker.track(t) == 1:
simplest = t
break
else:
break
assert list(strat.full_simplify(rnd, simplest)) == []
@specifier_test
def test_full_simplify_completes(self, template, rnd):
# Cut off at 100 for the occasional case where we get
# really very large templates which have too many simplifies.
for x in islice(strat.full_simplify(rnd, template), 100):
pass
@specifier_test
def test_does_not_increase_complexity(self, template, rnd):
for s in islice(strat.full_simplify(rnd, template), 100):
assert not strat.strictly_simpler(template, s)
@given(randoms(), settings=Settings(settings, max_examples=100))
def test_can_create_templates(self, random):
parameter = strat.draw_parameter(random)
try:
strat.draw_template(random, parameter)
except BadTemplateDraw:
assume(False)
return ValidationSuite
