def test_strategy_for_integer_range_produces_only_integers_in_that_range():
just_one_integer = strategy(specifiers.IntegerRange(1, 1))
for _ in hrange(100):
pv = just_one_integer.draw_parameter(random)
x = just_one_integer.produce_template(BuildContext(random), pv)
assert x == 1
some_integers = strategy(specifiers.IntegerRange(1, 10))
for _ in hrange(100):
pv = some_integers.produce_parameter(random)
x = some_integers.produce_template(BuildContext(random), pv)
assert 1 <= x <= 10
def test_errors_if_you_mark_bad_before_fetching():
source = ParameterSource(
context=BuildContext(random.Random()),
strategy=strategy(int),
)
with pytest.raises(ValueError):
source.mark_bad()
def test_can_perform_all_basic_operations(self, random):
parameter = strat.draw_parameter(random)
template = strat.draw_template(BuildContext(random), parameter)
minimal_template = list(
strat.simplify_such_that(template, lambda x: True))[-1]
strat.reify(minimal_template)
assert (strat.to_basic(minimal_template) == strat.to_basic(
strat.from_basic(strat.to_basic(minimal_template))))
def new_template(self, seed, parameter_seed):
context = BuildContext(Random(seed))
parameter = self.source_strategy.draw_parameter(Random(parameter_seed))
def templates():
while True:
yield self.source_strategy.draw_template(context, parameter)
return StreamTemplate(seed, parameter_seed, templates())
def test_errors_if_you_mark_bad_twice():
source = ParameterSource(
context=BuildContext(random.Random()),
strategy=strategy(int),
)
next(iter(source))
source.mark_bad()
with pytest.raises(ValueError):
source.mark_bad()
def test_can_perform_all_basic_operations(specifier, random):
strat = strategy(specifier, settings)
parameter = strat.draw_parameter(random)
template = strat.produce_template(BuildContext(random), parameter)
assert (strat.to_basic(template) == strat.to_basic(
strat.from_basic(strat.to_basic(template))))
minimal_template = last(strat.simplify_such_that(template, lambda x: True))
strat.reify(minimal_template)
assert (strat.to_basic(minimal_template) == strat.to_basic(
strat.from_basic(strat.to_basic(minimal_template))))
def test_named_tuples_always_produce_named_tuples():
s = strategy(Litter(int, int))
for i in hrange(100):
assert isinstance(
s.produce_template(BuildContext(random),
s.produce_parameter(random)), Litter)
for x in s.simplify(Litter(100, 100)):
assert isinstance(x, Litter)
def test_strategy_for_integer_range_can_produce_end_points():
some_integers = strategy(specifiers.IntegerRange(1, 10))
found = set()
for _ in hrange(1000): # pragma: no branch
pv = some_integers.produce_parameter(random)
x = some_integers.produce_template(BuildContext(random), pv)
found.add(x)
if 1 in found and 10 in found:
break
else:
assert False # pragma: no cover
assert 1 in found
assert 10 in found
def test_tries_each_parameter_at_least_min_index_times():
source = ParameterSource(context=BuildContext(random.Random()),
strategy=strategy(int),
min_tries=5)
i = 0
for x in source.examples():
i += 1
if i > 500:
break
if i % 2:
source.mark_bad()
# The last index may not have been fully populated
assert all(c >= 5 for c in source.counts[:-1])
def test_negative_is_not_too_far_off_mean():
source = ParameterSource(
context=BuildContext(random.Random()),
strategy=strategy(int),
)
positive = 0
i = 0
for example in source.examples():
if example >= 0:
positive += 1
i += 1
if i >= N_EXAMPLES:
break
assert 0.3 <= float(positive) / N_EXAMPLES <= 0.7
def test_marking_negative_avoids_similar_examples():
source = ParameterSource(
context=BuildContext(random.Random()),
strategy=strategy(int),
)
positive = 0
i = 0
for example in source.examples():
if example >= 0:
positive += 1
else:
source.mark_bad()
i += 1
if i >= N_EXAMPLES:
break
assert float(positive) / N_EXAMPLES >= 0.8
def test_can_grow_the_set_of_available_parameters_if_doing_badly():
runs = 10
number_grown = 0
for _ in hrange(runs):
source = ParameterSource(
context=BuildContext(random.Random()),
strategy=strategy(int),
min_parameters=1,
)
i = 0
for example in source.examples():
if example < 0:
source.mark_bad()
i += 1
if i >= 100:
break
if len(source.parameters) > 1:
number_grown += 1
assert len(source.parameters) < 100
def run_test():
if condition is None:
_condition = lambda x: True
condition_string = ''
else:
_condition = condition
condition_string = strip_lambda(
reflection.get_pretty_function_description(condition))
count = 0
successful_runs = 0
s = strategy(specifier)
for _ in hrange(MAX_RUNS):
pv = s.draw_parameter(random)
x = s.reify(s.draw_template(BuildContext(random), pv))
if not _condition(x):
continue
successful_runs += 1
if predicate(x):
count += 1
if successful_runs < MIN_RUNS:
raise ConditionTooHard(
('Unable to find enough examples satisfying predicate %s '
'only found %d but required at least %d for validity') %
(condition_string, successful_runs, MIN_RUNS))
result = Result(
count,
successful_runs,
q,
predicate,
condition_string,
)
p = cumulative_binomial_probability(successful_runs, q, count)
run_test.test_result = result
# The test passes if we fail to reject the null hypothesis that
# the probability is at least q
if p < REQUIRED_P:
result.failed = True
raise HypothesisFalsified(result.description() + ' rejected')
def test_can_recover_from_bad_data_in_mapped_strategy(r):
param = OrderedPairs.draw_parameter(r)
template = OrderedPairs.draw_template(BuildContext(r), param)
OrderedPairs.reify(template)
for simplification in OrderedPairs.full_simplify(r, template):
if isinstance(simplification, OrderedPairs.TemplateFromTemplate):
break
else:
assume(False)
assume(isinstance(simplification, OrderedPairs.TemplateFromTemplate))
basic = OrderedPairs.to_basic(simplification)
assert len(basic) == 4
assert isinstance(basic, list)
assert isinstance(basic[-1], list)
basic[-1] = 1
new_template = OrderedPairs.from_basic(basic)
assert isinstance(new_template, OrderedPairs.TemplateFromBasic)
reified = OrderedPairs.reify(new_template)
assert type(reified) == tuple
x, y = reified
assert x < y
def simplify(self, davt):
random = RandomWithSeed(davt.random)
for d in self.specifier_strategy.simplify(davt.specifier):
new_template = self.strategy(
self.specifier_strategy.reify(d)).draw_and_produce(
BuildContext(random))
yield DescriptorWithValue(
specifier=d,
template=new_template,
value=None,
random=davt.random,
)
strat = self.strategy(self.specifier_strategy.reify(davt.specifier))
for v in strat.simplify(davt.template):
yield DescriptorWithValue(
specifier=davt.specifier,
template=v,
value=None,
random=davt.random,
)
def basic_simplify(self, random, davt):
assert isinstance(random, Random)
for d in self.specifier_strategy.full_simplify(random, davt.specifier):
new_template = self.strategy(
self.specifier_strategy.reify(d)).draw_and_produce(
BuildContext(random))
yield DescriptorWithValue(
specifier=d,
template=new_template,
value=None,
random=davt.random,
)
strat = self.strategy(
self.specifier_strategy.reify(davt.specifier))
for v in strat.full_simplify(random, davt.template):
yield DescriptorWithValue(
specifier=davt.specifier,
template=v,
value=None,
random=davt.random,
)
def test_random_repr_has_seed():
strat = strategy(random.Random)
rnd = strat.reify(
strat.produce_template(BuildContext(random.Random()), None))
seed = rnd.seed
assert text_type(seed) in repr(rnd)
def test_can_create_templates(self, random):
parameter = strat.draw_parameter(random)
strat.draw_template(BuildContext(random), parameter)
def test_float_strategy_does_not_overflow():
s = strategy(float)
for _ in hrange(100):
s.produce_template(BuildContext(random), s.produce_parameter(random))
def test_random_only_produces_special_random():
st = strategy(random.Random)
assert isinstance(
st.reify(
st.produce_template(BuildContext(random),
st.draw_parameter(random))), RandomWithSeed)
def test_template_is_hashable(specifier, random):
strat = strategy(specifier, settings)
parameter = strat.draw_parameter(random)
template = strat.produce_template(BuildContext(random), parameter)
hash(template)
def test_copies_all_its_values_correctly(desc, random):
strat = strategy(desc, settings)
value = strat.produce_template(
BuildContext(random), strat.draw_parameter(random))
assert show(strat.reify(value)) == show(strat.reify(value))
def falsify(self, hypothesis, *argument_types, **kwargs): # pylint: disable=too-many-locals,too-many-branches
"""
Attempt to construct an example tuple x matching argument_types such
that hypothesis(*x) returns a falsey value
"""
teardown_example = kwargs.get('teardown_example') or (lambda x: None)
setup_example = kwargs.get('setup_example') or (lambda: None)
random = self.random
if random is None:
random = Random(function_digest(hypothesis))
build_context = BuildContext(random)
search_strategy = strategy(argument_types, self.settings)
storage = None
if self.database is not None:
storage = self.database.storage_for(argument_types)
def falsifies(args): # pylint: disable=missing-docstring
example = None
try:
try:
setup_example()
example = search_strategy.reify(args)
return not hypothesis(*example)
except UnsatisfiedAssumption:
return False
finally:
teardown_example(example)
track_seen = Tracker()
falsifying_examples = []
if storage:
for example in storage.fetch():
track_seen.track(example)
if falsifies(example):
falsifying_examples = [example]
break
satisfying_examples = 0
timed_out = False
max_examples = self.max_examples
min_satisfying_examples = self.min_satisfying_examples
parameter_source = ParameterSource(context=build_context,
strategy=search_strategy,
min_parameters=max(
2,
int(float(max_examples) / 10)))
start_time = time.time()
def time_to_call_it_a_day():
"""Have we exceeded our timeout?"""
if self.timeout <= 0:
return False
return time.time() >= start_time + self.timeout
for parameter in islice(parameter_source,
max_examples - len(track_seen)):
if len(track_seen) >= search_strategy.size_upper_bound:
break
if falsifying_examples:
break
if time_to_call_it_a_day():
break
args = search_strategy.produce_template(build_context, parameter)
if track_seen.track(args) > 1:
parameter_source.mark_bad()
continue
try:
setup_example()
a = None
try:
a = search_strategy.reify(args)
is_falsifying_example = not hypothesis(*a)
finally:
teardown_example(a)
except UnsatisfiedAssumption:
parameter_source.mark_bad()
continue
satisfying_examples += 1
if is_falsifying_example:
falsifying_examples.append(args)
run_time = time.time() - start_time
timed_out = self.timeout >= 0 and run_time >= self.timeout
if not falsifying_examples:
if (satisfying_examples
and len(track_seen) >= search_strategy.size_lower_bound):
raise Exhausted(hypothesis, satisfying_examples)
elif satisfying_examples < min_satisfying_examples:
if timed_out:
raise Timeout(hypothesis, satisfying_examples, run_time)
else:
raise Unsatisfiable(hypothesis, satisfying_examples,
run_time)
else:
raise Unfalsifiable(hypothesis)
for example in falsifying_examples:
if not falsifies(example):
raise Flaky(hypothesis, example)
best_example = falsifying_examples[0]
for simpler in search_strategy.simplify_such_that(
random,
best_example,
falsifies,
tracker=track_seen,
):
best_example = simpler
if time_to_call_it_a_day():
# We no cover in here because it's a bit sensitive to timing
# and tends to make tests flaky. There are tests that mean
# this is definitely covered most of the time.
break # pragma: no cover
if storage is not None:
storage.save(best_example)
setup_example()
return search_strategy.reify(best_example)
def find_satisfying_template(search_strategy,
random,
condition,
tracker,
settings,
storage=None):
"""Attempt to find a template for search_strategy such that condition is
truthy.
Exceptions other than UnsatisfiedAssumption will be immediately propagated.
UnsatisfiedAssumption will indicate that similar examples should be avoided
in future.
Returns such a template as soon as it is found, otherwise stops after
settings.max_examples examples have been considered or settings.timeout
seconds have passed (if settings.timeout > 0).
May raise a variety of exceptions depending on exact circumstances, but
these will all subclass either Unsatisfiable (to indicate not enough
examples were found which did not raise UnsatisfiedAssumption to consider
this a valid test) or NoSuchExample (to indicate that this probably means
that condition is true with very high probability).
"""
satisfying_examples = 0
timed_out = False
max_examples = settings.max_examples
min_satisfying_examples = settings.min_satisfying_examples
start_time = time.time()
if storage:
for example in storage.fetch():
if time_to_call_it_a_day(settings, start_time):
break
tracker.track(example)
try:
if condition(example):
return example
satisfying_examples += 1
except UnsatisfiedAssumption:
pass
build_context = BuildContext(random)
parameter_source = ParameterSource(context=build_context,
strategy=search_strategy,
min_parameters=max(
2, int(float(max_examples) / 10)))
for parameter in islice(parameter_source, max_examples - len(tracker)):
if len(tracker) >= search_strategy.size_upper_bound:
break
if time_to_call_it_a_day(settings, start_time):
break
example = search_strategy.produce_template(build_context, parameter)
if tracker.track(example) > 1:
parameter_source.mark_bad()
continue
try:
if condition(example):
return example
except UnsatisfiedAssumption:
parameter_source.mark_bad()
continue
satisfying_examples += 1
run_time = time.time() - start_time
timed_out = settings.timeout >= 0 and run_time >= settings.timeout
if (satisfying_examples
and len(tracker) >= search_strategy.size_lower_bound):
raise DefinitelyNoSuchExample(
get_pretty_function_description(condition),
satisfying_examples,
)
elif satisfying_examples < min_satisfying_examples:
if timed_out:
raise Timeout(condition, satisfying_examples, run_time)
else:
raise Unsatisfiable(condition, satisfying_examples, run_time)
else:
raise NoSuchExample(get_pretty_function_description(condition))
def some_minimal_element(s):
strat = strategy(s)
template = strat.draw_and_produce(BuildContext(random))
for t in strat.simplify_such_that(template, lambda _: True):
template = t
return strat.reify(template)
def some_template(spec):
return strategy(spec).draw_and_produce(BuildContext(Random()))
def test_string_strategy_produces_strings():
strings = strategy(text_type)
result = strings.produce_template(BuildContext(random),
strings.produce_parameter(random))
assert result is not None
