def test_errors_if_you_mark_bad_before_fetching():
source = ParameterSource(
context=BuildContext(random.Random()),
strategy=strategy(int),
)
with pytest.raises(ValueError):
source.mark_bad()
def test_errors_if_you_mark_bad_before_fetching():
source = ParameterSource(
context=BuildContext(random.Random()),
strategy=strategy(int),
)
with pytest.raises(ValueError):
source.mark_bad()
def test_can_use_none_max_tries():
source = ParameterSource(
random=random.Random(),
strategy=integers(),
max_tries=None,
)
source.pick_a_parameter()
def test_errors_if_you_mark_bad_before_fetching():
source = ParameterSource(
random=random.Random(),
strategy=integers(),
)
with pytest.raises(ValueError):
source.mark_bad()
def test_errors_if_you_mark_bad_twice():
source = ParameterSource(
context=BuildContext(random.Random()),
strategy=strategy(int),
)
next(iter(source))
source.mark_bad()
with pytest.raises(ValueError):
source.mark_bad()
def test_tries_each_parameter_at_least_min_index_times():
source = ParameterSource(context=BuildContext(random.Random()),
strategy=strategy(int),
min_tries=5)
i = 0
for x in source.examples():
i += 1
if i > 500:
break
if i % 2:
source.mark_bad()
# The last index may not have been fully populated
assert all(c >= 5 for c in source.counts[:-1])
def test_negative_is_not_too_far_off_mean():
source = ParameterSource(
random=random.Random(),
strategy=integers(),
)
positive = 0
i = 0
for example in source.examples():
if example >= 0:
positive += 1
i += 1
if i >= N_EXAMPLES:
break
assert 0.3 <= float(positive) / N_EXAMPLES <= 0.7
def test_tries_each_parameter_at_least_min_index_times():
source = ParameterSource(
context=BuildContext(random.Random()),
strategy=strategy(int),
min_tries=5
)
i = 0
for x in source.examples():
i += 1
if i > 500:
break
if i % 2:
source.mark_bad()
# The last index may not have been fully populated
assert all(c >= 5 for c in source.counts[:-1])
def test_culls_valid_parameters_if_lots_are_bad():
source = ParameterSource(
context=BuildContext(random.Random()),
strategy=strategy(int),
min_tries=5
)
for _ in islice(source, 200):
source.mark_bad()
for i in hrange(len(source.parameters)):
assert source.counts[i] >= source.bad_counts[i]
for i in hrange(len(source.parameters)):
assert source.counts[i] == source.bad_counts[i]
assert len(source.valid_parameters) <= 1
def test_marking_negative_avoids_similar_examples():
source = ParameterSource(
context=BuildContext(random.Random()),
strategy=strategy(int),
)
positive = 0
i = 0
for example in source.examples():
if example >= 0:
positive += 1
else:
source.mark_bad()
i += 1
if i >= N_EXAMPLES:
break
assert float(positive) / N_EXAMPLES >= 0.8
def test_marking_negative_avoids_similar_examples():
source = ParameterSource(
context=BuildContext(random.Random()),
strategy=strategy(int),
)
positive = 0
i = 0
for example in source.examples():
if example >= 0:
positive += 1
else:
source.mark_bad()
i += 1
if i >= N_EXAMPLES:
break
assert float(positive) / N_EXAMPLES >= 0.8
def test_marking_negative_avoids_similar_examples():
positive = 0
k = 10
for _ in hrange(k):
source = ParameterSource(
random=random.Random(),
strategy=integers(),
)
n = N_EXAMPLES // k
for example in islice(source.examples(), n):
if example >= 0:
positive += 1
else:
source.mark_bad()
assert float(positive) / N_EXAMPLES >= 0.7
def test_can_grow_the_set_of_available_parameters_if_doing_badly():
runs = 10
number_grown = 0
for _ in hrange(runs):
source = ParameterSource(
context=BuildContext(random.Random()),
strategy=strategy(int),
min_parameters=1,
)
i = 0
for example in source.examples():
if example < 0:
source.mark_bad()
i += 1
if i >= 100:
break
if len(source.parameters) > 1:
number_grown += 1
assert len(source.parameters) < 100
def test_can_grow_the_set_of_available_parameters_if_doing_badly():
runs = 10
number_grown = 0
for _ in hrange(runs):
source = ParameterSource(
context=BuildContext(random.Random()),
strategy=strategy(int),
min_parameters=1,
)
i = 0
for example in source.examples():
if example < 0:
source.mark_bad()
i += 1
if i >= 100:
break
if len(source.parameters) > 1:
number_grown += 1
assert len(source.parameters) < 100
def test_eventually_culls_parameters_which_stop_being_valid():
source = ParameterSource(
context=BuildContext(random.Random()),
strategy=strategy(bool),
min_tries=5
)
seen = set()
for p in islice(source, 200):
if p in seen:
source.mark_bad()
else:
seen.add(p)
for i in hrange(len(source.parameters)):
assert source.counts[i] >= source.bad_counts[i]
for i in hrange(len(source.parameters)):
assert source.counts[i] == source.bad_counts[i] + 1
assert len(source.valid_parameters) <= len(source.parameters) // 2
def test_errors_if_you_mark_bad_twice():
source = ParameterSource(
random=random.Random(),
strategy=integers(),
)
next(iter(source))
source.mark_bad()
with pytest.raises(ValueError):
source.mark_bad()
def test_errors_if_you_mark_bad_twice():
source = ParameterSource(
context=BuildContext(random.Random()),
strategy=strategy(int),
)
next(iter(source))
source.mark_bad()
with pytest.raises(ValueError):
source.mark_bad()
def find_satisfying_template(
search_strategy, random, condition, tracker, settings, storage=None,
max_parameter_tries=None,
):
"""Attempt to find a template for search_strategy such that condition is
truthy.
Exceptions other than UnsatisfiedAssumption will be immediately propagated.
UnsatisfiedAssumption will indicate that similar examples should be avoided
in future.
Returns such a template as soon as it is found, otherwise stops after
settings.max_examples examples have been considered or settings.timeout
seconds have passed (if settings.timeout > 0).
May raise a variety of exceptions depending on exact circumstances, but
these will all subclass either Unsatisfiable (to indicate not enough
examples were found which did not raise UnsatisfiedAssumption to consider
this a valid test) or NoSuchExample (to indicate that this probably means
that condition is true with very high probability).
"""
satisfying_examples = 0
examples_considered = 0
timed_out = False
max_iterations = max(settings.max_iterations, settings.max_examples)
max_examples = min(max_iterations, settings.max_examples)
min_satisfying_examples = min(
settings.min_satisfying_examples,
max_examples,
)
start_time = time.time()
if storage:
for example in storage.fetch(search_strategy):
if examples_considered >= max_iterations:
break
examples_considered += 1
if time_to_call_it_a_day(settings, start_time):
break
tracker.track(example)
try:
if condition(example):
return example
satisfying_examples += 1
except UnsatisfiedAssumption:
pass
if satisfying_examples >= max_examples:
break
parameter_source = ParameterSource(
random=random, strategy=search_strategy,
max_tries=max_parameter_tries,
)
assert search_strategy.template_upper_bound >= 0
if isinstance(search_strategy.template_upper_bound, float):
assert math.isinf(search_strategy.template_upper_bound)
else:
assert isinstance(search_strategy.template_upper_bound, int)
for parameter in parameter_source: # pragma: no branch
if len(tracker) >= search_strategy.template_upper_bound:
break
if examples_considered >= max_iterations:
break
if satisfying_examples >= max_examples:
break
if time_to_call_it_a_day(settings, start_time):
break
examples_considered += 1
try:
example = search_strategy.draw_template(
random, parameter
)
except BadTemplateDraw:
debug_report(u'Failed attempt to draw a template')
parameter_source.mark_bad()
continue
if tracker.track(example) > 1:
debug_report(u'Skipping duplicate example')
parameter_source.mark_bad()
continue
try:
if condition(example):
return example
except UnsatisfiedAssumption:
parameter_source.mark_bad()
continue
satisfying_examples += 1
run_time = time.time() - start_time
timed_out = settings.timeout >= 0 and run_time >= settings.timeout
if (
satisfying_examples and
len(tracker) >= search_strategy.template_upper_bound
):
raise DefinitelyNoSuchExample(
get_pretty_function_description(condition),
satisfying_examples,
)
elif satisfying_examples < min_satisfying_examples:
if timed_out:
raise Timeout((
u'Ran out of time before finding a satisfying example for '
u'%s. Only found %d examples (%d satisfying assumptions) in ' +
u'%.2fs.'
) % (
get_pretty_function_description(condition),
len(tracker), satisfying_examples, run_time
))
else:
raise Unsatisfiable((
u'Unable to satisfy assumptions of hypothesis %s. ' +
u'Only %d out of %d examples considered satisfied assumptions'
) % (
get_pretty_function_description(condition),
satisfying_examples, len(tracker)))
else:
raise NoSuchExample(get_pretty_function_description(condition))
def find_satisfying_template(search_strategy,
random,
condition,
tracker,
settings,
storage=None):
"""Attempt to find a template for search_strategy such that condition is
truthy.
Exceptions other than UnsatisfiedAssumption will be immediately propagated.
UnsatisfiedAssumption will indicate that similar examples should be avoided
in future.
Returns such a template as soon as it is found, otherwise stops after
settings.max_examples examples have been considered or settings.timeout
seconds have passed (if settings.timeout > 0).
May raise a variety of exceptions depending on exact circumstances, but
these will all subclass either Unsatisfiable (to indicate not enough
examples were found which did not raise UnsatisfiedAssumption to consider
this a valid test) or NoSuchExample (to indicate that this probably means
that condition is true with very high probability).
"""
satisfying_examples = 0
timed_out = False
max_examples = settings.max_examples
min_satisfying_examples = settings.min_satisfying_examples
start_time = time.time()
if storage:
for example in storage.fetch():
if time_to_call_it_a_day(settings, start_time):
break
tracker.track(example)
try:
if condition(example):
return example
satisfying_examples += 1
except UnsatisfiedAssumption:
pass
build_context = BuildContext(random)
parameter_source = ParameterSource(context=build_context,
strategy=search_strategy,
min_parameters=max(
2, int(float(max_examples) / 10)))
for parameter in islice(parameter_source, max_examples - len(tracker)):
if len(tracker) >= search_strategy.size_upper_bound:
break
if time_to_call_it_a_day(settings, start_time):
break
example = search_strategy.produce_template(build_context, parameter)
if tracker.track(example) > 1:
parameter_source.mark_bad()
continue
try:
if condition(example):
return example
except UnsatisfiedAssumption:
parameter_source.mark_bad()
continue
satisfying_examples += 1
run_time = time.time() - start_time
timed_out = settings.timeout >= 0 and run_time >= settings.timeout
if (satisfying_examples
and len(tracker) >= search_strategy.size_lower_bound):
raise DefinitelyNoSuchExample(
get_pretty_function_description(condition),
satisfying_examples,
)
elif satisfying_examples < min_satisfying_examples:
if timed_out:
raise Timeout(condition, satisfying_examples, run_time)
else:
raise Unsatisfiable(condition, satisfying_examples, run_time)
else:
raise NoSuchExample(get_pretty_function_description(condition))
def find_satisfying_template(
search_strategy, random, condition, tracker, settings, storage=None
):
"""Attempt to find a template for search_strategy such that condition is
truthy.
Exceptions other than UnsatisfiedAssumption will be immediately propagated.
UnsatisfiedAssumption will indicate that similar examples should be avoided
in future.
Returns such a template as soon as it is found, otherwise stops after
settings.max_examples examples have been considered or settings.timeout
seconds have passed (if settings.timeout > 0).
May raise a variety of exceptions depending on exact circumstances, but
these will all subclass either Unsatisfiable (to indicate not enough
examples were found which did not raise UnsatisfiedAssumption to consider
this a valid test) or NoSuchExample (to indicate that this probably means
that condition is true with very high probability).
"""
satisfying_examples = 0
timed_out = False
max_examples = settings.max_examples
min_satisfying_examples = settings.min_satisfying_examples
start_time = time.time()
if storage:
for example in storage.fetch():
if time_to_call_it_a_day(settings, start_time):
break
tracker.track(example)
try:
if condition(example):
return example
satisfying_examples += 1
except UnsatisfiedAssumption:
pass
build_context = BuildContext(random)
parameter_source = ParameterSource(
context=build_context, strategy=search_strategy,
min_parameters=max(2, int(float(max_examples) / 10))
)
for parameter in islice(
parameter_source, max_examples - len(tracker)
):
if len(tracker) >= search_strategy.size_upper_bound:
break
if time_to_call_it_a_day(settings, start_time):
break
example = search_strategy.produce_template(
build_context, parameter
)
if tracker.track(example) > 1:
parameter_source.mark_bad()
continue
try:
if condition(example):
return example
except UnsatisfiedAssumption:
parameter_source.mark_bad()
continue
satisfying_examples += 1
run_time = time.time() - start_time
timed_out = settings.timeout >= 0 and run_time >= settings.timeout
if (
satisfying_examples and
len(tracker) >= search_strategy.size_lower_bound
):
raise DefinitelyNoSuchExample(
get_pretty_function_description(condition),
satisfying_examples,
)
elif satisfying_examples < min_satisfying_examples:
if timed_out:
raise Timeout(condition, satisfying_examples, run_time)
else:
raise Unsatisfiable(
condition, satisfying_examples, run_time)
else:
raise NoSuchExample(get_pretty_function_description(condition))
def falsify(self, hypothesis, *argument_types, **kwargs): # pylint: disable=too-many-locals,too-many-branches
"""
Attempt to construct an example tuple x matching argument_types such
that hypothesis(*x) returns a falsey value
"""
teardown_example = kwargs.get('teardown_example') or (lambda x: None)
setup_example = kwargs.get('setup_example') or (lambda: None)
random = self.random
if random is None:
random = Random(function_digest(hypothesis))
build_context = BuildContext(random)
search_strategy = strategy(argument_types, self.settings)
storage = None
if self.database is not None:
storage = self.database.storage_for(argument_types)
def falsifies(args): # pylint: disable=missing-docstring
example = None
try:
try:
setup_example()
example = search_strategy.reify(args)
return not hypothesis(*example)
except UnsatisfiedAssumption:
return False
finally:
teardown_example(example)
track_seen = Tracker()
falsifying_examples = []
if storage:
for example in storage.fetch():
track_seen.track(example)
if falsifies(example):
falsifying_examples = [example]
break
satisfying_examples = 0
timed_out = False
max_examples = self.max_examples
min_satisfying_examples = self.min_satisfying_examples
parameter_source = ParameterSource(context=build_context,
strategy=search_strategy,
min_parameters=max(
2,
int(float(max_examples) / 10)))
start_time = time.time()
def time_to_call_it_a_day():
"""Have we exceeded our timeout?"""
if self.timeout <= 0:
return False
return time.time() >= start_time + self.timeout
for parameter in islice(parameter_source,
max_examples - len(track_seen)):
if len(track_seen) >= search_strategy.size_upper_bound:
break
if falsifying_examples:
break
if time_to_call_it_a_day():
break
args = search_strategy.produce_template(build_context, parameter)
if track_seen.track(args) > 1:
parameter_source.mark_bad()
continue
try:
setup_example()
a = None
try:
a = search_strategy.reify(args)
is_falsifying_example = not hypothesis(*a)
finally:
teardown_example(a)
except UnsatisfiedAssumption:
parameter_source.mark_bad()
continue
satisfying_examples += 1
if is_falsifying_example:
falsifying_examples.append(args)
run_time = time.time() - start_time
timed_out = self.timeout >= 0 and run_time >= self.timeout
if not falsifying_examples:
if (satisfying_examples
and len(track_seen) >= search_strategy.size_lower_bound):
raise Exhausted(hypothesis, satisfying_examples)
elif satisfying_examples < min_satisfying_examples:
if timed_out:
raise Timeout(hypothesis, satisfying_examples, run_time)
else:
raise Unsatisfiable(hypothesis, satisfying_examples,
run_time)
else:
raise Unfalsifiable(hypothesis)
for example in falsifying_examples:
if not falsifies(example):
raise Flaky(hypothesis, example)
best_example = falsifying_examples[0]
for simpler in search_strategy.simplify_such_that(
random,
best_example,
falsifies,
tracker=track_seen,
):
best_example = simpler
if time_to_call_it_a_day():
# We no cover in here because it's a bit sensitive to timing
# and tends to make tests flaky. There are tests that mean
# this is definitely covered most of the time.
break # pragma: no cover
if storage is not None:
storage.save(best_example)
setup_example()
return search_strategy.reify(best_example)
def find_satisfying_template(
search_strategy,
random,
condition,
tracker,
settings,
storage=None,
max_parameter_tries=None,
):
"""Attempt to find a template for search_strategy such that condition is
truthy.
Exceptions other than UnsatisfiedAssumption will be immediately propagated.
UnsatisfiedAssumption will indicate that similar examples should be avoided
in future.
Returns such a template as soon as it is found, otherwise stops after
settings.max_examples examples have been considered or settings.timeout
seconds have passed (if settings.timeout > 0).
May raise a variety of exceptions depending on exact circumstances, but
these will all subclass either Unsatisfiable (to indicate not enough
examples were found which did not raise UnsatisfiedAssumption to consider
this a valid test) or NoSuchExample (to indicate that this probably means
that condition is true with very high probability).
"""
satisfying_examples = 0
examples_considered = 0
timed_out = False
max_iterations = max(settings.max_iterations, settings.max_examples)
max_examples = min(max_iterations, settings.max_examples)
min_satisfying_examples = min(
settings.min_satisfying_examples,
max_examples,
)
start_time = time.time()
if storage:
for example in storage.fetch(search_strategy):
if examples_considered >= max_iterations:
break
examples_considered += 1
if time_to_call_it_a_day(settings, start_time):
break
tracker.track(example)
try:
if condition(example):
return example
satisfying_examples += 1
except UnsatisfiedAssumption:
pass
if satisfying_examples >= max_examples:
break
parameter_source = ParameterSource(
random=random,
strategy=search_strategy,
max_tries=max_parameter_tries,
)
assert search_strategy.template_upper_bound >= 0
if isinstance(search_strategy.template_upper_bound, float):
assert math.isinf(search_strategy.template_upper_bound)
else:
assert isinstance(search_strategy.template_upper_bound, int)
for parameter in parameter_source: # pragma: no branch
if len(tracker) >= search_strategy.template_upper_bound:
break
if examples_considered >= max_iterations:
break
if satisfying_examples >= max_examples:
break
if time_to_call_it_a_day(settings, start_time):
break
examples_considered += 1
try:
example = search_strategy.draw_template(random, parameter)
except BadTemplateDraw:
debug_report('Failed attempt to draw a template')
parameter_source.mark_bad()
continue
if tracker.track(example) > 1:
debug_report('Skipping duplicate example')
parameter_source.mark_bad()
continue
try:
if condition(example):
return example
except UnsatisfiedAssumption:
parameter_source.mark_bad()
continue
satisfying_examples += 1
run_time = time.time() - start_time
timed_out = settings.timeout >= 0 and run_time >= settings.timeout
if (satisfying_examples
and len(tracker) >= search_strategy.template_upper_bound):
raise DefinitelyNoSuchExample(
get_pretty_function_description(condition),
satisfying_examples,
)
elif satisfying_examples < min_satisfying_examples:
if timed_out:
raise Timeout(
('Ran out of time before finding a satisfying example for '
'%s. Only found %d examples (%d satisfying assumptions) in ' +
'%.2fs.') % (get_pretty_function_description(condition),
len(tracker), satisfying_examples, run_time))
else:
raise Unsatisfiable(
('Unable to satisfy assumptions of hypothesis %s. ' +
'Only %d out of %d examples considered satisfied assumptions')
% (get_pretty_function_description(condition),
satisfying_examples, len(tracker)))
else:
raise NoSuchExample(get_pretty_function_description(condition))
def falsify(
self, hypothesis,
*argument_types,
**kwargs
): # pylint: disable=too-many-locals,too-many-branches
"""
Attempt to construct an example tuple x matching argument_types such
that hypothesis(*x) returns a falsey value
"""
teardown_example = kwargs.get('teardown_example') or (lambda x: None)
setup_example = kwargs.get('setup_example') or (lambda: None)
random = self.random
if random is None:
random = Random(
function_digest(hypothesis)
)
build_context = BuildContext(random)
search_strategy = strategy(argument_types, self.settings)
storage = None
if self.database is not None:
storage = self.database.storage_for(argument_types)
def falsifies(args): # pylint: disable=missing-docstring
example = None
try:
try:
setup_example()
example = search_strategy.reify(args)
return not hypothesis(*example)
except UnsatisfiedAssumption:
return False
finally:
teardown_example(example)
track_seen = Tracker()
falsifying_examples = []
if storage:
for example in storage.fetch():
track_seen.track(example)
if falsifies(example):
falsifying_examples = [example]
break
satisfying_examples = 0
timed_out = False
max_examples = self.max_examples
min_satisfying_examples = self.min_satisfying_examples
parameter_source = ParameterSource(
context=build_context, strategy=search_strategy,
min_parameters=max(2, int(float(max_examples) / 10))
)
start_time = time.time()
def time_to_call_it_a_day():
"""Have we exceeded our timeout?"""
if self.timeout <= 0:
return False
return time.time() >= start_time + self.timeout
for parameter in islice(
parameter_source, max_examples - len(track_seen)
):
if len(track_seen) >= search_strategy.size_upper_bound:
break
if falsifying_examples:
break
if time_to_call_it_a_day():
break
args = search_strategy.produce_template(
build_context, parameter
)
if track_seen.track(args) > 1:
parameter_source.mark_bad()
continue
try:
setup_example()
a = None
try:
a = search_strategy.reify(args)
is_falsifying_example = not hypothesis(*a)
finally:
teardown_example(a)
except UnsatisfiedAssumption:
parameter_source.mark_bad()
continue
satisfying_examples += 1
if is_falsifying_example:
falsifying_examples.append(args)
run_time = time.time() - start_time
timed_out = self.timeout >= 0 and run_time >= self.timeout
if not falsifying_examples:
if (
satisfying_examples and
len(track_seen) >= search_strategy.size_lower_bound
):
raise Exhausted(
hypothesis, satisfying_examples)
elif satisfying_examples < min_satisfying_examples:
if timed_out:
raise Timeout(hypothesis, satisfying_examples, run_time)
else:
raise Unsatisfiable(
hypothesis, satisfying_examples, run_time)
else:
raise Unfalsifiable(hypothesis)
for example in falsifying_examples:
if not falsifies(example):
raise Flaky(hypothesis, example)
best_example = falsifying_examples[0]
for simpler in search_strategy.simplify_such_that(
random,
best_example, falsifies,
tracker=track_seen,
):
best_example = simpler
if time_to_call_it_a_day():
# We no cover in here because it's a bit sensitive to timing
# and tends to make tests flaky. There are tests that mean
# this is definitely covered most of the time.
break # pragma: no cover
if storage is not None:
storage.save(best_example)
setup_example()
return search_strategy.reify(best_example)
