def simplify_such_that(self, t, f):
"""Perform a greedy search to produce a "simplest" version of t that
satisfies the predicate s. As each simpler version is found, yield it
in turn. Stops when it has a value such that no value in simplify on
the last value found satisfies f.
Care is taken to avoid cycles in simplify.
f should produce the same result deterministically. This function may
raise an error given f such that f(t) returns False sometimes and True
some other times.
"""
assert self.could_have_produced(t)
if not f(t):
raise ValueError('%r does not satisfy predicate %s' % (t, f))
tracker = Tracker()
yield t
while True:
simpler = self.simplify(t)
for s in simpler:
assert self.could_have_produced(s)
if tracker.track(s) > 1:
continue
if f(s):
yield s
t = s
break
else:
break
def simplify_such_that(self, random, t, f, tracker=None):
"""Perform a greedy search to produce a "simplest" version of a
template that satisfies some predicate.
Care is taken to avoid cycles in simplify.
f should produce the same result deterministically. This function may
raise an error given f such that f(t) returns False sometimes and True
some other times.
"""
assert isinstance(random, Random)
if tracker is None:
tracker = Tracker()
yield t
changed = True
while changed:
changed = False
for simplify in self.simplifiers(t):
while True:
simpler = simplify(random, t)
for s in simpler:
if tracker.track(s) > 1:
continue
if f(s):
changed = True
yield s
t = s
break
else:
break
def simplify_such_that(self, t, f):
"""Perform a greedy search to produce a "simplest" version of a
template that satisfies some predicate.
Care is taken to avoid cycles in simplify.
f should produce the same result deterministically. This function may
raise an error given f such that f(t) returns False sometimes and True
some other times.
"""
if not f(t):
raise ValueError(
'%r does not satisfy predicate %s' % (t, f))
tracker = Tracker()
yield t
while True:
simpler = self.simplify(t)
for s in simpler:
if tracker.track(s) > 1:
continue
if f(s):
yield s
t = s
break
else:
break
def simplify_such_that(self, random, t, f, tracker=None):
"""Perform a greedy search to produce a "simplest" version of a
template that satisfies some predicate.
Care is taken to avoid cycles in simplify.
f should produce the same result deterministically. This function may
raise an error given f such that f(t) returns False sometimes and True
some other times.
"""
assert isinstance(random, Random)
if tracker is None:
tracker = Tracker()
yield t
changed = True
while changed:
changed = False
for simplify in self.simplifiers(t):
while True:
simpler = simplify(random, t)
for s in simpler:
if tracker.track(s) > 1:
continue
if f(s):
changed = True
yield s
t = s
break
else:
break
def simplify(self, x):
t = Tracker()
for cs in self.element_strategies:
if cs.could_have_produced(x):
for y in cs.simplify(x):
if t.track(y) == 1:
yield y
def simplify_such_that(self, t, f):
"""Perform a greedy search to produce a "simplest" version of a
template that satisfies some predicate.
Care is taken to avoid cycles in simplify.
f should produce the same result deterministically. This function may
raise an error given f such that f(t) returns False sometimes and True
some other times.
"""
if not f(t):
raise ValueError('%r does not satisfy predicate %s' % (t, f))
tracker = Tracker()
yield t
while True:
simpler = self.simplify(t)
for s in simpler:
if tracker.track(s) > 1:
continue
if f(s):
yield s
t = s
break
else:
break
def test_track_complex_with_nan():
t = Tracker()
nan = float('nan')
assert t.track(complex(nan, 2)) == 1
assert t.track(complex(nan, 2)) == 2
assert t.track(complex(0, nan)) == 1
assert t.track(complex(0, nan)) == 2
assert t.track(complex(nan, nan)) == 1
assert t.track(complex(nan, nan)) == 2
def test_track_complex_with_nan():
t = Tracker()
nan = float(u'nan')
assert t.track(complex(nan, 2)) == 1
assert t.track(complex(nan, 2)) == 2
assert t.track(complex(0, nan)) == 1
assert t.track(complex(0, nan)) == 2
assert t.track(complex(nan, nan)) == 1
assert t.track(complex(nan, nan)) == 2
def test_can_minimize_to_empty(self, template, rnd):
simplest = template
tracker = Tracker()
while True:
for t in strat.full_simplify(rnd, simplest):
if tracker.track(t) == 1:
simplest = t
break
else:
break
assert list(strat.full_simplify(rnd, simplest)) == []
def test_can_minimize_to_empty(self, template, rnd):
simplest = template
tracker = Tracker()
while True:
for t in strat.full_simplify(rnd, simplest):
if tracker.track(t) == 1:
simplest = t
break
else:
break
assert list(strat.full_simplify(rnd, simplest)) == []
def test_can_track_morphers():
t = Tracker()
assert t.track(Morpher(0, 0)) == 1
assert t.track(Morpher(0, 0)) == 2
m1 = Morpher(0, 1)
m2 = Morpher(0, 1)
m1.become(s.lists(s.integers()))
m2.become(s.lists(s.integers()))
assert t.track(m1) == 1
assert t.track(m2) == 2
def test_can_track_morphers():
t = Tracker()
assert t.track(Morpher(0, 0)) == 1
assert t.track(Morpher(0, 0)) == 2
m1 = Morpher(0, 1)
m2 = Morpher(0, 1)
m1.become(s.lists(s.integers()))
m2.become(s.lists(s.integers()))
assert t.track(m1) == 1
assert t.track(m2) == 2
def find(specifier, condition, settings=None, random=None, storage=None):
settings = settings or Settings(
max_examples=2000,
min_satisfying_examples=0,
max_shrinks=2000,
)
search = strategy(specifier, settings)
if storage is None and settings.database is not None:
storage = settings.database.storage(
u'find(%s)' %
(binascii.hexlify(function_digest(condition)).decode(u'ascii'), ))
random = random or Random()
successful_examples = [0]
def template_condition(template):
with BuildContext():
result = search.reify(template)
success = condition(result)
if success:
successful_examples[0] += 1
if not successful_examples[0]:
verbose_report(lambda: u'Trying example %s' % (repr(result), ))
elif success:
if successful_examples[0] == 1:
verbose_report(lambda: u'Found satisfying example %s' %
(repr(result), ))
else:
verbose_report(lambda: u'Shrunk example to %s' %
(repr(result), ))
return success
template_condition.__name__ = condition.__name__
tracker = Tracker()
try:
template = best_satisfying_template(
search,
random,
template_condition,
settings,
tracker=tracker,
max_parameter_tries=2,
storage=storage,
)
with BuildContext():
return search.reify(template)
except Timeout:
raise
except NoSuchExample:
if search.template_upper_bound <= len(tracker):
raise DefinitelyNoSuchExample(
get_pretty_function_description(condition),
search.template_upper_bound,
)
raise NoSuchExample(get_pretty_function_description(condition))
def minimal_element(strategy, random):
tracker = Tracker()
element = some_template(strategy, random)
while True:
for new_element in strategy.full_simplify(random, element):
if tracker.track(new_element) > 1:
continue
try:
strategy.reify(new_element)
element = new_element
break
except UnsatisfiedAssumption:
pass
else:
break
return element
def best_satisfying_template(search_strategy,
random,
condition,
settings,
storage,
tracker=None):
"""Find and then minimize a satisfying template.
First look in storage if it is not None, then attempt to generate
one. May throw all the exceptions of find_satisfying_template. Once
an example has been found it will be further minimized.
"""
if tracker is None:
tracker = Tracker()
start_time = time.time()
satisfying_example = find_satisfying_template(search_strategy, random,
condition, tracker, settings,
storage)
for simpler in simplify_template_such_that(search_strategy, random,
satisfying_example, condition,
tracker):
satisfying_example = simpler
if time_to_call_it_a_day(settings, start_time):
# It's very hard to reliably hit this line even though we have
# tests for it. No cover prevents this from causing a flaky build.
break # pragma: no cover
if storage is not None:
storage.save(satisfying_example)
return satisfying_example
def minimal_element(strategy, random):
tracker = Tracker()
element = some_template(strategy, random)
while True:
for new_element in strategy.full_simplify(random, element):
if tracker.track(new_element) > 1:
continue
try:
strategy.reify(new_element)
element = new_element
break
except UnsatisfiedAssumption:
pass
else:
break
return element
def find(specifier, condition, settings=None, random=None):
settings = settings or Settings(
max_examples=2000,
min_satisfying_examples=0,
)
search = strategy(specifier, settings)
random = random or Random()
def template_condition(template):
return assume(condition(search.reify(template)))
template_condition.__name__ = condition.__name__
tracker = Tracker()
try:
return search.reify(
best_satisfying_template(
search,
random,
template_condition,
settings,
None,
tracker=tracker,
))
except NoSuchExample:
if search.size_upper_bound <= len(tracker):
raise DefinitelyNoSuchExample(
get_pretty_function_description(condition),
search.size_upper_bound,
)
raise NoSuchExample(get_pretty_function_description(condition))
def best_satisfying_template(
search_strategy,
random,
condition,
settings,
storage,
tracker=None,
max_parameter_tries=None,
start_time=None,
):
"""Find and then minimize a satisfying template.
First look in storage if it is not None, then attempt to generate
one. May throw all the exceptions of find_satisfying_template. Once
an example has been found it will be further minimized.
"""
if tracker is None:
tracker = Tracker()
if start_time is None:
start_time = time.time()
successful_shrinks = -1
with settings:
satisfying_example = find_satisfying_template(
search_strategy,
random,
condition,
tracker,
settings,
storage,
max_parameter_tries=max_parameter_tries,
)
for simpler in simplify_template_such_that(
search_strategy,
random,
satisfying_example,
condition,
tracker,
settings,
start_time,
):
successful_shrinks += 1
satisfying_example = simpler
if storage is not None:
storage.save(satisfying_example, search_strategy)
if not successful_shrinks:
verbose_report('Could not shrink example')
elif successful_shrinks == 1:
verbose_report('Successfully shrunk example once')
else:
verbose_report('Successfully shrunk example %d times' %
(successful_shrinks, ))
return satisfying_example
def test_stops_after_max_examples_if_satisfying():
tracker = Tracker()
def track(x):
tracker.track(x)
return False
max_examples = 100
with pytest.raises(NoSuchExample):
find(
s.integers(0, 10000),
track, settings=Settings(max_examples=max_examples))
assert len(tracker) == max_examples
def find(specifier, condition, settings=None, random=None):
settings = settings or Settings(
max_examples=2000,
min_satisfying_examples=0,
)
search = strategy(specifier, settings)
random = random or Random()
successful_examples = [0]
def template_condition(template):
result = search.reify(template)
success = condition(result)
if success:
successful_examples[0] += 1
if not successful_examples[0]:
verbose_report(lambda: 'Trying example %s' % (show(result), ))
elif success:
if successful_examples[0] == 1:
verbose_report(lambda: 'Found satisfying example %s' %
(show(result), ))
else:
verbose_report(lambda: 'Shrunk example to %s' %
(show(result), ))
return assume(success)
template_condition.__name__ = condition.__name__
tracker = Tracker()
try:
return search.reify(
best_satisfying_template(
search,
random,
template_condition,
settings,
None,
tracker=tracker,
))
except NoSuchExample:
if search.size_upper_bound <= len(tracker):
raise DefinitelyNoSuchExample(
get_pretty_function_description(condition),
search.size_upper_bound,
)
raise NoSuchExample(get_pretty_function_description(condition))
def test_stops_after_max_iterations_if_not_satisfying():
tracker = Tracker()
def track(x):
tracker.track(x)
assume(False)
max_examples = 100
max_iterations = 200
with pytest.raises(Unsatisfiable):
find(
s.integers(0, 10000),
track, settings=Settings(
max_examples=max_examples, max_iterations=max_iterations))
# May be less because of duplication
assert len(tracker) <= max_iterations
def save(self, value):
if not self.strategy.could_have_produced(value):
raise ValueError('Argument %r does not match description %s' %
(value, self.key))
tracker = Tracker()
def do_save(d, v):
if tracker.track((d, v)) > 1:
return
try:
s = self.database.storage_for(d)
except NotSerializeable:
return
converted = s.converter.to_basic(v)
serialized = s.format.serialize_basic(converted)
s.backend.save(s.key, serialized)
for d2, v2 in s.strategy.decompose(v):
do_save(d2, v2)
do_save(self.descriptor, value)
def test_track_lists():
t = Tracker()
assert t.track([1]) == 1
assert t.track([1]) == 2
def test_track_ints():
t = Tracker()
assert t.track(1) == 1
assert t.track(1) == 2
def test_nested_unhashables():
t = Tracker()
x = {'foo': [1, 2, set((3, 4, 5, 6))], 'bar': 10}
assert t.track(x) == 1
assert t.track(x) == 2
def assert_no_duplicates_in_simplify(s, x):
s = strategy(s)
t = Tracker()
t.track(x)
for y in s.simplify(x):
assert t.track(y) == 1
def assert_no_duplicates_in_simplify(s, x):
s = strategy(s)
t = Tracker()
t.track(x)
for y in s.simplify(x):
assert t.track(y) == 1
def test_nested_unhashables():
t = Tracker()
x = {u'foo': [1, 2, set((3, 4, 5, 6))], u'bar': 10}
assert t.track(x) == 1
assert t.track(x) == 2
def falsify(
self, hypothesis,
*argument_types,
**kwargs
): # pylint: disable=too-many-locals,too-many-branches
"""
Attempt to construct an example tuple x matching argument_types such
that hypothesis(*x) returns a falsey value
"""
teardown_example = kwargs.get('teardown_example') or (lambda x: None)
setup_example = kwargs.get('setup_example') or (lambda: None)
random = self.random
if random is None:
random = Random(
function_digest(hypothesis)
)
build_context = BuildContext(random)
search_strategy = strategy(argument_types, self.settings)
storage = None
if self.database is not None:
storage = self.database.storage_for(argument_types)
def falsifies(args): # pylint: disable=missing-docstring
example = None
try:
try:
setup_example()
example = search_strategy.reify(args)
return not hypothesis(*example)
except UnsatisfiedAssumption:
return False
finally:
teardown_example(example)
track_seen = Tracker()
falsifying_examples = []
if storage:
for example in storage.fetch():
track_seen.track(example)
if falsifies(example):
falsifying_examples = [example]
break
satisfying_examples = 0
timed_out = False
max_examples = self.max_examples
min_satisfying_examples = self.min_satisfying_examples
parameter_source = ParameterSource(
context=build_context, strategy=search_strategy,
min_parameters=max(2, int(float(max_examples) / 10))
)
start_time = time.time()
def time_to_call_it_a_day():
"""Have we exceeded our timeout?"""
if self.timeout <= 0:
return False
return time.time() >= start_time + self.timeout
for parameter in islice(
parameter_source, max_examples - len(track_seen)
):
if len(track_seen) >= search_strategy.size_upper_bound:
break
if falsifying_examples:
break
if time_to_call_it_a_day():
break
args = search_strategy.produce_template(
build_context, parameter
)
if track_seen.track(args) > 1:
parameter_source.mark_bad()
continue
try:
setup_example()
a = None
try:
a = search_strategy.reify(args)
is_falsifying_example = not hypothesis(*a)
finally:
teardown_example(a)
except UnsatisfiedAssumption:
parameter_source.mark_bad()
continue
satisfying_examples += 1
if is_falsifying_example:
falsifying_examples.append(args)
run_time = time.time() - start_time
timed_out = self.timeout >= 0 and run_time >= self.timeout
if not falsifying_examples:
if (
satisfying_examples and
len(track_seen) >= search_strategy.size_lower_bound
):
raise Exhausted(
hypothesis, satisfying_examples)
elif satisfying_examples < min_satisfying_examples:
if timed_out:
raise Timeout(hypothesis, satisfying_examples, run_time)
else:
raise Unsatisfiable(
hypothesis, satisfying_examples, run_time)
else:
raise Unfalsifiable(hypothesis)
for example in falsifying_examples:
if not falsifies(example):
raise Flaky(hypothesis, example)
best_example = falsifying_examples[0]
for simpler in search_strategy.simplify_such_that(
random,
best_example, falsifies,
tracker=track_seen,
):
best_example = simpler
if time_to_call_it_a_day():
# We no cover in here because it's a bit sensitive to timing
# and tends to make tests flaky. There are tests that mean
# this is definitely covered most of the time.
break # pragma: no cover
if storage is not None:
storage.save(best_example)
setup_example()
return search_strategy.reify(best_example)
def test_track_ints():
t = Tracker()
assert t.track(1) == 1
assert t.track(1) == 2
def test_track_lists():
t = Tracker()
assert t.track([1]) == 1
assert t.track([1]) == 2
def falsify(self, hypothesis, *argument_types, **kwargs): # pylint: disable=too-many-locals,too-many-branches
"""
Attempt to construct an example tuple x matching argument_types such
that hypothesis(*x) returns a falsey value
"""
teardown_example = kwargs.get('teardown_example') or (lambda x: None)
setup_example = kwargs.get('setup_example') or (lambda: None)
random = self.random
if random is None:
random = Random(function_digest(hypothesis))
build_context = BuildContext(random)
search_strategy = strategy(argument_types, self.settings)
storage = None
if self.database is not None:
storage = self.database.storage_for(argument_types)
def falsifies(args): # pylint: disable=missing-docstring
example = None
try:
try:
setup_example()
example = search_strategy.reify(args)
return not hypothesis(*example)
except UnsatisfiedAssumption:
return False
finally:
teardown_example(example)
track_seen = Tracker()
falsifying_examples = []
if storage:
for example in storage.fetch():
track_seen.track(example)
if falsifies(example):
falsifying_examples = [example]
break
satisfying_examples = 0
timed_out = False
max_examples = self.max_examples
min_satisfying_examples = self.min_satisfying_examples
parameter_source = ParameterSource(context=build_context,
strategy=search_strategy,
min_parameters=max(
2,
int(float(max_examples) / 10)))
start_time = time.time()
def time_to_call_it_a_day():
"""Have we exceeded our timeout?"""
if self.timeout <= 0:
return False
return time.time() >= start_time + self.timeout
for parameter in islice(parameter_source,
max_examples - len(track_seen)):
if len(track_seen) >= search_strategy.size_upper_bound:
break
if falsifying_examples:
break
if time_to_call_it_a_day():
break
args = search_strategy.produce_template(build_context, parameter)
if track_seen.track(args) > 1:
parameter_source.mark_bad()
continue
try:
setup_example()
a = None
try:
a = search_strategy.reify(args)
is_falsifying_example = not hypothesis(*a)
finally:
teardown_example(a)
except UnsatisfiedAssumption:
parameter_source.mark_bad()
continue
satisfying_examples += 1
if is_falsifying_example:
falsifying_examples.append(args)
run_time = time.time() - start_time
timed_out = self.timeout >= 0 and run_time >= self.timeout
if not falsifying_examples:
if (satisfying_examples
and len(track_seen) >= search_strategy.size_lower_bound):
raise Exhausted(hypothesis, satisfying_examples)
elif satisfying_examples < min_satisfying_examples:
if timed_out:
raise Timeout(hypothesis, satisfying_examples, run_time)
else:
raise Unsatisfiable(hypothesis, satisfying_examples,
run_time)
else:
raise Unfalsifiable(hypothesis)
for example in falsifying_examples:
if not falsifies(example):
raise Flaky(hypothesis, example)
best_example = falsifying_examples[0]
for simpler in search_strategy.simplify_such_that(
random,
best_example,
falsifies,
tracker=track_seen,
):
best_example = simpler
if time_to_call_it_a_day():
# We no cover in here because it's a bit sensitive to timing
# and tends to make tests flaky. There are tests that mean
# this is definitely covered most of the time.
break # pragma: no cover
if storage is not None:
storage.save(best_example)
setup_example()
return search_strategy.reify(best_example)
def test_track_iterables():
t = Tracker()
assert t.track(iter([1])) == 1
assert t.track(iter([1])) == 2
def test_track_dict():
t = Tracker()
assert t.track({1: 2}) == 1
assert t.track({1: 3}) == 1
def test_track_iterables():
t = Tracker()
assert t.track(iter([1])) == 1
assert t.track(iter([1])) == 2
def test_track_dict():
t = Tracker()
assert t.track({1: 2}) == 1
assert t.track({1: 3}) == 1
def test_tracking_classes_of_custom():
t = Tracker()
assert t.track(Foo) == 1
assert t.track(Foo) == 2
def test_track_nan():
t = Tracker()
assert t.track(float(u'nan')) == 1
assert t.track(float(u'nan')) == 2
def test_tracking_custom():
t = Tracker()
assert t.track(Foo()) == 1
assert t.track(Foo()) == 2
def test_includes_repr_in_marshal_error():
with pytest.raises(ValueError) as e:
Tracker().track(Hello())
assert u'hello world' in e.value.args[0]
def test_nested_unhashables():
t = Tracker()
x = {'foo': [1, 2, {3, 4, 5, 6}], 'bar': 10}
assert t.track(x) == 1
assert t.track(x) == 2
def falsify(self, hypothesis, *argument_types): # pylint: disable=too-many-locals,too-many-branches
"""
Attempt to construct an example tuple x matching argument_types such
that hypothesis(*x) returns a falsey value or throws an AssertionError
"""
random = self.random
if random is None:
random = Random(function_digest(hypothesis))
search_strategy = (
self.strategy_table.specification_for(argument_types))
storage = None
if self.database is not None:
try:
storage = self.database.storage_for(argument_types)
except NotSerializeable:
pass
def falsifies(args): # pylint: disable=missing-docstring
try:
return not hypothesis(*search_strategy.copy(args))
except AssertionError:
return True
except UnsatisfiedAssumption:
return False
track_seen = Tracker()
falsifying_examples = []
examples_found = 0
satisfying_examples = 0
timed_out = False
if argument_types:
max_examples = self.max_examples
min_satisfying_examples = self.min_satisfying_examples
else:
max_examples = 1
min_satisfying_examples = 1
example_source = ExampleSource(random=random,
strategy=search_strategy,
storage=storage,
min_parameters=max(
2, int(float(max_examples) / 10)))
start_time = time.time()
def time_to_call_it_a_day():
"""Have we exceeded our timeout?"""
return time.time() >= start_time + self.timeout
skipped_examples = 0
examples_seen = 0
# At present this loop will never exit normally . This needs proper
# testing when "database only" mode becomes available but right now
# it's not.
for args in example_source: # pragma: no branch
assert search_strategy.could_have_produced(args)
if falsifying_examples:
break
if examples_seen >= max_examples:
break
if time_to_call_it_a_day():
break
examples_seen += 1
if track_seen.track(args) > 1:
example_source.mark_bad()
skipped_examples += 1
if skipped_examples >= self.max_skipped_examples:
raise Exhausted(hypothesis, examples_found)
else:
# This really is covered. I suspect a bug in coverage that
# I have not yet narrowed down. It is impossible to execute
# the other branch without first executing this one and
# there is a test that cannot pass without executing the
# other branch.
continue # pragma: no cover
else:
skipped_examples = 0
examples_found += 1
try:
is_falsifying_example = not hypothesis(
*search_strategy.copy(args))
except AssertionError:
is_falsifying_example = True
except UnsatisfiedAssumption:
example_source.mark_bad()
continue
satisfying_examples += 1
if is_falsifying_example:
falsifying_examples.append(args)
run_time = time.time() - start_time
timed_out = run_time >= self.timeout
if not falsifying_examples:
if satisfying_examples < min_satisfying_examples:
raise Unsatisfiable(hypothesis, satisfying_examples, run_time)
elif timed_out:
raise Timeout(hypothesis, satisfying_examples, run_time)
else:
raise Unfalsifiable(hypothesis)
for example in falsifying_examples:
if not falsifies(example):
raise Flaky(hypothesis, example)
best_example = falsifying_examples[0]
for simpler in search_strategy.simplify_such_that(
best_example, falsifies):
best_example = simpler
if time_to_call_it_a_day():
break
if storage is not None:
storage.save(best_example)
return best_example
def test_tracking_custom():
t = Tracker()
assert t.track(Foo()) == 1
assert t.track(Foo()) == 2
def test_tracking_classes_of_custom():
t = Tracker()
assert t.track(Foo) == 1
assert t.track(Foo) == 2
def test_track_nan():
t = Tracker()
assert t.track(float('nan')) == 1
assert t.track(float('nan')) == 2
