def test_contains_the_test_function_name_in_the_exception_string():
calls = [0]
@given(integers(), settings=Settings(max_iterations=10, max_examples=10))
def this_has_a_totally_unique_name(x):
calls[0] += 1
assume(False)
with raises(Unsatisfiable) as e:
this_has_a_totally_unique_name()
print(u'Called %d times' % tuple(calls))
assert this_has_a_totally_unique_name.__name__ in e.value.args[0]
calls2 = [0]
class Foo(object):
@given(integers(), settings=Settings(
max_iterations=10, max_examples=10))
def this_has_a_unique_name_and_lives_on_a_class(self, x):
calls2[0] += 1
assume(False)
with raises(Unsatisfiable) as e:
Foo().this_has_a_unique_name_and_lives_on_a_class()
print(u'Called %d times' % tuple(calls2))
assert (
Foo.this_has_a_unique_name_and_lives_on_a_class.__name__
) in e.value.args[0]
def test_contains_the_test_function_name_in_the_exception_string():
look_for_one = settings(
max_examples=1, suppress_health_check=HealthCheck.all())
@given(integers())
@look_for_one
def this_has_a_totally_unique_name(x):
reject()
with raises(Unsatisfiable) as e:
this_has_a_totally_unique_name()
assert this_has_a_totally_unique_name.__name__ in e.value.args[0]
class Foo(object):
@given(integers())
@look_for_one
def this_has_a_unique_name_and_lives_on_a_class(self, x):
reject()
with raises(Unsatisfiable) as e:
Foo().this_has_a_unique_name_and_lives_on_a_class()
assert (
Foo.this_has_a_unique_name_and_lives_on_a_class.__name__
) in e.value.args[0]
def test_stability():
@given(
st.lists(st.text(min_size=1, max_size=1), unique=True, min_size=5),
st.choices(),
)
@settings(
database=ExampleDatabase(),
)
def test_choose_and_then_fail(ls, choice):
for _ in hrange(100):
choice(ls)
assert False
# Run once first for easier debugging
with raises(AssertionError):
test_choose_and_then_fail()
with capture_out() as o:
with raises(AssertionError):
test_choose_and_then_fail()
out1 = o.getvalue()
with capture_out() as o:
with raises(AssertionError):
test_choose_and_then_fail()
out2 = o.getvalue()
assert out1 == out2
assert 'Choice #100:' in out1
def test_rejects_invalid_step_sizes_in_data():
runner = DepthMachine.find_breaking_runner()
strategy = StateMachineSearchStrategy()
basic = strategy.to_basic(runner)
assert isinstance(basic[2], int)
basic[2] = -1
with raises(BadData):
strategy.from_basic(basic)
basic[2] = 1000000
with raises(BadData):
strategy.from_basic(basic)
def test_errors_on_extra_kwargs():
def foo(a):
pass
with raises(TypeError) as e:
convert_keyword_arguments(foo, (1,), {'b': 1})
assert 'keyword' in e.value.args[0]
with raises(TypeError) as e2:
convert_keyword_arguments(foo, (1,), {'b': 1, 'c': 2})
assert 'keyword' in e2.value.args[0]
def test_positional_errors_if_given_duplicate_kwargs():
def foo(a):
pass
with raises(TypeError) as e:
convert_positional_arguments(foo, (2,), {'a': 1})
assert 'multiple values' in e.value.args[0]
def test_positional_errors_if_given_bad_kwargs():
def foo(a):
pass
with raises(TypeError) as e:
convert_positional_arguments(foo, (), {'b': 1})
assert 'unexpected keyword argument' in e.value.args[0]
def test_can_run_without_database():
@given(integers())
@settings(database=None)
def test_blah(x):
assert False
with raises(AssertionError):
test_blah()
def test_errors_even_if_does_not_error_on_final_call():
@given(integers())
def rude(x):
assert not any(t[3] == u"best_satisfying_template" for t in inspect.getouterframes(inspect.currentframe()))
with raises(Flaky):
rude()
def test_positional_errors_if_too_many_args():
def foo(a):
pass
with raises(TypeError) as e:
convert_positional_arguments(foo, (1, 2), {})
assert '2 given' in e.value.args[0]
def test_reports_repr_diff_in_flaky_error():
@given(builds(DifferentReprEachTime))
def rude(x):
assert not any(t[3] == u"best_satisfying_template" for t in inspect.getouterframes(inspect.currentframe()))
with raises(Flaky) as e:
rude()
assert u"Call 1:" in e.value.args[0]
def test_machine_with_no_terminals_is_invalid():
class NonTerminalMachine(RuleBasedStateMachine):
@rule(value=Bundle(u"hi"))
def bye(self, hi):
pass
with raises(InvalidDefinition):
NonTerminalMachine.TestCase().runTest()
def test_does_not_accept_random_if_derandomize():
with raises(InvalidArgument):
@given(integers(), settings=Settings(derandomize=True), random=Random())
def test_blah(x):
pass
test_blah()
def test_still_minimizes_on_non_assertion_failures():
@given(integers(), settings=Settings(max_examples=50))
def is_not_too_large(x):
if x >= 10:
raise ValueError(u'No, %s is just too large. Sorry' % x)
with raises(ValueError) as exinfo:
is_not_too_large()
assert u' 10 ' in exinfo.value.args[0]
def test_does_not_attempt_to_shrink_flaky_errors():
values = []
@given(integers())
def test(x):
values.append(x)
assert len(values) != 1
with raises(Flaky):
test()
assert len(set(values)) == 1
def test_does_not_catch_interrupt_during_falsify():
calls = [0]
@given(integers())
def flaky_base_exception(x):
if not calls[0]:
calls[0] = 1
raise KeyboardInterrupt()
with raises(KeyboardInterrupt):
flaky_base_exception()
def test_prints_notes_once_on_failure():
@given(lists(integers()), settings=Settings(database=None))
def test(xs):
note('Hi there')
assert sum(xs) > 100
with capture_out() as out:
with reporting.with_reporter(reporting.default):
with raises(AssertionError):
test()
lines = out.getvalue().strip().splitlines()
assert len(lines) == 2
assert 'Hi there' in lines
def test_bad_machines_fail(machine):
test_class = machine.TestCase
try:
with capture_out() as o:
with raises(AssertionError):
test_class().runTest()
except Exception:
print_unicode(o.getvalue())
raise
v = o.getvalue()
print_unicode(v)
steps = [l for l in v.splitlines() if 'Step ' in l or 'state.' in l]
assert 1 <= len(steps) <= 50
def test_can_serialize_statemachine_execution(machine):
runner = machine.find_breaking_runner()
strategy = StateMachineSearchStrategy()
new_runner = strategy.from_basic(strategy.to_basic(runner))
with raises(AssertionError):
new_runner.run(machine())
r = Random(1)
for simplifier in strategy.simplifiers(r, new_runner):
try:
next(simplifier(r, new_runner))
except StopIteration:
pass
def test_prints_on_failure_by_default():
@given(integers(), integers(), settings=Settings(max_examples=200, timeout=-1))
def test_ints_are_sorted(balthazar, evans):
assume(evans >= 0)
assert balthazar <= evans
with raises(AssertionError):
with capture_out() as out:
with reporting.with_reporter(reporting.default):
test_ints_are_sorted()
out = out.getvalue()
lines = [l.strip() for l in out.split(u"\n")]
assert u"Falsifying example: test_ints_are_sorted(balthazar=1, evans=0)" in lines
def test_prints_notes_once_on_failure():
@given(lists(integers()))
@settings(database=None, verbosity=Verbosity.normal)
def test(xs):
note('Hi there')
if sum(xs) <= 100:
raise ValueError()
with capture_out() as out:
with reporting.with_reporter(reporting.default):
with raises(ValueError):
test()
lines = out.getvalue().strip().splitlines()
assert lines.count('Hi there') == 1
def test_bad_machines_fail(machine):
test_class = machine.TestCase
try:
with capture_out() as o:
with raises(AssertionError):
test_class().runTest()
except Exception:
print_unicode(o.getvalue())
raise
v = o.getvalue()
print_unicode(v)
assert u'Step #1' in v
assert u'Step #50' not in v
def test_can_timeout_during_an_unsuccessful_simplify():
record = []
@debug.timeout(3)
@given(lists(floats()), settings=Settings(timeout=1))
def first_bad_float_list(xs):
if record:
assert record[0] != xs
elif len(xs) >= 10 and any(math.isinf(x) for x in xs):
record.append(xs)
assert False
with raises(AssertionError):
first_bad_float_list()
def test_given_twice_is_same():
@given(st.data(), st.data())
def test(data1, data2):
data1.draw(st.integers())
data2.draw(st.integers())
raise ValueError()
with raises(ValueError):
with capture_out() as out:
with reporting.with_reporter(reporting.default):
test()
result = out.getvalue()
assert "Draw 1: 0" in result
assert "Draw 2: 0" in result
def test_when_set_to_no_simplifies_only_runs_failing_example_once():
failing = [0]
@given(integers(), settings=Settings(max_shrinks=0, max_examples=200))
def foo(x):
if x > 11:
failing[0] += 1
assert False
with Settings(verbosity=Verbosity.normal):
with raises(AssertionError):
with capture_out() as out:
foo()
assert failing == [1]
assert u'Trying example' in out.getvalue()
def test_prints_on_failure():
@given(st.data())
def test(data):
x = data.draw(st.lists(st.integers(), min_size=1))
y = data.draw(st.sampled_from(x))
assert y in x
x.remove(y)
assert y not in x
with raises(AssertionError):
with capture_out() as out:
with reporting.with_reporter(reporting.default):
test()
result = out.getvalue()
assert 'Draw 1: [0, 0]' in result
assert 'Draw 2: 0' in result
def test_prints_labels_if_given_on_failure():
@given(st.data())
def test(data):
x = data.draw(st.lists(st.integers(0, 10), min_size=2), label="Some numbers")
y = data.draw(st.sampled_from(x), label="A number")
assert y in x
x.remove(y)
assert y not in x
with raises(AssertionError):
with capture_out() as out:
with reporting.with_reporter(reporting.default):
test()
result = out.getvalue()
assert "Draw 1 (Some numbers): [0, 0]" in result
assert "Draw 2 (A number): 0" in result
def test_prints_on_failure():
@given(st.data())
def test(data):
x = data.draw(st.lists(st.integers(0, 10), min_size=2))
y = data.draw(st.sampled_from(x))
x.remove(y)
if y in x:
raise ValueError()
with raises(ValueError):
with capture_out() as out:
with reporting.with_reporter(reporting.default):
test()
result = out.getvalue()
assert "Draw 1: [0, 0]" in result
assert "Draw 2: 0" in result
def test_when_set_to_no_simplifies_runs_failing_example_twice():
failing = [0]
@given(integers())
@settings(phases=no_shrink, max_examples=100, verbosity=Verbosity.normal)
def foo(x):
if x > 11:
note('Lo')
failing[0] += 1
assert False
with raises(AssertionError):
with capture_out() as out:
foo()
assert failing == [2]
assert 'Falsifying example' in out.getvalue()
assert 'Lo' in out.getvalue()
def test_no_double_invariant():
"""The invariant decorator can't be applied multiple times to a single
function."""
with raises(InvalidDefinition):
class Invariant(RuleBasedStateMachine):
def __init__(self):
super(Invariant, self).__init__()
@invariant()
@invariant()
def test_blah(self):
pass
@rule()
def do_stuff(self):
pass
def test_errors_if_not_enough_args():
def foo(a, b, c, d=1):
pass # pragma: no cover
with raises(TypeError):
convert_keyword_arguments(foo, (1, 2), {'d': 4})
def test_copy_argspec_validates_arguments():
with raises(ValueError):
copy_argspec(
'hello_world',
ArgSpec(args=['a b'], varargs=None, keywords=None, defaults=None))
def test_flaky_raises_flaky():
with raises(Flaky):
FlakyStateMachine.TestCase().runTest()
def test_choice_raises_index_error_on_empty():
c = find(st.choices(), lambda c: True)
with raises(IndexError):
c([])
def test_define_function_signature_validates_arguments():
with raises(ValueError):
define_function_signature(
'hello_world', None,
ArgSpec(args=['a b'], varargs=None, keywords=None, defaults=None))
def test_copy_argspec_validates_function_name():
with raises(ValueError):
copy_argspec('hello world', ArgSpec(
args=['a', 'b'], varargs=None, keywords=None, defaults=None))
def test_saves_failing_example_in_database():
db = ExampleDatabase()
with raises(AssertionError):
run_state_machine_as_test(SetStateMachine, Settings(database=db))
assert len(list(db.data.keys())) == 1
def test_errors_if_not_enough_args():
def foo(a, b, c, d=1):
pass
with raises(TypeError):
convert_keyword_arguments(foo, (1, 2), {"d": 4})
def test_positional_errors_if_too_few_args():
def foo(a, b, c):
pass
with raises(TypeError):
convert_positional_arguments(foo, (1, 2), {})
def test_errors_on_bad_kwargs():
def bar():
pass
with raises(TypeError):
convert_keyword_arguments(bar, (), {"foo": 1})
def test_empty_machine_is_invalid():
class EmptyMachine(RuleBasedStateMachine):
pass
with raises(InvalidDefinition):
EmptyMachine.TestCase().runTest()
def test_can_use_factory_for_tests():
with raises(ValueError):
run_state_machine_as_test(lambda: RequiresInit(42))
def test_can_run_with_no_db():
with raises(AssertionError):
run_state_machine_as_test(SetStateMachine, Settings(database=None))
def test_ratchetting_raises_flaky():
with raises(Flaky):
FlakyRatchettingMachine.TestCase().runTest()
def test_errors_if_keyword_precedes_positional():
def foo(x, y):
pass # pragma: no cover
with raises(TypeError):
convert_keyword_arguments(foo, (1, ), {'x': 2})
def test_can_use_factory_for_tests():
with raises(ValueError):
run_state_machine_as_test(
lambda: RequiresInit(42), settings=Settings(max_examples=100)
)
def test_errors_if_keyword_precedes_positional():
def foo(x, y):
pass
with raises(TypeError):
convert_keyword_arguments(foo, (1,), {"x": 2})
def test_can_choose_within_stateful():
with raises(AssertionError):
run_state_machine_as_test(ChoosingStateMachine)
def test_errors_when_asked_for_example():
with raises(InvalidArgument):
st.data().example()
def test_define_function_signature_validates_function_name():
with raises(ValueError):
define_function_signature('hello world', None, FullArgSpec(
args=['a', 'b'], varargs=None, varkw=None, defaults=None,
kwonlyargs=[], kwonlydefaults=None, annotations={}))
def test_errors_when_normal_strategy_functions_are_used(f):
with raises(InvalidArgument):
getattr(st.data(), f)(lambda x: 1)
def test_errors_when_used_in_find():
with raises(InvalidArgument):
find(st.data(), lambda x: x.draw(st.booleans()))
def test_errors_on_bad_kwargs():
def bar():
pass # pragma: no cover
with raises(TypeError):
convert_keyword_arguments(bar, (), {'foo': 1})
def test_saves_failing_example_in_database():
db = ExampleDatabase(':memory:')
with raises(AssertionError):
run_state_machine_as_test(SetStateMachine, Settings(database=db))
assert any(list(db.data.values()))
def test_flaky_draw_less_raises_flaky():
with raises(Flaky):
FlakyDrawLessMachine.TestCase().runTest()
