def test_contains_the_test_function_name_in_the_exception_string(): calls = [0] @given(integers(), settings=Settings(max_iterations=10, max_examples=10)) def this_has_a_totally_unique_name(x): calls[0] += 1 assume(False) with raises(Unsatisfiable) as e: this_has_a_totally_unique_name() print(u'Called %d times' % tuple(calls)) assert this_has_a_totally_unique_name.__name__ in e.value.args[0] calls2 = [0] class Foo(object): @given(integers(), settings=Settings( max_iterations=10, max_examples=10)) def this_has_a_unique_name_and_lives_on_a_class(self, x): calls2[0] += 1 assume(False) with raises(Unsatisfiable) as e: Foo().this_has_a_unique_name_and_lives_on_a_class() print(u'Called %d times' % tuple(calls2)) assert ( Foo.this_has_a_unique_name_and_lives_on_a_class.__name__ ) in e.value.args[0]
def test_contains_the_test_function_name_in_the_exception_string(): look_for_one = settings( max_examples=1, suppress_health_check=HealthCheck.all()) @given(integers()) @look_for_one def this_has_a_totally_unique_name(x): reject() with raises(Unsatisfiable) as e: this_has_a_totally_unique_name() assert this_has_a_totally_unique_name.__name__ in e.value.args[0] class Foo(object): @given(integers()) @look_for_one def this_has_a_unique_name_and_lives_on_a_class(self, x): reject() with raises(Unsatisfiable) as e: Foo().this_has_a_unique_name_and_lives_on_a_class() assert ( Foo.this_has_a_unique_name_and_lives_on_a_class.__name__ ) in e.value.args[0]
def test_stability(): @given( st.lists(st.text(min_size=1, max_size=1), unique=True, min_size=5), st.choices(), ) @settings( database=ExampleDatabase(), ) def test_choose_and_then_fail(ls, choice): for _ in hrange(100): choice(ls) assert False # Run once first for easier debugging with raises(AssertionError): test_choose_and_then_fail() with capture_out() as o: with raises(AssertionError): test_choose_and_then_fail() out1 = o.getvalue() with capture_out() as o: with raises(AssertionError): test_choose_and_then_fail() out2 = o.getvalue() assert out1 == out2 assert 'Choice #100:' in out1
def test_rejects_invalid_step_sizes_in_data(): runner = DepthMachine.find_breaking_runner() strategy = StateMachineSearchStrategy() basic = strategy.to_basic(runner) assert isinstance(basic[2], int) basic[2] = -1 with raises(BadData): strategy.from_basic(basic) basic[2] = 1000000 with raises(BadData): strategy.from_basic(basic)
def test_errors_on_extra_kwargs(): def foo(a): pass with raises(TypeError) as e: convert_keyword_arguments(foo, (1,), {'b': 1}) assert 'keyword' in e.value.args[0] with raises(TypeError) as e2: convert_keyword_arguments(foo, (1,), {'b': 1, 'c': 2}) assert 'keyword' in e2.value.args[0]
def test_positional_errors_if_given_duplicate_kwargs(): def foo(a): pass with raises(TypeError) as e: convert_positional_arguments(foo, (2,), {'a': 1}) assert 'multiple values' in e.value.args[0]
def test_positional_errors_if_given_bad_kwargs(): def foo(a): pass with raises(TypeError) as e: convert_positional_arguments(foo, (), {'b': 1}) assert 'unexpected keyword argument' in e.value.args[0]
def test_can_run_without_database(): @given(integers()) @settings(database=None) def test_blah(x): assert False with raises(AssertionError): test_blah()
def test_errors_even_if_does_not_error_on_final_call(): @given(integers()) def rude(x): assert not any(t[3] == u"best_satisfying_template" for t in inspect.getouterframes(inspect.currentframe())) with raises(Flaky): rude()
def test_positional_errors_if_too_many_args(): def foo(a): pass with raises(TypeError) as e: convert_positional_arguments(foo, (1, 2), {}) assert '2 given' in e.value.args[0]
def test_reports_repr_diff_in_flaky_error(): @given(builds(DifferentReprEachTime)) def rude(x): assert not any(t[3] == u"best_satisfying_template" for t in inspect.getouterframes(inspect.currentframe())) with raises(Flaky) as e: rude() assert u"Call 1:" in e.value.args[0]
def test_machine_with_no_terminals_is_invalid(): class NonTerminalMachine(RuleBasedStateMachine): @rule(value=Bundle(u"hi")) def bye(self, hi): pass with raises(InvalidDefinition): NonTerminalMachine.TestCase().runTest()
def test_does_not_accept_random_if_derandomize(): with raises(InvalidArgument): @given(integers(), settings=Settings(derandomize=True), random=Random()) def test_blah(x): pass test_blah()
def test_still_minimizes_on_non_assertion_failures(): @given(integers(), settings=Settings(max_examples=50)) def is_not_too_large(x): if x >= 10: raise ValueError(u'No, %s is just too large. Sorry' % x) with raises(ValueError) as exinfo: is_not_too_large() assert u' 10 ' in exinfo.value.args[0]
def test_does_not_attempt_to_shrink_flaky_errors(): values = [] @given(integers()) def test(x): values.append(x) assert len(values) != 1 with raises(Flaky): test() assert len(set(values)) == 1
def test_does_not_catch_interrupt_during_falsify(): calls = [0] @given(integers()) def flaky_base_exception(x): if not calls[0]: calls[0] = 1 raise KeyboardInterrupt() with raises(KeyboardInterrupt): flaky_base_exception()
def test_prints_notes_once_on_failure(): @given(lists(integers()), settings=Settings(database=None)) def test(xs): note('Hi there') assert sum(xs) > 100 with capture_out() as out: with reporting.with_reporter(reporting.default): with raises(AssertionError): test() lines = out.getvalue().strip().splitlines() assert len(lines) == 2 assert 'Hi there' in lines
def test_bad_machines_fail(machine): test_class = machine.TestCase try: with capture_out() as o: with raises(AssertionError): test_class().runTest() except Exception: print_unicode(o.getvalue()) raise v = o.getvalue() print_unicode(v) steps = [l for l in v.splitlines() if 'Step ' in l or 'state.' in l] assert 1 <= len(steps) <= 50
def test_can_serialize_statemachine_execution(machine): runner = machine.find_breaking_runner() strategy = StateMachineSearchStrategy() new_runner = strategy.from_basic(strategy.to_basic(runner)) with raises(AssertionError): new_runner.run(machine()) r = Random(1) for simplifier in strategy.simplifiers(r, new_runner): try: next(simplifier(r, new_runner)) except StopIteration: pass
def test_prints_on_failure_by_default(): @given(integers(), integers(), settings=Settings(max_examples=200, timeout=-1)) def test_ints_are_sorted(balthazar, evans): assume(evans >= 0) assert balthazar <= evans with raises(AssertionError): with capture_out() as out: with reporting.with_reporter(reporting.default): test_ints_are_sorted() out = out.getvalue() lines = [l.strip() for l in out.split(u"\n")] assert u"Falsifying example: test_ints_are_sorted(balthazar=1, evans=0)" in lines
def test_prints_notes_once_on_failure(): @given(lists(integers())) @settings(database=None, verbosity=Verbosity.normal) def test(xs): note('Hi there') if sum(xs) <= 100: raise ValueError() with capture_out() as out: with reporting.with_reporter(reporting.default): with raises(ValueError): test() lines = out.getvalue().strip().splitlines() assert lines.count('Hi there') == 1
def test_bad_machines_fail(machine): test_class = machine.TestCase try: with capture_out() as o: with raises(AssertionError): test_class().runTest() except Exception: print_unicode(o.getvalue()) raise v = o.getvalue() print_unicode(v) assert u'Step #1' in v assert u'Step #50' not in v
def test_can_timeout_during_an_unsuccessful_simplify(): record = [] @debug.timeout(3) @given(lists(floats()), settings=Settings(timeout=1)) def first_bad_float_list(xs): if record: assert record[0] != xs elif len(xs) >= 10 and any(math.isinf(x) for x in xs): record.append(xs) assert False with raises(AssertionError): first_bad_float_list()
def test_given_twice_is_same(): @given(st.data(), st.data()) def test(data1, data2): data1.draw(st.integers()) data2.draw(st.integers()) raise ValueError() with raises(ValueError): with capture_out() as out: with reporting.with_reporter(reporting.default): test() result = out.getvalue() assert "Draw 1: 0" in result assert "Draw 2: 0" in result
def test_when_set_to_no_simplifies_only_runs_failing_example_once(): failing = [0] @given(integers(), settings=Settings(max_shrinks=0, max_examples=200)) def foo(x): if x > 11: failing[0] += 1 assert False with Settings(verbosity=Verbosity.normal): with raises(AssertionError): with capture_out() as out: foo() assert failing == [1] assert u'Trying example' in out.getvalue()
def test_prints_on_failure(): @given(st.data()) def test(data): x = data.draw(st.lists(st.integers(), min_size=1)) y = data.draw(st.sampled_from(x)) assert y in x x.remove(y) assert y not in x with raises(AssertionError): with capture_out() as out: with reporting.with_reporter(reporting.default): test() result = out.getvalue() assert 'Draw 1: [0, 0]' in result assert 'Draw 2: 0' in result
def test_prints_labels_if_given_on_failure(): @given(st.data()) def test(data): x = data.draw(st.lists(st.integers(0, 10), min_size=2), label="Some numbers") y = data.draw(st.sampled_from(x), label="A number") assert y in x x.remove(y) assert y not in x with raises(AssertionError): with capture_out() as out: with reporting.with_reporter(reporting.default): test() result = out.getvalue() assert "Draw 1 (Some numbers): [0, 0]" in result assert "Draw 2 (A number): 0" in result
def test_prints_on_failure(): @given(st.data()) def test(data): x = data.draw(st.lists(st.integers(0, 10), min_size=2)) y = data.draw(st.sampled_from(x)) x.remove(y) if y in x: raise ValueError() with raises(ValueError): with capture_out() as out: with reporting.with_reporter(reporting.default): test() result = out.getvalue() assert "Draw 1: [0, 0]" in result assert "Draw 2: 0" in result
def test_when_set_to_no_simplifies_runs_failing_example_twice(): failing = [0] @given(integers()) @settings(phases=no_shrink, max_examples=100, verbosity=Verbosity.normal) def foo(x): if x > 11: note('Lo') failing[0] += 1 assert False with raises(AssertionError): with capture_out() as out: foo() assert failing == [2] assert 'Falsifying example' in out.getvalue() assert 'Lo' in out.getvalue()
def test_no_double_invariant(): """The invariant decorator can't be applied multiple times to a single function.""" with raises(InvalidDefinition): class Invariant(RuleBasedStateMachine): def __init__(self): super(Invariant, self).__init__() @invariant() @invariant() def test_blah(self): pass @rule() def do_stuff(self): pass
def test_errors_if_not_enough_args(): def foo(a, b, c, d=1): pass # pragma: no cover with raises(TypeError): convert_keyword_arguments(foo, (1, 2), {'d': 4})
def test_copy_argspec_validates_arguments(): with raises(ValueError): copy_argspec( 'hello_world', ArgSpec(args=['a b'], varargs=None, keywords=None, defaults=None))
def test_flaky_raises_flaky(): with raises(Flaky): FlakyStateMachine.TestCase().runTest()
def test_choice_raises_index_error_on_empty(): c = find(st.choices(), lambda c: True) with raises(IndexError): c([])
def test_define_function_signature_validates_arguments(): with raises(ValueError): define_function_signature( 'hello_world', None, ArgSpec(args=['a b'], varargs=None, keywords=None, defaults=None))
def test_copy_argspec_validates_function_name(): with raises(ValueError): copy_argspec('hello world', ArgSpec( args=['a', 'b'], varargs=None, keywords=None, defaults=None))
def test_saves_failing_example_in_database(): db = ExampleDatabase() with raises(AssertionError): run_state_machine_as_test(SetStateMachine, Settings(database=db)) assert len(list(db.data.keys())) == 1
def test_errors_if_not_enough_args(): def foo(a, b, c, d=1): pass with raises(TypeError): convert_keyword_arguments(foo, (1, 2), {"d": 4})
def test_positional_errors_if_too_few_args(): def foo(a, b, c): pass with raises(TypeError): convert_positional_arguments(foo, (1, 2), {})
def test_errors_on_bad_kwargs(): def bar(): pass with raises(TypeError): convert_keyword_arguments(bar, (), {"foo": 1})
def test_empty_machine_is_invalid(): class EmptyMachine(RuleBasedStateMachine): pass with raises(InvalidDefinition): EmptyMachine.TestCase().runTest()
def test_can_use_factory_for_tests(): with raises(ValueError): run_state_machine_as_test(lambda: RequiresInit(42))
def test_can_run_with_no_db(): with raises(AssertionError): run_state_machine_as_test(SetStateMachine, Settings(database=None))
def test_ratchetting_raises_flaky(): with raises(Flaky): FlakyRatchettingMachine.TestCase().runTest()
def test_errors_if_keyword_precedes_positional(): def foo(x, y): pass # pragma: no cover with raises(TypeError): convert_keyword_arguments(foo, (1, ), {'x': 2})
def test_can_use_factory_for_tests(): with raises(ValueError): run_state_machine_as_test( lambda: RequiresInit(42), settings=Settings(max_examples=100) )
def test_errors_if_keyword_precedes_positional(): def foo(x, y): pass with raises(TypeError): convert_keyword_arguments(foo, (1,), {"x": 2})
def test_can_choose_within_stateful(): with raises(AssertionError): run_state_machine_as_test(ChoosingStateMachine)
def test_errors_when_asked_for_example(): with raises(InvalidArgument): st.data().example()
def test_define_function_signature_validates_function_name(): with raises(ValueError): define_function_signature('hello world', None, FullArgSpec( args=['a', 'b'], varargs=None, varkw=None, defaults=None, kwonlyargs=[], kwonlydefaults=None, annotations={}))
def test_errors_when_normal_strategy_functions_are_used(f): with raises(InvalidArgument): getattr(st.data(), f)(lambda x: 1)
def test_errors_when_used_in_find(): with raises(InvalidArgument): find(st.data(), lambda x: x.draw(st.booleans()))
def test_errors_on_bad_kwargs(): def bar(): pass # pragma: no cover with raises(TypeError): convert_keyword_arguments(bar, (), {'foo': 1})
def test_saves_failing_example_in_database(): db = ExampleDatabase(':memory:') with raises(AssertionError): run_state_machine_as_test(SetStateMachine, Settings(database=db)) assert any(list(db.data.values()))
def test_flaky_draw_less_raises_flaky(): with raises(Flaky): FlakyDrawLessMachine.TestCase().runTest()