def _():
events = []
@fixture(scope=Scope.Global)
def a():
events.append("resolve")
yield "a"
events.append("teardown")
def test1(a=a):
events.append("test1")
def test2(a=a):
events.append("test2")
def test3(a=a):
events.append("test3")
suite = Suite(tests=[
Test(fn=test1, module_name="module1"),
Test(fn=test2, module_name="module2"),
Test(fn=test3, module_name="module2"),
])
list(suite.generate_test_runs())
expect(events).equals([
"resolve", # Resolve at start of run only
"test1",
"test2",
"test3",
"teardown", # Teardown only at end of run
])
expect(len(suite.cache)).equals(0) # Teardown includes cache cleanup
def _(module=module):
events = []
@fixture
def a():
events.append(1)
# Both of the fixtures below depend on 'a', but 'a' should only be executed once.
@fixture
def b(a=a):
events.append(2)
@fixture
def c(a=a):
events.append(3)
@testable_test
def test(b=b, c=c):
pass
suite = Suite(tests=[Test(fn=test, module_name=module)])
list(suite.generate_test_runs())
assert events == [1, 2, 3]
def _(module=module):
events = []
@fixture
def fix_a():
events.append(1)
yield "a"
events.append(3)
@fixture
def fix_b():
events.append(2)
return "b"
@testable_test
def my_test(fix_a=fix_a, fix_b=fix_b):
assert fix_a == "a"
assert fix_b == "b"
suite = Suite(tests=[Test(fn=my_test, module_name=module)])
# Exhaust the test runs generator
list(suite.generate_test_runs())
assert events == [1, 2, 3]
def _(module=module):
events = []
@fixture
def fix_a():
events.append(1)
yield "a"
events.append(4)
@fixture
def fix_b():
events.append(2)
return "b"
@fixture
def fix_c(fix_b=fix_b):
events.append(3)
yield "c"
events.append(5)
def my_test(fix_a=fix_a, fix_c=fix_c):
expect(fix_a).equals("a")
expect(fix_c).equals("c")
suite = Suite(tests=[Test(fn=my_test, module_name=module)])
# Exhaust the test runs generator
list(suite.generate_test_runs())
expect(events).equals([1, 2, 3, 4, 5])
def _():
events = []
@fixture(scope=Scope.Module)
def a():
events.append("resolve a")
yield "a"
events.append("teardown a")
@testable_test
def test_1(a=each(a, "second", a)):
events.append("running test")
suite = Suite(tests=[Test(fn=test_1, module_name="module1")])
list(suite.generate_test_runs())
# Ensure that each parameterised instance of the final test in the
# module runs before the module-level teardown occurs.
assert events == [
"resolve a",
"running test",
"running test",
"running test",
"teardown a",
]
def _():
events = []
@fixture(scope=Scope.Global)
def a():
events.append("resolve a")
yield "a"
events.append("teardown a")
@fixture(scope=Scope.Module)
def b():
events.append("resolve b")
yield "b"
events.append("teardown b")
@fixture(scope=Scope.Test)
def c():
events.append("resolve c")
yield "c"
events.append("teardown c")
def test1(a=a, b=b, c=c):
events.append("test1")
def test2(a=a, b=b, c=c):
events.append("test2")
def test3(a=a, b=b, c=c):
events.append("test3")
suite = Suite(tests=[
Test(fn=test1, module_name="module1"),
Test(fn=test2, module_name="module2"),
Test(fn=test3, module_name="module2"),
])
list(suite.generate_test_runs())
# Note that the ordering of the final teardowns aren't well-defined
expect(events).equals([
"resolve a", # global fixture so resolved at start
"resolve b", # module fixture resolved at start of module1
"resolve c", # test fixture resolved at start of test1
"test1",
"teardown c", # test fixture teardown at start of test1
"teardown b", # module fixture teardown at end of module1
"resolve b", # module fixture resolved at start of module2
"resolve c", # test fixture resolved at start of test2
"test2",
"teardown c", # test fixture teardown at start of test2
"resolve c", # test fixture resolved at start of test3
"test3",
"teardown c", # test fixture teardown at end of test3
"teardown a", # global fixtures are torn down at the very end
"teardown b", # module fixture teardown at end of module2
])
expect(len(suite.cache)).equals(0)
def _(skipped=skipped_test, example=example_test):
suite = Suite(tests=[example, skipped])
test_runs = list(suite.generate_test_runs())
expected_runs = [
TestResult(example, TestOutcome.PASS, None, ""),
TestResult(skipped, TestOutcome.SKIP, None, ""),
]
assert test_runs == expected_runs
def _(skipped=skipped_test, example=example_test):
suite = Suite(tests=[example, skipped])
test_runs = list(suite.generate_test_runs(dry_run=True))
expected_runs = [
TestResult(example, TestOutcome.DRYRUN, None, ""),
TestResult(skipped, TestOutcome.DRYRUN, None, ""),
]
assert test_runs == expected_runs
def test(
ctx: click.Context,
config: str,
config_path: Optional[Path],
path: Tuple[str],
exclude: Tuple[str],
search: Optional[str],
tags: Optional[Expression],
fail_limit: Optional[int],
test_output_style: str,
order: str,
capture_output: bool,
show_slowest: int,
show_diff_symbols: bool,
dry_run: bool,
):
"""Run tests."""
start_run = default_timer()
paths = [Path(p) for p in path]
mod_infos = get_info_for_modules(paths, exclude)
modules = list(load_modules(mod_infos))
unfiltered_tests = get_tests_in_modules(modules, capture_output)
filtered_tests = list(
filter_tests(
unfiltered_tests,
query=search,
tag_expr=tags,
))
# Rewrite assertions in each test
tests = rewrite_assertions_in_tests(filtered_tests)
time_to_collect = default_timer() - start_run
suite = Suite(tests=tests)
test_results = suite.generate_test_runs(order=order, dry_run=dry_run)
writer = SimpleTestResultWrite(
suite=suite,
test_output_style=test_output_style,
config_path=config_path,
show_diff_symbols=show_diff_symbols,
)
writer.output_header(time_to_collect=time_to_collect)
results = writer.output_all_test_results(test_results,
fail_limit=fail_limit)
time_taken = default_timer() - start_run
writer.output_test_result_summary(results, time_taken, show_slowest)
exit_code = get_exit_code(results)
sys.exit(exit_code.value)
def _():
events = []
@fixture(scope=Scope.Module)
def a():
events.append("resolve")
yield "a"
events.append("teardown")
@testable_test
def test1(a=a):
events.append("test1")
@testable_test
def test2(a=a):
events.append("test2")
@testable_test
def test3(a=a):
events.append("test3")
# For testing purposes we need to assign paths ourselves,
# since our test functions are all defined at the same path
test1.ward_meta.path = "module1"
test2.ward_meta.path = "module2"
test3.ward_meta.path = "module2"
suite = Suite(
tests=[
Test(fn=test1, module_name="module1"),
Test(fn=test2, module_name="module2"),
Test(fn=test3, module_name="module2"),
]
)
list(suite.generate_test_runs())
expect(events).equals(
[
"resolve", # Resolve at start of module1
"test1",
"teardown", # Teardown at end of module1
"resolve", # Resolve at start of module2
"test2",
"test3",
"teardown", # Teardown at end of module2
]
)
def _(module=module):
@testable_test
def _():
assert False
t = Test(fn=_, module_name=module)
failing_suite = Suite(tests=[t])
results = failing_suite.generate_test_runs()
result = next(results)
expected_result = TestResult(
test=t, outcome=TestOutcome.FAIL, error=mock.ANY, message=""
)
assert result == expected_result
assert isinstance(result.error, AssertionError)
def _(module=module):
def test_i_fail():
assert False
test = Test(fn=test_i_fail, module_name=module)
failing_suite = Suite(tests=[test])
results = failing_suite.generate_test_runs()
result = next(results)
expected_result = TestResult(test=test,
outcome=TestOutcome.FAIL,
error=mock.ANY,
message="")
expect(result).equals(expected_result)
expect(result.error).instance_of(AssertionError)
def _():
@testable_test
def test_1(a=each(1, 2), b=each(1, 2, 3)):
pass
@testable_test
def test_2(a=each(1, 2), b=each(1, 2)):
pass
suite = Suite(tests=[
Test(fn=test_1, module_name="module1"),
Test(fn=test_2, module_name="module2"),
])
results = list(suite.generate_test_runs())
assert (
len(results) == 1 + 2
) # the first test doesn't expand, and the 2nd test expands into 2 tests
assert type(results[0].error) == ParameterisationError
def run(
ctx: click.Context,
path: Tuple[str],
exclude: Tuple[str],
search: Optional[str],
fail_limit: Optional[int],
test_output_style: str,
order: str,
capture_output: bool,
config: str,
show_slowest: int,
dry_run: bool,
):
start_run = default_timer()
paths = [Path(p) for p in path]
mod_infos = get_info_for_modules(paths, exclude)
modules = list(load_modules(mod_infos))
unfiltered_tests = get_tests_in_modules(modules, capture_output)
tests = list(search_generally(unfiltered_tests, query=search))
# Rewrite assertions in each test
tests = rewrite_assertions_in_tests(tests)
time_to_collect = default_timer() - start_run
suite = Suite(tests=tests)
test_results = suite.generate_test_runs(order=order, dry_run=dry_run)
writer = SimpleTestResultWrite(suite=suite,
test_output_style=test_output_style)
results = writer.output_all_test_results(test_results,
time_to_collect=time_to_collect,
fail_limit=fail_limit)
time_taken = default_timer() - start_run
writer.output_test_result_summary(results, time_taken, show_slowest)
exit_code = get_exit_code(results)
sys.exit(exit_code.value)
def run(path, search, fail_limit):
start_run = default_timer()
mod_infos = get_info_for_modules(path)
modules = list(load_modules(mod_infos))
unfiltered_tests = get_tests_in_modules(modules)
tests = search_generally(unfiltered_tests, query=search)
time_to_collect = default_timer() - start_run
suite = Suite(tests=list(tests))
test_results = suite.generate_test_runs()
writer = SimpleTestResultWrite(suite=suite)
results = writer.output_all_test_results(test_results,
time_to_collect=time_to_collect,
fail_limit=fail_limit)
time_taken = default_timer() - start_run
writer.output_test_result_summary(results, time_taken)
exit_code = get_exit_code(results)
sys.exit(exit_code.value)
def _():
events = []
@fixture(scope=Scope.Global)
def a():
events.append("resolve")
yield "a"
events.append("teardown")
@testable_test
def test1(a=a):
events.append("test1")
@testable_test
def test2(a=a):
events.append("test2")
@testable_test
def test3(a=a):
events.append("test3")
suite = Suite(
tests=[
Test(fn=test1, module_name="module1"),
Test(fn=test2, module_name="module2"),
Test(fn=test3, module_name="module2"),
]
)
list(suite.generate_test_runs())
assert events == [
"resolve", # Resolve at start of run only
"test1",
"test2",
"test3",
"teardown", # Teardown only at end of run
]
def test(
ctx: click.Context,
config: str,
config_path: Optional[Path],
path: Tuple[str],
exclude: Tuple[str],
search: Optional[str],
tags: Optional[Expression],
fail_limit: Optional[int],
test_output_style: str,
progress_style: List[str],
order: str,
capture_output: bool,
show_slowest: int,
show_diff_symbols: bool,
dry_run: bool,
):
"""Run tests."""
progress_styles = [TestProgressStyle(ps) for ps in progress_style]
if TestProgressStyle.BAR in progress_styles and test_output_style in {
"dots-global",
"dots-module",
}:
raise click.BadOptionUsage(
"progress_style",
f"The '{TestProgressStyle.BAR}' progress style cannot be used with dots-based test output styles (you asked for '{test_output_style}').",
)
init_breakpointhooks(pdb, sys)
start_run = default_timer()
paths = [Path(p) for p in path]
mod_infos = get_info_for_modules(paths, exclude)
modules = list(load_modules(mod_infos))
unfiltered_tests = get_tests_in_modules(modules, capture_output)
filtered_tests = list(
filter_tests(
unfiltered_tests,
query=search,
tag_expr=tags,
))
tests = rewrite_assertions_in_tests(filtered_tests)
time_to_collect = default_timer() - start_run
suite = Suite(tests=tests)
test_results = suite.generate_test_runs(order=order, dry_run=dry_run)
writer = SimpleTestResultWrite(
suite=suite,
test_output_style=test_output_style,
progress_styles=progress_styles,
config_path=config_path,
show_diff_symbols=show_diff_symbols,
)
writer.output_header(time_to_collect=time_to_collect)
results = writer.output_all_test_results(test_results,
fail_limit=fail_limit)
time_taken = default_timer() - start_run
writer.output_test_result_summary(results, time_taken, show_slowest)
exit_code = get_exit_code(results)
sys.exit(exit_code.value)
def suite(example_test=example_test):
return Suite(tests=[example_test] * NUMBER_OF_TESTS)
def _():
events = []
@fixture(scope=Scope.Global)
async def a():
events.append("resolve a")
yield "a"
events.append("teardown a")
@fixture(scope=Scope.Module)
async def b():
events.append("resolve b")
yield "b"
events.append("teardown b")
@fixture(scope=Scope.Test)
async def c():
events.append("resolve c")
yield "c"
events.append("teardown c")
@testable_test
async def test_1(a=a, b=b, c=c):
events.append("test1")
@testable_test
async def test_2(a=a, b=b, c=c):
events.append("test2")
@testable_test
async def test_3(a=a, b=b, c=c):
events.append("test3")
test_1.ward_meta.path = "module1"
test_2.ward_meta.path = "module2"
test_3.ward_meta.path = "module1"
suite = Suite(
tests=[
# Module ordering is intentionally off here, to ensure correct
# interaction between module-scope fixtures and random ordering
Test(fn=test_1, module_name="module1"),
Test(fn=test_2, module_name="module2"),
Test(fn=test_3, module_name="module1"),
]
)
list(suite.generate_test_runs())
assert events == [
"resolve a", # global fixture so resolved at start
"resolve b", # module fixture resolved at start of module1
"resolve c", # test fixture resolved at start of test1
"test1",
"teardown c", # test fixture teardown at start of test1
"resolve b", # module fixture resolved at start of module2
"resolve c", # test fixture resolved at start of test2
"test2",
"teardown c", # test fixture teardown at start of test2
"teardown b", # module fixture teardown at end of module2
"resolve c", # test fixture resolved at start of test3
"test3",
"teardown c", # test fixture teardown at end of test3
"teardown b", # module fixture teardown at end of module1
"teardown a", # global fixtures are torn down at the very end
]
