def test_dynamic_rerun_triggers_can_handle_regexes(testdir, rerun_regex,
should_rerun):
rerun_amount = 2
testdir.makeini("""
[pytest]
dynamic_rerun_attempts = {}
dynamic_rerun_schedule = * * * * * *
dynamic_rerun_triggers = {}
""".format(rerun_amount, rerun_regex))
testdir.makepyfile(
"def test_all_seems_well_but(): print('My print output')")
result = testdir.runpytest("-v")
if should_rerun:
failed_amount = 1
dynamic_rerun_amount = rerun_amount
passed_amount = 0
assert result.ret == pytest.ExitCode.TESTS_FAILED
else:
failed_amount = 0
dynamic_rerun_amount = 0
passed_amount = 1
assert result.ret == pytest.ExitCode.OK
_assert_result_outcomes(
result,
dynamic_rerun=dynamic_rerun_amount,
failed=failed_amount,
passed=passed_amount,
)
def test_plugin_doesnt_reread_old_sections_on_rerun(testdir):
# NOTE: We intentionally raise an exception instead of printing something else
# since exceptions don't add to the report.sections object
testdir.makepyfile("""
COUNTER = 0
def test_usually_prints_foo():
global COUNTER
COUNTER = COUNTER + 1
if COUNTER == 5:
raise ValueError("bar")
else:
print("foo")
""")
result = testdir.runpytest(
"-v",
"--dynamic-rerun-attempts=10",
"--dynamic-rerun-schedule='* * * * * */5'",
"--dynamic-rerun-triggers=foo",
)
assert result.ret == pytest.ExitCode.TESTS_FAILED
_assert_result_outcomes(result, dynamic_rerun=4, failed=1)
def test_triggering_passing_and_failing_tests_properly_run_in_same_collection(
testdir, test_body):
testdir.makepyfile(test_body)
result = testdir.runpytest("-v")
assert result.ret == pytest.ExitCode.TESTS_FAILED
_assert_result_outcomes(result, dynamic_rerun=3, failed=2, passed=1)
def test_dynamic_rerun_properly_adheres_to_schedule(
testdir, rerun_amount, rerun_regex, rerun_schedule, max_wait_seconds, should_rerun
):
if should_rerun:
failed_amount = 1
dynamic_rerun_amount = rerun_amount
passed_amount = 0
else:
failed_amount = 0
dynamic_rerun_amount = 0
passed_amount = 1
testdir.makeini(
"""
[pytest]
dynamic_rerun_attempts = {}
dynamic_rerun_schedule = {}
dynamic_rerun_triggers = {}
""".format(
rerun_amount, rerun_schedule, rerun_regex
)
)
testdir.makeconftest(
"""
def pytest_sessionfinish(session, exitstatus):
if {}:
# A little hacky, but we know we can only ever have 1 item
sleep_times_for_item = session.dynamic_rerun_items[0].dynamic_rerun_sleep_times
assert len(sleep_times_for_item) == {}
for sleep_time in sleep_times_for_item:
assert sleep_time.days == 0
assert sleep_time.seconds >= 0 and sleep_time.seconds <= {}
assert sleep_time.microseconds
else:
assert not session.dynamic_rerun_items
""".format(
should_rerun, dynamic_rerun_amount, max_wait_seconds
)
)
testdir.makepyfile("def test_print_message(): print('My print message')")
result = testdir.runpytest("-v")
if should_rerun:
assert result.ret == pytest.ExitCode.TESTS_FAILED
else:
assert result.ret == pytest.ExitCode.OK
_assert_result_outcomes(
result,
dynamic_rerun=dynamic_rerun_amount,
failed=failed_amount,
passed=passed_amount,
)
def test_flags_take_precedence_over_ini_file(testdir):
attempts = 5
failed_amount = 1
rerun_triggers = "foo"
rerun_schedule = "* * * * * *"
testdir.makepyfile("def test_print_foo(): print('foo')")
testdir.makeconftest("""
def pytest_sessionfinish(session, exitstatus):
# a little hacky, but we know we can only ever have 1 item
rerun_item = session.dynamic_rerun_items[0]
# first, check the sleep times
sleep_times_for_item = rerun_item.dynamic_rerun_sleep_times
assert len(sleep_times_for_item) == {0}
for sleep_time in sleep_times_for_item:
assert sleep_time.days == 0
assert sleep_time.seconds >= 0
assert sleep_time.microseconds
# Then, the triggers, schedule, and rerun attempts
assert rerun_item.dynamic_rerun_triggers == ["{1}"]
assert rerun_item.dynamic_rerun_schedule == "{2}"
assert rerun_item.max_allowed_dynamic_rerun_attempts == {0}
""".format(attempts, rerun_triggers, rerun_schedule))
testdir.makeini("""
[pytest]
dynamic_rerun_attempts = 2
dynamic_rerun_schedule = * * * * *
dynamic_rerun_triggers = blah
""")
# NOTE: Intentionally leaving dynamic-rerun-triggers unquoted.
# When having it quoted this test fails, but I verified that
# failure will not happen, and that the test behaves as expected
# when run via command line. Seems to be a testdir bug
result = testdir.runpytest(
"-v",
"--dynamic-rerun-attempts={}".format(attempts),
"--dynamic-rerun-schedule='{}'".format(rerun_schedule),
"--dynamic-rerun-triggers={}".format(rerun_triggers),
)
assert result.ret == pytest.ExitCode.TESTS_FAILED
_assert_result_outcomes(
result,
dynamic_rerun=attempts,
failed=failed_amount,
)
def test_one_dynamic_rerun_attempt_by_default(testdir):
testdir.makeini(
"""
[pytest]
dynamic_rerun_schedule = * * * * * *
"""
)
testdir.makepyfile("def test_always_false(): assert False")
result = testdir.runpytest("-v")
assert result.ret == pytest.ExitCode.TESTS_FAILED
_assert_result_outcomes(result, dynamic_rerun=1, failed=1)
def test_errors_no_longer_rerun_by_default_when_dynamic_rerun_triggers_provided(
testdir, ):
testdir.makeini("""
[pytest]
dynamic_rerun_schedule = * * * * * *
dynamic_rerun_attempts = 99
dynamic_rerun_triggers = this will trigger a rerun
""")
testdir.makepyfile("def test_always_false(): assert False")
result = testdir.runpytest("-v")
assert result.ret == pytest.ExitCode.TESTS_FAILED
_assert_result_outcomes(result, failed=1)
def test_success_stops_dynamic_rerun_attempts(testdir, pytest_file, expected_reruns):
testdir.makeini(
"""
[pytest]
dynamic_rerun_schedule = * * * * * *
dynamic_rerun_attempts = 99
"""
)
testdir.makepyfile(pytest_file)
result = testdir.runpytest("-v")
assert result.ret == pytest.ExitCode.OK
_assert_result_outcomes(result, dynamic_rerun=expected_reruns, passed=1)
def test_mark_takes_precedence_over_ini_file(testdir):
attempts = 5
failed_amount = 1
rerun_triggers = "foo"
rerun_schedule = "* * * * * *"
testdir.makepyfile("""
import pytest
@pytest.mark.dynamicrerun(attempts={}, triggers="{}", schedule="{}")
def test_assert_false():
print("foo")
""".format(attempts, rerun_triggers, rerun_schedule))
testdir.makeconftest("""
def pytest_sessionfinish(session, exitstatus):
# a little hacky, but we know we can only ever have 1 item
rerun_item = session.dynamic_rerun_items[0]
# first, check the sleep times
sleep_times_for_item = rerun_item.dynamic_rerun_sleep_times
assert len(sleep_times_for_item) == {0}
for sleep_time in sleep_times_for_item:
assert sleep_time.days == 0
assert sleep_time.seconds >= 0
assert sleep_time.microseconds
# Then, the triggers, schedule, and rerun attempts
assert rerun_item.dynamic_rerun_triggers == ["{1}"]
assert rerun_item.dynamic_rerun_schedule == "{2}"
assert rerun_item.max_allowed_dynamic_rerun_attempts == {0}
""".format(attempts, rerun_triggers, rerun_schedule))
testdir.makeini("""
[pytest]
dynamic_rerun_attempts = 2
dynamic_rerun_schedule = * * * * *
dynamic_rerun_triggers = blah
""")
result = testdir.runpytest("-v")
assert result.ret == pytest.ExitCode.TESTS_FAILED
_assert_result_outcomes(
result,
dynamic_rerun=attempts,
failed=failed_amount,
)
def test_dynamic_rerun_disabled_works_for_true_values(testdir,
dynamic_rerun_disabled):
testdir.makeini("""
[pytest]
dynamic_rerun_attempts = 3
dynamic_rerun_disabled = {}
dynamic_rerun_schedule = * * * * * *
""".format(dynamic_rerun_disabled))
testdir.makepyfile("def test_always_false(): assert False")
result = testdir.runpytest("-v")
assert result.ret == pytest.ExitCode.TESTS_FAILED
_assert_result_outcomes(result, dynamic_rerun=0, failed=1)
def test_positive_integer_dynamic_rerun_attempts_accepted(testdir, rerun_amount):
testdir.makeini(
"""
[pytest]
dynamic_rerun_schedule = * * * * * *
dynamic_rerun_attempts = {}
""".format(
rerun_amount
)
)
testdir.makepyfile("def test_always_false(): assert False")
result = testdir.runpytest("-v")
assert result.ret == pytest.ExitCode.TESTS_FAILED
failed_amount = 1
_assert_result_outcomes(result, dynamic_rerun=rerun_amount, failed=failed_amount)
def test_dynamic_rerun_disabled_false_by_default(testdir):
dynamic_rerun_attempts = 3
testdir.makeini("""
[pytest]
dynamic_rerun_attempts = {}
dynamic_rerun_schedule = * * * * * *
""".format(dynamic_rerun_attempts))
testdir.makepyfile("def test_always_false(): assert False")
result = testdir.runpytest("-v")
assert result.ret == pytest.ExitCode.TESTS_FAILED
_assert_result_outcomes(result,
dynamic_rerun=dynamic_rerun_attempts,
failed=1)
def test_output_properly_shown(testdir, ini_text, test_body,
would_normally_pass):
dynamic_rerun_attempts = 3
failed_amount = 1
testdir.makeini(ini_text.format(dynamic_rerun_attempts))
test_file_name = "test_output_properly_shown.py"
test_name = "test_output"
testdir.makepyfile("def {}(): {}".format(test_name, test_body))
expected_output = []
expected_output.append("=* test session starts *=")
for rerun_attempt in range(dynamic_rerun_attempts):
expected_output.append("*{}::{} DYNAMIC_RERUN*".format(
test_file_name, test_name))
expected_output.append("*{}::{} FAILED*".format(test_file_name, test_name))
expected_output.append("=* FAILURES *=")
expected_output.append("_* {} *_".format(test_name))
expected_output.append("=* Dynamically rerun tests *=")
for rerun_attempt in range(dynamic_rerun_attempts):
expected_output.append("*{}::{}".format(test_file_name, test_name))
expected_output.append("=* short test summary info *=")
if would_normally_pass:
expected_output.append("FAILED {}::{}*".format(test_file_name,
test_name))
else:
expected_output.append("FAILED {}::{} - {}*".format(
test_file_name, test_name, test_body))
expected_output.append("=*{} failed, {} dynamicrerun in *s *=".format(
failed_amount, dynamic_rerun_attempts))
result = testdir.runpytest("-v")
result.stdout.fnmatch_lines(expected_output)
assert result.ret == pytest.ExitCode.TESTS_FAILED
_assert_result_outcomes(
result,
dynamic_rerun=dynamic_rerun_attempts,
failed=failed_amount,
)
def test_invalid_dynamic_rerun_schedule_ignored(testdir, rerun_schedule):
rerun_amount = 2
failed_amount = 1
passed_amount = 0
testdir.makeini(
"""
[pytest]
dynamic_rerun_attempts = {}
dynamic_rerun_schedule = {}
""".format(
rerun_amount, rerun_schedule
)
)
testdir.makeconftest(
"""
def pytest_sessionfinish(session, exitstatus):
# A little hacky, but we know we can only ever have 1 item
sleep_times_for_item = session.dynamic_rerun_items[0].dynamic_rerun_sleep_times
assert len(sleep_times_for_item) == {}
for sleep_time in sleep_times_for_item:
assert sleep_time.days == 0
assert sleep_time.seconds >= 0
assert sleep_time.microseconds
""".format(
rerun_amount
)
)
testdir.makepyfile("def test_always_false(): assert False")
result = testdir.runpytest("-v")
result.stdout.fnmatch_lines(
[
"*Can't parse invalid dynamic rerun schedule '{}'. "
"Ignoring dynamic rerun schedule and using default '* * * * * *'*".format(
rerun_schedule
)
]
)
assert result.ret == pytest.ExitCode.TESTS_FAILED
_assert_result_outcomes(
result, dynamic_rerun=rerun_amount, failed=failed_amount, passed=passed_amount,
)
def test_non_positive_integer_rerun_attempts_rejected(testdir, rerun_amount):
testdir.makeini(
"""
[pytest]
dynamic_rerun_schedule = * * * * * *
dynamic_rerun_attempts = {}
""".format(
rerun_amount
)
)
testdir.makepyfile("def test_always_false(): assert False")
result = testdir.runpytest("-v")
result.stdout.fnmatch_lines(
["*Rerun attempts must be a positive integer. Using default value '1'*"]
)
assert result.ret == pytest.ExitCode.TESTS_FAILED
_assert_result_outcomes(result, dynamic_rerun=1, failed=1)
def test_stdout_checked_by_dynamic_rerun_triggers(testdir, rerun_trigger_text):
rerun_amount = 3
testdir.makeini("""
[pytest]
dynamic_rerun_schedule = * * * * * *
dynamic_rerun_attempts = {}
dynamic_rerun_triggers = {}
""".format(rerun_amount, rerun_trigger_text))
testdir.makepyfile(
"def test_all_seems_well_but(): print('Please rerun me')")
result = testdir.runpytest("-v")
assert result.ret == pytest.ExitCode.TESTS_FAILED
failed_amount = 1
_assert_result_outcomes(result,
dynamic_rerun=rerun_amount,
failed=failed_amount)
def test_exceptions_output_checked_by_dynamic_rerun_triggers(
testdir, rerun_trigger_text):
rerun_amount = 3
testdir.makeini("""
[pytest]
dynamic_rerun_schedule = * * * * * *
dynamic_rerun_attempts = {}
dynamic_rerun_triggers = {}
""".format(rerun_amount, rerun_trigger_text))
testdir.makepyfile(
"def test_value_error(): raise ValueError('A value error')")
result = testdir.runpytest("-v")
assert result.ret == pytest.ExitCode.TESTS_FAILED
failed_amount = 1
_assert_result_outcomes(result,
dynamic_rerun=rerun_amount,
failed=failed_amount)
def test_plugin_options_are_ini_configurable(testdir, ini_key_name,
ini_key_set_value,
ini_key_fetch_value):
testdir.makeini("""
[pytest]
{} = {}
""".format(ini_key_name, ini_key_set_value))
testdir.makepyfile("""
import pytest
@pytest.fixture
def fetch_ini_key(request):
return request.config.getini('{}')
def test_ini_key_fetch(fetch_ini_key):
assert fetch_ini_key == {}
""".format(ini_key_name, ini_key_fetch_value))
result = testdir.runpytest("-v")
result.stdout.fnmatch_lines(["*::test_ini_key_fetch PASSED*"])
assert result.ret == 0
_assert_result_outcomes(result, passed=1)
def test_plugin_marker_arguments_are_recognized(testdir, plugin_disabled):
requested_rerun_attempts = 3
rerun_schedule = "* * * * * *"
rerun_triggers = "foo"
dynamic_rerun_attempts = 3
failed_amount = 1
passed_amount = 1
pytest_exit_status = pytest.ExitCode.TESTS_FAILED
if plugin_disabled:
dynamic_rerun_attempts = 0
failed_amount = 0
passed_amount = 2
pytest_exit_status = pytest.ExitCode.OK
testdir.makepyfile("""
import pytest
@pytest.mark.dynamicrerun(attempts={0}, disabled={1}, schedule="{2}", triggers="{3}")
def test_prints_foo():
print("foo")
@pytest.mark.dynamicrerun(attempts={0}, disabled={1}, schedule="{2}", triggers="{3}")
def test_prints_bar():
print("bar")
""".format(requested_rerun_attempts, plugin_disabled, rerun_schedule,
rerun_triggers))
result = testdir.runpytest("-v")
assert result.ret == pytest_exit_status
_assert_result_outcomes(
result,
dynamic_rerun=dynamic_rerun_attempts,
failed=failed_amount,
passed=passed_amount,
)
def test_plugin_flags_are_recognized(testdir, plugin_disabled):
requested_rerun_attempts = 3
dynamic_rerun_attempts = 3
failed_amount = 1
passed_amount = 1
pytest_exit_status = pytest.ExitCode.TESTS_FAILED
if plugin_disabled:
dynamic_rerun_attempts = 0
failed_amount = 0
passed_amount = 2
pytest_exit_status = pytest.ExitCode.OK
testdir.makepyfile("""
def test_prints_foo():
print("foo")
def test_prints_bar():
print("bar")
""")
result = testdir.runpytest(
"-v",
"--dynamic-rerun-attempts={}".format(requested_rerun_attempts),
"--dynamic-rerun-disabled={}".format(plugin_disabled),
"--dynamic-rerun-schedule='* * * * * *'",
"--dynamic-rerun-triggers=foo",
)
assert result.ret == pytest_exit_status
_assert_result_outcomes(
result,
dynamic_rerun=dynamic_rerun_attempts,
failed=failed_amount,
passed=passed_amount,
)
def test_can_handle_multiple_dynamic_rerun_triggers(testdir, print_output,
should_rerun,
parameter_pass_level):
rerun_amount = 3
rerun_triggers = [
"Rerun on this text",
"Also rerun on this one",
"Finally also check this one",
]
if parameter_pass_level == ParameterPassLevel.FLAG:
testdir.makepyfile(
"def test_all_seems_well_but(): print('{}')".format(print_output))
testdir.makeini("""
[pytest]
dynamic_rerun_attempts = {}
dynamic_rerun_schedule = * * * * * *
""".format(rerun_amount))
# NOTE: We append the flag and its argument seperately since testidr will wrap
# each passed argument in a string. Thus, pasing --foo="bar baz" really results
# in passing '--foo="bar baz"' which is not equivalent.
flags = ["-v"]
trigger_flag = "--dynamic-rerun-triggers"
for trigger in rerun_triggers:
flags.append(trigger_flag)
flags.append(trigger)
result = testdir.runpytest(*flags)
elif parameter_pass_level == ParameterPassLevel.INI_KEY:
testdir.makepyfile(
"def test_all_seems_well_but(): print('{}')".format(print_output))
testdir.makeini("""
[pytest]
dynamic_rerun_attempts = {}
dynamic_rerun_schedule = * * * * * *
dynamic_rerun_triggers = {}
""".format(rerun_amount, "\n\t".join(rerun_triggers)))
result = testdir.runpytest("-v")
else: # ParameterPassLevel.MARKER
formatted_rerun_triggers = [
'"{}"'.format(trigger) for trigger in rerun_triggers
]
testdir.makepyfile("""
import pytest
@pytest.mark.dynamicrerun(triggers=[{}])
def test_all_seems_well_but():
print('{}')
""".format(",".join(formatted_rerun_triggers), print_output))
testdir.makeini("""
[pytest]
dynamic_rerun_attempts = {}
dynamic_rerun_schedule = * * * * * *
""".format(rerun_amount))
result = testdir.runpytest("-v")
if should_rerun:
failed_amount = 1
dynamic_rerun_amount = rerun_amount
passed_amount = 0
assert result.ret == pytest.ExitCode.TESTS_FAILED
else:
failed_amount = 0
dynamic_rerun_amount = 0
passed_amount = 1
assert result.ret == pytest.ExitCode.OK
_assert_result_outcomes(
result,
dynamic_rerun=dynamic_rerun_amount,
failed=failed_amount,
passed=passed_amount,
)
def test_dynamic_reruns_batched_by_rerun_time(testdir):
testdir.makepyfile("""
import pytest
@pytest.mark.dynamicrerun(attempts=3, triggers="foo", schedule="* * * * * *")
def test_dynamically_rerun_every_second_a():
print("foo")
@pytest.mark.dynamicrerun(attempts=4, triggers="foo", schedule="* * * * * *")
def test_dynamically_rerun_every_second_b():
print("foo")
@pytest.mark.dynamicrerun(attempts=11, triggers="foo", schedule="* * * * * *")
def test_dynamically_rerun_every_second_c():
print("foo")
@pytest.mark.dynamicrerun(attempts=2, triggers="foo", schedule="* * * * * */5")
def test_dynamically_rerun_every_five_seconds_a():
print("foo")
@pytest.mark.dynamicrerun(attempts=3, triggers="foo", schedule="* * * * * */5")
def test_dynamically_rerun_every_five_seconds_b():
print("foo")
""")
expected_stdout = [
"*test_dynamic_reruns_batched_by_rerun_time.py::test_dynamically_rerun_every_second_a DYNAMIC_RERUN*",
"*test_dynamic_reruns_batched_by_rerun_time.py::test_dynamically_rerun_every_second_b DYNAMIC_RERUN*",
"*test_dynamic_reruns_batched_by_rerun_time.py::test_dynamically_rerun_every_second_c DYNAMIC_RERUN*",
"*test_dynamic_reruns_batched_by_rerun_time.py::test_dynamically_rerun_every_five_seconds_a DYNAMIC_RERUN*",
"*test_dynamic_reruns_batched_by_rerun_time.py::test_dynamically_rerun_every_five_seconds_b DYNAMIC_RERUN*",
"*test_dynamic_reruns_batched_by_rerun_time.py::test_dynamically_rerun_every_second_a DYNAMIC_RERUN*",
"*test_dynamic_reruns_batched_by_rerun_time.py::test_dynamically_rerun_every_second_b DYNAMIC_RERUN*",
"*test_dynamic_reruns_batched_by_rerun_time.py::test_dynamically_rerun_every_second_c DYNAMIC_RERUN*",
"*test_dynamic_reruns_batched_by_rerun_time.py::test_dynamically_rerun_every_second_a DYNAMIC_RERUN*",
"*test_dynamic_reruns_batched_by_rerun_time.py::test_dynamically_rerun_every_second_b DYNAMIC_RERUN*",
"*test_dynamic_reruns_batched_by_rerun_time.py::test_dynamically_rerun_every_second_c DYNAMIC_RERUN*",
"*test_dynamic_reruns_batched_by_rerun_time.py::test_dynamically_rerun_every_second_a FAILED*",
"*test_dynamic_reruns_batched_by_rerun_time.py::test_dynamically_rerun_every_second_b DYNAMIC_RERUN*",
"*test_dynamic_reruns_batched_by_rerun_time.py::test_dynamically_rerun_every_second_c DYNAMIC_RERUN*",
"*test_dynamic_reruns_batched_by_rerun_time.py::test_dynamically_rerun_every_second_b FAILED*",
"*test_dynamic_reruns_batched_by_rerun_time.py::test_dynamically_rerun_every_second_c DYNAMIC_RERUN*",
"*test_dynamic_reruns_batched_by_rerun_time.py::test_dynamically_rerun_every_second_c DYNAMIC_RERUN*",
"*test_dynamic_reruns_batched_by_rerun_time.py::test_dynamically_rerun_every_five_seconds_a DYNAMIC_RERUN*",
"*test_dynamic_reruns_batched_by_rerun_time.py::test_dynamically_rerun_every_five_seconds_b DYNAMIC_RERUN*",
"*test_dynamic_reruns_batched_by_rerun_time.py::test_dynamically_rerun_every_second_c DYNAMIC_RERUN*",
"*test_dynamic_reruns_batched_by_rerun_time.py::test_dynamically_rerun_every_second_c DYNAMIC_RERUN*",
"*test_dynamic_reruns_batched_by_rerun_time.py::test_dynamically_rerun_every_second_c DYNAMIC_RERUN*",
"*test_dynamic_reruns_batched_by_rerun_time.py::test_dynamically_rerun_every_second_c DYNAMIC_RERUN*",
"*test_dynamic_reruns_batched_by_rerun_time.py::test_dynamically_rerun_every_second_c DYNAMIC_RERUN*",
"*test_dynamic_reruns_batched_by_rerun_time.py::test_dynamically_rerun_every_five_seconds_a FAILED*",
"*test_dynamic_reruns_batched_by_rerun_time.py::test_dynamically_rerun_every_five_seconds_b DYNAMIC_RERUN*",
"*test_dynamic_reruns_batched_by_rerun_time.py::test_dynamically_rerun_every_second_c FAILED*",
"*test_dynamic_reruns_batched_by_rerun_time.py::test_dynamically_rerun_every_five_seconds_b FAILED*",
]
# wait until seconds are at a *0 value for consistency ( 00, 10, 20, 30, ..., etc )
# this isn't exactly perfect since microseconds will be non zero, but it's close enough
current_time = datetime.now()
sleep_time_microseconds = 1 - (current_time.microsecond / 1000000)
sleep_time_seconds = 10 - (current_time.second % 10)
if sleep_time_microseconds != 0:
sleep_time_seconds -= 1
sleep_time = sleep_time_seconds + sleep_time_microseconds
time.sleep(sleep_time)
result = testdir.runpytest("-v")
assert result.ret == pytest.ExitCode.TESTS_FAILED
result.stdout.fnmatch_lines(expected_stdout)
_assert_result_outcomes(result, dynamic_rerun=23, failed=5)