def test_xdist_all_tests_failed_with_stats(testdir):
artifact_path = testdir.tmpdir.strpath + '/artifact.json'
make_simple_pytest_suite(testdir,
total_failures=2,
expected_reruns=1,
has_failure=True)
result = testdir.runpytest(
'--reruns',
'1',
'-r',
'R',
'-n',
'2',
'--reruns-artifact-path',
artifact_path,
)
assert_outcomes(result, passed=2, rerun=2, failed=2)
with open(artifact_path) as artifact:
artifact_data = json.load(artifact)
assert artifact_data['total_reruns'] == 2
assert artifact_data['total_failed'] == 2
assert artifact_data['total_resolved_by_reruns'] == 0
assert artifact_data['rerun_tests'][0]['status'] == 'failed'
assert artifact_data['rerun_tests'][1]['status'] == 'failed'
assert set([
artifact_data['rerun_tests'][0]['nodeid'],
artifact_data['rerun_tests'][1]['nodeid']
]) == set([
'test_xdist_all_tests_failed_with_stats.py::test_test_failing_1',
'test_xdist_all_tests_failed_with_stats.py::test_test_failing_0'
])
def test_xdist_all_tests_failed_with_junit(testdir):
artifact_path = testdir.tmpdir.strpath + '/artifact.xml'
make_simple_pytest_suite(testdir,
total_failures=2,
expected_reruns=1,
has_failure=True)
result = testdir.runpytest(
'--reruns',
'1',
'-r',
'R',
'-n',
'2',
'--junitxml',
artifact_path,
)
assert_outcomes(result, passed=2, rerun=2, failed=2)
with open(artifact_path) as artifact:
artifact_data = xmltodict.parse(artifact.read())
if artifact_data.get('testsuites'):
artifact_data = artifact_data['testsuites']
assert artifact_data['testsuite']['@errors'] == '0'
assert artifact_data['testsuite']['@failures'] == '2'
assert artifact_data['testsuite']['@tests'] == '4'
assert len(artifact_data['testsuite']['testcase']) == 4
assert len([
t for t in artifact_data['testsuite']['testcase'] if 'failure' in t
]) == 2
def test_max_reruns_reached(testdir):
artifact_path = testdir.tmpdir.strpath + '/artifact.json'
make_simple_pytest_suite(testdir,
total_failures=2,
expected_reruns=1,
has_failure=True)
result = testdir.runpytest('--reruns', '1', '-r', 'R',
'--reruns-artifact-path', artifact_path,
'--max-tests-rerun', '1')
assert_outcomes(result, passed=2, rerun=0, failed=2)
def test_reruns_stats_max_reruns_reached(testdir):
artifact_path = testdir.tmpdir.strpath + '/artifact.json'
make_simple_pytest_suite(testdir,
total_failures=2,
expected_reruns=1,
has_failure=True)
result = testdir.runpytest('--reruns', '1', '-r', 'R',
'--reruns-artifact-path', artifact_path,
'--max-tests-rerun', '1')
assert_outcomes(result, passed=2, rerun=0, failed=2)
with open(artifact_path) as artifact:
artifact_data = json.load(artifact)
assert artifact_data == {
'total_reruns': 0,
'total_failed': 0,
'total_resolved_by_reruns': 0,
'rerun_tests': []
}
def test_reruns_junit_max_reruns_reached(testdir):
artifact_path = testdir.tmpdir.strpath + '/artifact.xml'
make_simple_pytest_suite(testdir,
total_failures=2,
expected_reruns=1,
has_failure=True)
result = testdir.runpytest('--reruns', '1', '-r', 'R', '--junitxml',
artifact_path, '--max-tests-rerun', '1')
assert_outcomes(result, passed=2, rerun=0, failed=2)
with open(artifact_path) as artifact:
artifact_data = xmltodict.parse(artifact.read())
if artifact_data.get('testsuites'):
artifact_data = artifact_data['testsuites']
assert artifact_data['testsuite']['@errors'] == '0'
assert artifact_data['testsuite']['@failures'] == '2'
assert artifact_data['testsuite']['@tests'] == '4'
assert len(artifact_data['testsuite']['testcase']) == 4
assert artifact_data['testsuite']['testcase'][2]['failure']
assert artifact_data['testsuite']['testcase'][3]['failure']
def test_reruns_stats_all_tests_passed(testdir):
artifact_path = testdir.tmpdir.strpath + '/artifact.json'
make_simple_pytest_suite(testdir, expected_reruns=0, has_failure=False)
result = testdir.runpytest(
'--reruns',
'1',
'-r',
'R',
'--reruns-artifact-path',
artifact_path,
)
assert_outcomes(result, passed=3, rerun=0)
with open(artifact_path) as artifact:
artifact_data = json.load(artifact)
assert artifact_data == {
'total_reruns': 0,
'total_failed': 0,
'total_resolved_by_reruns': 0,
'rerun_tests': []
}
def test_reruns_junit_all_tests_passed(testdir):
artifact_path = testdir.tmpdir.strpath + '/artifact.xml'
make_simple_pytest_suite(testdir, expected_reruns=0, has_failure=False)
result = testdir.runpytest(
'--reruns',
'1',
'-r',
'R',
'--junitxml',
artifact_path,
)
assert_outcomes(result, passed=3, rerun=0)
with open(artifact_path) as artifact:
artifact_data = xmltodict.parse(artifact.read())
if artifact_data.get('testsuites'):
artifact_data = artifact_data['testsuites']
assert artifact_data['testsuite']['@errors'] == '0'
assert artifact_data['testsuite']['@failures'] == '0'
assert artifact_data['testsuite']['@tests'] == '3'
assert len(artifact_data['testsuite']['testcase']) == 3
def test_xdist_worker_rerun_stats(testdir):
artifact_path = testdir.tmpdir.strpath + '/artifact.json'
make_simple_pytest_suite(testdir,
total_failures=2,
expected_reruns=1,
has_failure=False)
result = testdir.runpytest(
'--reruns',
'1',
'-r',
'R',
'-n',
'2',
'--dist',
'loadfile',
'--reruns-artifact-path',
artifact_path,
'--xdist-worker-reruns-artifact',
)
assert_outcomes(result, passed=4, rerun=2)
if os.path.isfile(testdir.tmpdir.strpath + '/gw0_artifact.json'):
xdist_artifact_path = testdir.tmpdir.strpath + '/gw0_artifact.json'
else:
xdist_artifact_path = testdir.tmpdir.strpath + '/gw1_artifact.json'
with open(xdist_artifact_path) as artifact:
artifact_data = json.load(artifact)
assert artifact_data['total_reruns'] == 2
assert artifact_data['total_failed'] == 2
assert artifact_data['total_resolved_by_reruns'] == 2
assert artifact_data['rerun_tests'][0]['status'] == 'flake'
assert artifact_data['rerun_tests'][1]['status'] == 'flake'
assert set([
artifact_data['rerun_tests'][0]['nodeid'],
artifact_data['rerun_tests'][1]['nodeid']
]) == set([
'test_xdist_worker_rerun_stats.py::test_test_failing_1',
'test_xdist_worker_rerun_stats.py::test_test_failing_0'
])
def test_reruns_stats_2_tests_failed(testdir):
artifact_path = testdir.tmpdir.strpath + '/artifact.json'
make_simple_pytest_suite(testdir,
total_failures=2,
expected_reruns=1,
has_failure=True)
result = testdir.runpytest(
'--reruns',
'1',
'-r',
'R',
'--reruns-artifact-path',
artifact_path,
)
assert_outcomes(result, passed=2, rerun=2, failed=2)
with open(artifact_path) as artifact:
artifact_data = json.load(artifact)
assert artifact_data['total_reruns'] == 2
assert artifact_data['total_failed'] == 2
assert artifact_data['total_resolved_by_reruns'] == 0
assert len(artifact_data['rerun_tests']) == 2
assert artifact_data['rerun_tests'][0]['status'] == 'failed'
assert artifact_data['rerun_tests'][1]['status'] == 'failed'
def test_reruns_stats_all_tests_resolved(testdir):
artifact_path = testdir.tmpdir.strpath + '/artifact.json'
make_simple_pytest_suite(testdir, expected_reruns=1, has_failure=False)
result = testdir.runpytest(
'--reruns',
'1',
'-r',
'R',
'--reruns-artifact-path',
artifact_path,
)
assert_outcomes(result, passed=3, rerun=1)
with open(artifact_path) as artifact:
artifact_data = json.load(artifact)
assert artifact_data == {
'total_reruns':
1,
'total_failed':
1,
'total_resolved_by_reruns':
1,
'rerun_tests': [
{
'nodeid':
'test_reruns_stats_all_tests_resolved.py::test_test_failing_0',
'status': 'flake',
'rerun_trace': {
'teardown': {
'caplog': '',
'capstderr': '',
'capstdout':
'session_fixture_2 setup\nsession_fixture_1 setup\nsession_fixture_2 setup\nsession_fixture_2 teardown\nsession_fixture_1 teardown\n',
'text_repr': ''
},
'setup': {
'caplog': '',
'capstderr': '',
'capstdout':
'session_fixture_2 setup\nsession_fixture_1 setup\nsession_fixture_2 setup\n',
'text_repr': ''
},
'call': {
'caplog': '',
'capstderr': '',
'capstdout':
'session_fixture_2 setup\nsession_fixture_1 setup\nsession_fixture_2 setup\n',
'text_repr': ''
}
},
'original_trace': {
'call': {
'caplog':
'',
'capstderr':
'',
'capstdout':
'session_fixture_2 setup\n',
'text_repr':
'session_fixture_2 = None\n\n def test_test_failing_0(session_fixture_2):\n global number_0\n number_0 += 1\n> assert number_0 == 1 + 1\nE assert 1 == (1 + 1)\n\ntest_reruns_stats_all_tests_resolved.py:60: AssertionError',
},
'setup': {
'caplog': '',
'capstderr': '',
'capstdout': 'session_fixture_2 setup\n',
'text_repr': ''
},
'teardown': {
'caplog': '',
'capstderr': '',
'capstdout': 'session_fixture_2 setup\n',
'text_repr': ''
}
}
},
]
}