def test_missing(self):
out, err, return_code = RunTest(
[
'--clusters', '10',
'--jobs', '1',
'--result', self.filename,
os.path.join(
ROOT_DIR, 'tests', 'gtest_fake', 'gtest_fake_missing.py'),
self.tempdirpath,
])
self.assertEqual(1, return_code)
expected_out_re = [
r'\[0\/3\] \d\.\d\ds ',
r'\[1\/4\] \d\.\d\ds Foo\.Bar1 \(\d+\.\d+s\)',
# Only the header is included since the test was not run alone.
re.escape('Note: Google Test filter = Foo.Bar3:Foo.Bar2:Foo.Bar1'),
'',
re.escape('[==========] Running 1 test from 1 test case.'),
re.escape('[----------] Global test environment set-up.'),
re.escape('[ RUN ] Foo.Bar1'),
re.escape('[ FAILED ] Foo.Bar1 (100 ms)'),
'',
r'\[2/4\] \d\.\d\ds Foo\.Bar3 \(\d+\.\d+s\) *',
r'\[3/5\] \d\.\d\ds Foo\.Bar2 \<unknown\> *',
r'\[4/5\] \d\.\d\ds Foo\.Bar1 \(\d+\.\d+s\) \- retry \#1',
# Both the header and footer is included since the test was run alone.
] + get_whole_test_re('Foo.Bar1', False, '100', 'Foo.Bar1') + [
r'\[5/6\] \d\.\d\ds Foo\.Bar2 \<unknown\> \- retry \#1 *',
] + [
re.escape(l) for l in run_test_cases.running_serial_warning()
] + [
r'\[6/6\] \d\.\d\ds Foo\.Bar2 \<unknown\> \- retry \#2 *',
re.escape('Flaky tests:'),
re.escape(' Foo.Bar1 (tried 2 times)'),
re.escape('Failed tests:'),
re.escape(' Foo.Bar2'),
re.escape('Summary:'),
re.escape(' Success: 1 33.33% ') + r' +\d+\.\d\ds',
re.escape(' Flaky: 1 33.33% ') + r' +\d+\.\d\ds',
re.escape(' Fail: 1 33.33% ') + r' +\d+\.\d\ds',
r' \d+\.\d\ds Done running 3 tests with 6 executions. '
'\d+\.\d\d test/s',
]
self._check_results(expected_out_re, out, err)
test_cases = [
('Foo.Bar1', 2),
('Foo.Bar2', 3),
('Foo.Bar3', 1)
]
self._check_results_file(
fail=[u'Foo.Bar2'],
flaky=[u'Foo.Bar1'],
missing=[],
success=[u'Foo.Bar3'],
test_cases=test_cases,
duration=True)
def test_simple_fail(self):
out, err, return_code = RunTest(
[
# Linearize execution.
'--clusters', '1',
'--jobs', '1',
'--result', self.filename,
os.path.join(ROOT_DIR, 'tests', 'gtest_fake', 'gtest_fake_fail.py'),
])
self.assertEqual(1, return_code)
test_failure_output = get_whole_test_re('Baz.Fail', True, '100', 'Baz.Fail')
expected_out_re = [
r'\[0/\d\] \d\.\d\ds ',
r'\[1/\d\] \d\.\d\ds .+',
r'\[2/\d\] \d\.\d\ds .+',
r'\[3/\d\] \d\.\d\ds .+',
r'\[4/\d\] \d\.\d\ds .+',
] + test_failure_output + [
# Retries
r'\[5/\d\] \d\.\d\ds .+ retry \#1',
] + test_failure_output + [
re.escape(l) for l in run_test_cases.running_serial_warning()
] + [
r'\[6/\d\] \d\.\d\ds .+ retry \#2',
] + test_failure_output + [
re.escape('Failed tests:'),
re.escape(' Baz.Fail'),
re.escape('Summary:'),
re.escape(' Success: 3 75.00%') + r' +\d+\.\d\ds',
re.escape(' Flaky: 0 0.00%') + r' +\d+\.\d\ds',
re.escape(' Fail: 1 25.00%') + r' +\d+\.\d\ds',
r' \d+\.\d\ds Done running 4 tests with 6 executions. \d+\.\d\d test/s',
]
self._check_results(expected_out_re, out, err)
test_cases = [
('Foo.Bar1', 1),
('Foo.Bar2', 1),
('Foo.Bar3', 1),
('Baz.Fail', 3)
]
self._check_results_file(
fail=['Baz.Fail'],
flaky=[],
missing=[],
success=[u'Foo.Bar1', u'Foo.Bar2', u'Foo.Bar3'],
test_cases=test_cases,
duration=True)
def test_confused_fail(self):
# The test case prints that it failed but the process exit code is 0.
out, err, return_code = RunTest(
[
'--result', self.filename,
os.path.join(
ROOT_DIR, 'tests', 'gtest_fake', 'gtest_fake_confused_fail.py'),
])
self.assertEqual(1, return_code)
expected_out_re = [
r'\[0\/1\] \d\.\d\ds ',
r'\[1\/2\] \d\.\d\ds Foo\.Bar1 \(\d+\.\d\ds\)',
] + get_whole_test_re('Foo.Bar1', True, '100', 'Foo.Bar1') + [
r'\[2\/3\] \d\.\d\ds Foo\.Bar1 \(\d+\.\d\ds\) - retry #1',
] + get_whole_test_re('Foo.Bar1', True, '100', 'Foo.Bar1') + [
] + [
re.escape(l) for l in run_test_cases.running_serial_warning()
] + [
r'\[3\/3\] \d\.\d\ds Foo\.Bar1 \(\d+\.\d\ds\) - retry #2',
] + get_whole_test_re('Foo.Bar1', True, '100', 'Foo.Bar1') + [
re.escape('Failed tests:'),
re.escape(' Foo.Bar1'),
re.escape('Summary:'),
re.escape(' Success: 0 0.00% 0.00s'),
re.escape(' Flaky: 0 0.00% 0.00s'),
re.escape(' Fail: 1 100.00% ') + r' +\d+\.\d\ds',
r' \d+\.\d\ds Done running 1 tests with 3 executions. \d+\.\d\d test/s',
]
self._check_results(expected_out_re, out, err)
test_cases = [
('Foo.Bar1', 3),
]
self._check_results_file(
fail=[u'Foo.Bar1'],
flaky=[],
missing=[],
success=[],
test_cases=test_cases,
duration=True)
def test_simple_fail_verbose(self):
# We take verbosity seriously so test it.
out, err, return_code = RunTest(
[
# Linearize execution.
'--clusters', '1',
'--jobs', '1',
'--verbose',
'--result', self.filename,
os.path.join(ROOT_DIR, 'tests', 'gtest_fake', 'gtest_fake_fail.py'),
])
self.assertEqual(1, return_code)
expected_out_re = [
r'\[0/\d\] \d\.\d\ds '
]
test_cases = (
'Foo.Bar3',
'Foo.Bar1',
'Foo.Bar2',
'Baz.Fail',
'Baz.Fail',
'Baz.Fail',
)
for index, name in enumerate(test_cases):
if index + 1 == len(test_cases):
# We are about to retry the test serially, so check for the warning.
expected_out_re.extend(
re.escape(l) for l in run_test_cases.running_serial_warning())
expected_out_re.append(
r'\[%d/\d\] \d\.\d\ds ' % (index + 1) + re.escape(name) + ' .+')
expected_out_re.extend(
get_whole_test_re(name, 'Fail' in name, '100', name))
expected_out_re.extend([
re.escape('Failed tests:'),
re.escape(' Baz.Fail'),
re.escape('Summary:'),
re.escape(' Success: 3 75.00%') + r' +\d+\.\d\ds',
re.escape(' Flaky: 0 0.00%') + r' +\d+\.\d\ds',
re.escape(' Fail: 1 25.00%') + r' +\d+\.\d\ds',
r' \d+\.\d\ds Done running 4 tests with 6 executions. \d+\.\d\d test/s',
])
self._check_results(expected_out_re, out, '')
# Test 'err' manually.
self.assertTrue(
re.match(
r'INFO run_test_cases\(\d+\)\: Found 4 test cases in \S+ '
r'\S+gtest_fake_fail.py',
err.strip()),
err)
test_cases = [
('Foo.Bar1', 1),
('Foo.Bar2', 1),
('Foo.Bar3', 1),
('Baz.Fail', 3)
]
self._check_results_file(
fail=['Baz.Fail'],
flaky=[],
missing=[],
success=[u'Foo.Bar1', u'Foo.Bar2', u'Foo.Bar3'],
test_cases=test_cases,
duration=True)
