def test_flake_restart_pass(self):
expectations = {'flake': flags.FlagSet(flags.FLAKY)}
sb = scoreboard.Scoreboard('suite', expectations)
tests = ['flake']
self._register_tests(sb, tests)
sb.start(tests)
# Fail the test the first time.
actuals = {'flake': test_method_result.TestMethodResult.FAIL}
self._update_tests(sb, actuals)
results = {
'total': 1,
'completed': 1,
'get_flaky_tests': ['flake'],
'overall_status': scoreboard_constants.EXPECTED_PASS,
}
self._check_scoreboard(sb, results)
# Restart the tests.
sb.restart(len(tests))
sb.start(tests)
# Pass the test the second time.
actuals = {'flake': test_method_result.TestMethodResult.PASS}
self._update_tests(sb, actuals)
results = {
'total': 2,
'restarts': 1,
'completed': 2,
'passed': 1,
'expected_passed': 1,
'get_expected_passing_tests': ['flake'],
'overall_status': scoreboard_constants.EXPECTED_PASS,
}
self._check_scoreboard(sb, results)
# Verify finalized results.
sb.finalize()
results = {
'total': 2,
'restarts': 1,
'completed': 2,
'passed': 1,
'expected_passed': 1,
'get_expected_passing_tests': ['flake'],
'overall_status': scoreboard_constants.EXPECTED_PASS,
}
self._check_scoreboard(sb, results)
# Note: the number of completed tests cannot be determined by simply
# adding up the individual results. This is because flaky tests that
# failed are neither passing nor failing nor skipped (but they most
# definitely were completed).
self.assertNotEqual(sb.completed, sb.passed + sb.failed + sb.skipped)
def _run(self, suite_name, tests, results):
expectations = dict((t.name, t.expectation) for t in tests)
register_tests = [t.name for t in tests]
start_tests = [t.name for t in tests if t.result]
actuals = dict((t.name, t.result) for t in tests if t.result)
sb = scoreboard.Scoreboard(suite_name, expectations)
self._register_tests(sb, register_tests)
sb.start(start_tests)
self._update_tests(sb, actuals)
sb.finalize()
self._check_scoreboard(sb, results)
def test_flake_incomplete(self):
expectations = {'flake': flags.FlagSet(flags.FLAKY)}
sb = scoreboard.Scoreboard('suite', expectations)
tests = ['flake']
self._register_tests(sb, tests)
sb.start(tests)
# The test is never run.
self.assertEquals(self._test_counter[scoreboard_constants.INCOMPLETE],
0)
sb.finalize()
self.assertEquals(self._test_counter[scoreboard_constants.INCOMPLETE],
1)
def test_no_test_reported(self):
results = {
'total': 0,
'incompleted': 0,
'overall_status': scoreboard_constants.EXPECTED_PASS,
}
self._expected_test_count = 0
expectations = {}
start_tests = []
sb = scoreboard.Scoreboard('suite', expectations)
sb.start(start_tests)
sb.finalize()
self._check_scoreboard(sb, results)
def test_flake_restart_fail(self):
expectations = {'flake': flags.FlagSet(flags.FLAKY)}
sb = scoreboard.Scoreboard('suite', expectations)
tests = ['flake']
self._register_tests(sb, tests)
sb.start(tests)
# Fail the test the first time.
actuals = {'flake': test_method_result.TestMethodResult.FAIL}
self._update_tests(sb, actuals)
results = {
'total': 1,
'completed': 1,
'get_flaky_tests': ['flake'],
'overall_status': scoreboard_constants.EXPECTED_PASS,
}
self._check_scoreboard(sb, results)
# Restart the tests.
sb.restart(len(tests))
sb.start(tests)
# Pass the test the second time.
actuals = {'flake': test_method_result.TestMethodResult.FAIL}
self._update_tests(sb, actuals)
results = {
'total': 2,
'restarts': 1,
'completed': 2,
'get_flaky_tests': ['flake'],
'overall_status': scoreboard_constants.EXPECTED_PASS,
}
self._check_scoreboard(sb, results)
# Verify finalized results.
sb.finalize()
results = {
'total': 2,
'completed': 2,
'failed': 1,
'restarts': 1,
'unexpected_failed': 1,
'get_unexpected_failing_tests': ['flake'],
'overall_status': scoreboard_constants.UNEXPECTED_FAIL,
}
self._check_scoreboard(sb, results)
def test_get_expectations_works_with_named_tests(self):
sb = scoreboard.Scoreboard(
'suite', {
'testPasses': flags.FlagSet(flags.PASS),
'testFails': flags.FlagSet(flags.FAIL),
'testTimesOut': flags.FlagSet(flags.TIMEOUT),
'testFlaky': flags.FlagSet(flags.FLAKY),
})
expectations = sb.get_expectations()
self.assertEquals(4, len(expectations))
self.assertEquals(scoreboard_constants.EXPECTED_PASS,
expectations['testPasses'])
self.assertEquals(scoreboard_constants.EXPECTED_FAIL,
expectations['testFails'])
self.assertEquals(scoreboard_constants.SKIPPED,
expectations['testTimesOut'])
self.assertEquals(scoreboard_constants.EXPECTED_FLAKE,
expectations['testFlaky'])
def __init__(self, suite_runner, test_expectations, tests_to_run,
try_count, stop_on_unexpected_failures):
self._suite_runner = suite_runner
self._test_expectations = test_expectations.copy()
self._tests_to_run = suite_runner.apply_test_ordering(tests_to_run)
self._run_remaining_count = try_count if tests_to_run else 0
self._stop_on_unexpected_failures = stop_on_unexpected_failures
self._scoreboard = scoreboard.Scoreboard(suite_runner.name,
suite_runner.expectation_map)
# Mark planned tests INCOMPLETE to distinguish them from skipped tests.
self._scoreboard.reset_results(self._tests_to_run)
# Whether or not this test has been finalized.
# finalize() can be called on testing thread (in common case) or the main
# thread (on timeout), sometimes on both (edge case). So, mutex lock is
# needed.
self._finalized_lock = threading.Lock()
self._finalized = False
def test_blacklist(self):
expectations = {
'alpha': flags.FlagSet(flags.PASS),
'beta': flags.FlagSet(flags.PASS),
'gamma': flags.FlagSet(flags.PASS),
}
sb = scoreboard.Scoreboard('suite', expectations)
self._register_tests(sb, ['alpha', 'beta', 'gamma'])
# Run and pass just the first test.
sb.start(['alpha', 'beta', 'gamma'])
actuals = {'alpha': test_method_result.TestMethodResult.PASS}
self._update_tests(sb, actuals)
results = {
'total': 3,
'completed': 1,
'incompleted': 2,
'passed': 1,
'expected_passed': 1,
'get_expected_passing_tests': ['alpha'],
'get_incomplete_tests': ['beta', 'gamma'],
'overall_status': scoreboard_constants.INCOMPLETE,
}
self._check_scoreboard(sb, results)
# Restart.
sb.restart(2)
results = {
'total': 3,
'restarts': 1,
'completed': 1,
'incompleted': 2,
'passed': 1,
'expected_passed': 1,
'get_expected_passing_tests': ['alpha'],
'get_incomplete_tests': ['beta', 'gamma'],
'overall_status': scoreboard_constants.INCOMPLETE,
}
self._check_scoreboard(sb, results)
# Run the remaining two tests.
sb.start(['beta', 'gamma'])
actuals = {'beta': test_method_result.TestMethodResult.PASS}
self._update_tests(sb, actuals)
results = {
'total': 3,
'restarts': 1,
'completed': 2,
'incompleted': 1,
'passed': 2,
'expected_passed': 2,
'get_expected_passing_tests': ['alpha', 'beta'],
'get_incomplete_tests': ['gamma'],
'overall_status': scoreboard_constants.INCOMPLETE,
}
self._check_scoreboard(sb, results)
# After this restart, 'gamma' will have been incomplete twice, so
# it should get added to the blacklist.
sb.restart(1)
results = {
'total': 3,
'restarts': 2,
'completed': 2,
'incompleted': 1,
'passed': 2,
'expected_passed': 2,
'get_expected_passing_tests': ['alpha', 'beta'],
'get_incomplete_tests': ['gamma'],
'get_incomplete_blacklist': ['gamma'],
'overall_status': scoreboard_constants.INCOMPLETE,
}
self._check_scoreboard(sb, results)
# Finally run the last test. Now 'gamma' should no longer be in the
# blacklist since it ran.
sb.start(['gamma'])
actuals = {'gamma': test_method_result.TestMethodResult.PASS}
self._update_tests(sb, actuals)
results = {
'total': 3,
'completed': 3,
'passed': 3,
'expected_passed': 3,
'restarts': 2,
'get_expected_passing_tests': ['alpha', 'beta', 'gamma'],
'overall_status': scoreboard_constants.EXPECTED_PASS,
}
self._check_scoreboard(sb, results)
# Verify all results.
sb.finalize()
results = {
'total': 3,
'completed': 3,
'passed': 3,
'expected_passed': 3,
'restarts': 2,
'get_expected_passing_tests': ['alpha', 'beta', 'gamma'],
'overall_status': scoreboard_constants.EXPECTED_PASS,
}
self._check_scoreboard(sb, results)