def test_summarized_results_with_iterations(self):
test_name = 'passes/text.html'
expectations = test_expectations.TestExpectations(self.port)
initial_results = test_run_results.TestRunResults(
expectations, 3, None)
initial_results.add(get_result(test_name, ResultType.Crash), False,
False)
initial_results.add(get_result(test_name, ResultType.Failure), False,
False)
initial_results.add(get_result(test_name, ResultType.Timeout), False,
False)
all_retry_results = [
test_run_results.TestRunResults(expectations, 2, None)
]
all_retry_results[0].add(get_result(test_name, ResultType.Failure),
False, False)
all_retry_results[0].add(get_result(test_name, ResultType.Failure),
False, False)
summary = test_run_results.summarize_results(self.port, expectations,
initial_results,
all_retry_results)
self.assertEquals(summary['tests']['passes']['text.html']['expected'],
'PASS')
self.assertEquals(summary['tests']['passes']['text.html']['actual'],
'CRASH FAIL TIMEOUT FAIL FAIL')
self.assertEquals(summary['num_flaky'], 0)
self.assertEquals(summary['num_passes'], 0)
self.assertEquals(summary['num_regressions'], 1)
def test_timeout_then_unexpected_pass(self):
test_name = 'failures/expected/text.html'
expectations = test_expectations.TestExpectations(self.port)
initial_results = test_run_results.TestRunResults(
expectations, 1, None)
initial_results.add(
get_result(test_name, ResultType.Timeout, run_time=1), False,
False)
all_retry_results = [
test_run_results.TestRunResults(expectations, 1, None),
test_run_results.TestRunResults(expectations, 1, None),
test_run_results.TestRunResults(expectations, 1, None)
]
all_retry_results[0].add(
get_result(test_name, ResultType.Failure, run_time=0.1), False,
False)
all_retry_results[1].add(
get_result(test_name, ResultType.Pass, run_time=0.1), False, False)
all_retry_results[2].add(
get_result(test_name, ResultType.Pass, run_time=0.1), False, False)
summary = test_run_results.summarize_results(self.port, expectations,
initial_results,
all_retry_results)
self.assertIn('is_unexpected',
summary['tests']['failures']['expected']['text.html'])
self.assertEquals(
summary['tests']['failures']['expected']['text.html']['expected'],
'FAIL')
self.assertEquals(
summary['tests']['failures']['expected']['text.html']['actual'],
'TIMEOUT FAIL PASS PASS')
self.assertEquals(summary['num_passes'], 1)
self.assertEquals(summary['num_regressions'], 0)
self.assertEquals(summary['num_flaky'], 0)
def test_summarized_results_flaky_pass_after_first_retry(self):
test_name = 'passes/text.html'
expectations = test_expectations.TestExpectations(self.port)
initial_results = test_run_results.TestRunResults(
expectations, 1, None)
initial_results.add(get_result(test_name, ResultType.Crash), False,
False)
all_retry_results = [
test_run_results.TestRunResults(expectations, 1, None),
test_run_results.TestRunResults(expectations, 1, None),
test_run_results.TestRunResults(expectations, 1, None)
]
all_retry_results[0].add(get_result(test_name, ResultType.Timeout),
False, False)
all_retry_results[1].add(get_result(test_name, ResultType.Pass), True,
False)
all_retry_results[2].add(get_result(test_name, ResultType.Pass), True,
False)
summary = test_run_results.summarize_results(self.port, expectations,
initial_results,
all_retry_results)
self.assertTrue(
'is_unexpected' not in summary['tests']['passes']['text.html'])
self.assertEquals(summary['tests']['passes']['text.html']['expected'],
'PASS')
self.assertEquals(summary['tests']['passes']['text.html']['actual'],
'CRASH TIMEOUT PASS PASS')
self.assertEquals(summary['num_flaky'], 1)
self.assertEquals(summary['num_passes'], 0)
self.assertEquals(summary['num_regressions'], 0)
def test_summarized_results_with_iterations(self):
test_name = 'passes/text.html'
expectations = test_expectations.TestExpectations(
self.port, [test_name])
initial_results = test_run_results.TestRunResults(expectations, 3)
initial_results.add(get_result(test_name, test_expectations.CRASH),
False, False)
initial_results.add(get_result(test_name, test_expectations.IMAGE),
False, False)
initial_results.add(get_result(test_name, test_expectations.TIMEOUT),
False, False)
all_retry_results = [test_run_results.TestRunResults(expectations, 2)]
all_retry_results[0].add(get_result(test_name, test_expectations.TEXT),
False, False)
all_retry_results[0].add(get_result(test_name, test_expectations.LEAK),
False, False)
summary = test_run_results.summarize_results(
self.port,
expectations,
initial_results,
all_retry_results,
enabled_pixel_tests_in_retry=True)
print summary
self.assertEquals(summary['tests']['passes']['text.html']['expected'],
'PASS')
self.assertEquals(summary['tests']['passes']['text.html']['actual'],
'CRASH IMAGE TIMEOUT TEXT LEAK')
self.assertEquals(summary['num_flaky'], 0)
self.assertEquals(summary['num_passes'], 0)
self.assertEquals(summary['num_regressions'], 1)
def test_summarized_results_flaky_pass_after_first_retry(self):
test_name = 'passes/text.html'
expectations = test_expectations.TestExpectations(
self.port, [test_name])
initial_results = test_run_results.TestRunResults(expectations, 1)
initial_results.add(get_result(test_name, test_expectations.CRASH),
False, False)
all_retry_results = [
test_run_results.TestRunResults(expectations, 1),
test_run_results.TestRunResults(expectations, 1),
test_run_results.TestRunResults(expectations, 1)
]
all_retry_results[0].add(
get_result(test_name, test_expectations.TIMEOUT), False, False)
all_retry_results[1].add(get_result(test_name, test_expectations.PASS),
True, False)
all_retry_results[2].add(get_result(test_name, test_expectations.PASS),
True, False)
summary = test_run_results.summarize_results(
self.port,
expectations,
initial_results,
all_retry_results,
enabled_pixel_tests_in_retry=True)
self.assertTrue(
'is_unexpected' not in summary['tests']['passes']['text.html'])
self.assertEquals(summary['tests']['passes']['text.html']['expected'],
'PASS')
self.assertEquals(summary['tests']['passes']['text.html']['actual'],
'CRASH TIMEOUT PASS PASS')
self.assertEquals(summary['num_flaky'], 1)
self.assertEquals(summary['num_passes'], 0)
self.assertEquals(summary['num_regressions'], 0)
def setUp(self):
expectations = test_expectations.TestExpectations(
MockHost().port_factory.get(port_name='test'))
self.results = test_run_results.TestRunResults(expectations, 1, None)
self.test = get_result('failures/expected/text.html',
ResultType.Timeout,
run_time=1)
def run_results(port, extra_skipped_tests=None):
tests = [
'passes/text.html', 'failures/expected/timeout.html',
'failures/expected/crash.html', 'failures/expected/leak.html',
'failures/expected/keyboard.html', 'failures/expected/audio.html',
'failures/expected/text.html', 'passes/skipped/skip.html'
]
expectations = test_expectations.TestExpectations(port, tests)
if extra_skipped_tests:
expectations.add_extra_skipped_tests(extra_skipped_tests)
return test_run_results.TestRunResults(expectations, len(tests))
def test_timeout_then_unexpected_pass(self):
test_name = 'failures/expected/text.html'
expectations = test_expectations.TestExpectations(self.port, [test_name])
initial_results = test_run_results.TestRunResults(expectations, 1)
initial_results.add(get_result(test_name, test_expectations.TIMEOUT, run_time=1), False, False)
all_retry_results = [test_run_results.TestRunResults(expectations, 1),
test_run_results.TestRunResults(expectations, 1),
test_run_results.TestRunResults(expectations, 1)]
all_retry_results[0].add(get_result(test_name, test_expectations.LEAK, run_time=0.1), False, False)
all_retry_results[1].add(get_result(test_name, test_expectations.PASS, run_time=0.1), False, False)
all_retry_results[2].add(get_result(test_name, test_expectations.PASS, run_time=0.1), False, False)
summary = test_run_results.summarize_results(
self.port, expectations, initial_results, all_retry_results,
enabled_pixel_tests_in_retry=True)
self.assertIn('is_unexpected', summary['tests']['failures']['expected']['text.html'])
self.assertEquals(summary['tests']['failures']['expected']['text.html']['expected'], 'FAIL')
self.assertEquals(summary['tests']['failures']['expected']['text.html']['actual'], 'TIMEOUT LEAK PASS PASS')
self.assertEquals(summary['num_passes'], 1)
self.assertEquals(summary['num_regressions'], 0)
self.assertEquals(summary['num_flaky'], 0)
def run_results(port, extra_skipped_tests=None):
tests = [
'passes/text.html', 'failures/expected/timeout.html',
'failures/expected/crash.html', 'failures/expected/leak.html',
'failures/expected/keyboard.html', 'failures/expected/audio.html',
'failures/expected/text.html', 'passes/skipped/skip.html'
]
expectations = test_expectations.TestExpectations(port)
if extra_skipped_tests:
extra_expectations = '# results: [ Skip ]'
for test in extra_skipped_tests:
extra_expectations += '\n%s [ Skip ]' % test
expectations.merge_raw_expectations(extra_expectations)
return test_run_results.TestRunResults(expectations, len(tests), None)
