def test_summarized_results_with_iterations(self): test_name = 'passes/text.html' expectations = test_expectations.TestExpectations(self.port) initial_results = test_run_results.TestRunResults( expectations, 3, None) initial_results.add(get_result(test_name, ResultType.Crash), False, False) initial_results.add(get_result(test_name, ResultType.Failure), False, False) initial_results.add(get_result(test_name, ResultType.Timeout), False, False) all_retry_results = [ test_run_results.TestRunResults(expectations, 2, None) ] all_retry_results[0].add(get_result(test_name, ResultType.Failure), False, False) all_retry_results[0].add(get_result(test_name, ResultType.Failure), False, False) summary = test_run_results.summarize_results(self.port, expectations, initial_results, all_retry_results) self.assertEquals(summary['tests']['passes']['text.html']['expected'], 'PASS') self.assertEquals(summary['tests']['passes']['text.html']['actual'], 'CRASH FAIL TIMEOUT FAIL FAIL') self.assertEquals(summary['num_flaky'], 0) self.assertEquals(summary['num_passes'], 0) self.assertEquals(summary['num_regressions'], 1)
def test_timeout_then_unexpected_pass(self): test_name = 'failures/expected/text.html' expectations = test_expectations.TestExpectations(self.port) initial_results = test_run_results.TestRunResults( expectations, 1, None) initial_results.add( get_result(test_name, ResultType.Timeout, run_time=1), False, False) all_retry_results = [ test_run_results.TestRunResults(expectations, 1, None), test_run_results.TestRunResults(expectations, 1, None), test_run_results.TestRunResults(expectations, 1, None) ] all_retry_results[0].add( get_result(test_name, ResultType.Failure, run_time=0.1), False, False) all_retry_results[1].add( get_result(test_name, ResultType.Pass, run_time=0.1), False, False) all_retry_results[2].add( get_result(test_name, ResultType.Pass, run_time=0.1), False, False) summary = test_run_results.summarize_results(self.port, expectations, initial_results, all_retry_results) self.assertIn('is_unexpected', summary['tests']['failures']['expected']['text.html']) self.assertEquals( summary['tests']['failures']['expected']['text.html']['expected'], 'FAIL') self.assertEquals( summary['tests']['failures']['expected']['text.html']['actual'], 'TIMEOUT FAIL PASS PASS') self.assertEquals(summary['num_passes'], 1) self.assertEquals(summary['num_regressions'], 0) self.assertEquals(summary['num_flaky'], 0)
def test_summarized_results_flaky_pass_after_first_retry(self): test_name = 'passes/text.html' expectations = test_expectations.TestExpectations(self.port) initial_results = test_run_results.TestRunResults( expectations, 1, None) initial_results.add(get_result(test_name, ResultType.Crash), False, False) all_retry_results = [ test_run_results.TestRunResults(expectations, 1, None), test_run_results.TestRunResults(expectations, 1, None), test_run_results.TestRunResults(expectations, 1, None) ] all_retry_results[0].add(get_result(test_name, ResultType.Timeout), False, False) all_retry_results[1].add(get_result(test_name, ResultType.Pass), True, False) all_retry_results[2].add(get_result(test_name, ResultType.Pass), True, False) summary = test_run_results.summarize_results(self.port, expectations, initial_results, all_retry_results) self.assertTrue( 'is_unexpected' not in summary['tests']['passes']['text.html']) self.assertEquals(summary['tests']['passes']['text.html']['expected'], 'PASS') self.assertEquals(summary['tests']['passes']['text.html']['actual'], 'CRASH TIMEOUT PASS PASS') self.assertEquals(summary['num_flaky'], 1) self.assertEquals(summary['num_passes'], 0) self.assertEquals(summary['num_regressions'], 0)
def test_summarized_results_with_iterations(self): test_name = 'passes/text.html' expectations = test_expectations.TestExpectations( self.port, [test_name]) initial_results = test_run_results.TestRunResults(expectations, 3) initial_results.add(get_result(test_name, test_expectations.CRASH), False, False) initial_results.add(get_result(test_name, test_expectations.IMAGE), False, False) initial_results.add(get_result(test_name, test_expectations.TIMEOUT), False, False) all_retry_results = [test_run_results.TestRunResults(expectations, 2)] all_retry_results[0].add(get_result(test_name, test_expectations.TEXT), False, False) all_retry_results[0].add(get_result(test_name, test_expectations.LEAK), False, False) summary = test_run_results.summarize_results( self.port, expectations, initial_results, all_retry_results, enabled_pixel_tests_in_retry=True) print summary self.assertEquals(summary['tests']['passes']['text.html']['expected'], 'PASS') self.assertEquals(summary['tests']['passes']['text.html']['actual'], 'CRASH IMAGE TIMEOUT TEXT LEAK') self.assertEquals(summary['num_flaky'], 0) self.assertEquals(summary['num_passes'], 0) self.assertEquals(summary['num_regressions'], 1)
def test_summarized_results_flaky_pass_after_first_retry(self): test_name = 'passes/text.html' expectations = test_expectations.TestExpectations( self.port, [test_name]) initial_results = test_run_results.TestRunResults(expectations, 1) initial_results.add(get_result(test_name, test_expectations.CRASH), False, False) all_retry_results = [ test_run_results.TestRunResults(expectations, 1), test_run_results.TestRunResults(expectations, 1), test_run_results.TestRunResults(expectations, 1) ] all_retry_results[0].add( get_result(test_name, test_expectations.TIMEOUT), False, False) all_retry_results[1].add(get_result(test_name, test_expectations.PASS), True, False) all_retry_results[2].add(get_result(test_name, test_expectations.PASS), True, False) summary = test_run_results.summarize_results( self.port, expectations, initial_results, all_retry_results, enabled_pixel_tests_in_retry=True) self.assertTrue( 'is_unexpected' not in summary['tests']['passes']['text.html']) self.assertEquals(summary['tests']['passes']['text.html']['expected'], 'PASS') self.assertEquals(summary['tests']['passes']['text.html']['actual'], 'CRASH TIMEOUT PASS PASS') self.assertEquals(summary['num_flaky'], 1) self.assertEquals(summary['num_passes'], 0) self.assertEquals(summary['num_regressions'], 0)
def setUp(self): expectations = test_expectations.TestExpectations( MockHost().port_factory.get(port_name='test')) self.results = test_run_results.TestRunResults(expectations, 1, None) self.test = get_result('failures/expected/text.html', ResultType.Timeout, run_time=1)
def run_results(port, extra_skipped_tests=None): tests = [ 'passes/text.html', 'failures/expected/timeout.html', 'failures/expected/crash.html', 'failures/expected/leak.html', 'failures/expected/keyboard.html', 'failures/expected/audio.html', 'failures/expected/text.html', 'passes/skipped/skip.html' ] expectations = test_expectations.TestExpectations(port, tests) if extra_skipped_tests: expectations.add_extra_skipped_tests(extra_skipped_tests) return test_run_results.TestRunResults(expectations, len(tests))
def test_timeout_then_unexpected_pass(self): test_name = 'failures/expected/text.html' expectations = test_expectations.TestExpectations(self.port, [test_name]) initial_results = test_run_results.TestRunResults(expectations, 1) initial_results.add(get_result(test_name, test_expectations.TIMEOUT, run_time=1), False, False) all_retry_results = [test_run_results.TestRunResults(expectations, 1), test_run_results.TestRunResults(expectations, 1), test_run_results.TestRunResults(expectations, 1)] all_retry_results[0].add(get_result(test_name, test_expectations.LEAK, run_time=0.1), False, False) all_retry_results[1].add(get_result(test_name, test_expectations.PASS, run_time=0.1), False, False) all_retry_results[2].add(get_result(test_name, test_expectations.PASS, run_time=0.1), False, False) summary = test_run_results.summarize_results( self.port, expectations, initial_results, all_retry_results, enabled_pixel_tests_in_retry=True) self.assertIn('is_unexpected', summary['tests']['failures']['expected']['text.html']) self.assertEquals(summary['tests']['failures']['expected']['text.html']['expected'], 'FAIL') self.assertEquals(summary['tests']['failures']['expected']['text.html']['actual'], 'TIMEOUT LEAK PASS PASS') self.assertEquals(summary['num_passes'], 1) self.assertEquals(summary['num_regressions'], 0) self.assertEquals(summary['num_flaky'], 0)
def run_results(port, extra_skipped_tests=None): tests = [ 'passes/text.html', 'failures/expected/timeout.html', 'failures/expected/crash.html', 'failures/expected/leak.html', 'failures/expected/keyboard.html', 'failures/expected/audio.html', 'failures/expected/text.html', 'passes/skipped/skip.html' ] expectations = test_expectations.TestExpectations(port) if extra_skipped_tests: extra_expectations = '# results: [ Skip ]' for test in extra_skipped_tests: extra_expectations += '\n%s [ Skip ]' % test expectations.merge_raw_expectations(extra_expectations) return test_run_results.TestRunResults(expectations, len(tests), None)