def test_summarized_results_flaky_pass_after_first_retry(self): test_name = 'passes/text.html' expectations = test_expectations.TestExpectations( self.port, [test_name]) initial_results = test_run_results.TestRunResults(expectations, 1) initial_results.add(get_result(test_name, test_expectations.CRASH), False, False) all_retry_results = [ test_run_results.TestRunResults(expectations, 1), test_run_results.TestRunResults(expectations, 1), test_run_results.TestRunResults(expectations, 1) ] all_retry_results[0].add( get_result(test_name, test_expectations.TIMEOUT), False, False) all_retry_results[1].add(get_result(test_name, test_expectations.PASS), True, False) all_retry_results[2].add(get_result(test_name, test_expectations.PASS), True, False) summary = test_run_results.summarize_results( self.port, expectations, initial_results, all_retry_results, enabled_pixel_tests_in_retry=True) self.assertTrue( 'is_unexpected' not in summary['tests']['passes']['text.html']) self.assertEquals(summary['tests']['passes']['text.html']['expected'], 'PASS') self.assertEquals(summary['tests']['passes']['text.html']['actual'], 'CRASH TIMEOUT PASS PASS') self.assertEquals(summary['num_flaky'], 1) self.assertEquals(summary['num_passes'], 0) self.assertEquals(summary['num_regressions'], 0)
def run_results(port, extra_skipped_tests=[]): tests = ['passes/text.html', 'failures/expected/timeout.html', 'failures/expected/crash.html', 'failures/expected/leak.html', 'failures/expected/keyboard.html', 'failures/expected/audio.html', 'failures/expected/text.html', 'passes/skipped/skip.html'] expectations = test_expectations.TestExpectations(port, tests) if extra_skipped_tests: expectations.add_extra_skipped_tests(extra_skipped_tests) return test_run_results.TestRunResults(expectations, len(tests))
def run_results(port): tests = [ 'passes/text.html', 'failures/expected/timeout.html', 'failures/expected/crash.html', 'failures/expected/hang.html', 'failures/expected/audio.html' ] expectations = test_expectations.TestExpectations(port, tests) return test_run_results.TestRunResults(expectations, len(tests))
def test_timeout_then_unexpected_pass(self): test_name = 'failures/expected/text.html' expectations = test_expectations.TestExpectations(self.port, [test_name]) initial_results = test_run_results.TestRunResults(expectations, 1) initial_results.add(get_result(test_name, test_expectations.TIMEOUT, run_time=1), False, False) all_retry_results = [test_run_results.TestRunResults(expectations, 1), test_run_results.TestRunResults(expectations, 1), test_run_results.TestRunResults(expectations, 1)] all_retry_results[0].add(get_result(test_name, test_expectations.LEAK, run_time=0.1), False, False) all_retry_results[1].add(get_result(test_name, test_expectations.PASS, run_time=0.1), False, False) all_retry_results[2].add(get_result(test_name, test_expectations.PASS, run_time=0.1), False, False) summary = test_run_results.summarize_results( self.port, expectations, initial_results, all_retry_results, enabled_pixel_tests_in_retry=True) self.assertTrue('is_unexpected' in summary['tests']['failures']['expected']['text.html']) self.assertEquals(summary['tests']['failures']['expected']['text.html']['expected'], 'FAIL') self.assertEquals(summary['tests']['failures']['expected']['text.html']['actual'], 'TIMEOUT LEAK PASS PASS') self.assertEquals(summary['num_passes'], 1) self.assertEquals(summary['num_regressions'], 0) self.assertEquals(summary['num_flaky'], 0)
def test_timeout_then_unexpected_pass(self): tests = ['failures/expected/image.html'] expectations = test_expectations.TestExpectations(self.port, tests) initial_results = test_run_results.TestRunResults( expectations, len(tests)) initial_results.add( get_result('failures/expected/image.html', test_expectations.TIMEOUT, run_time=1), False, False) retry_results = test_run_results.TestRunResults( expectations, len(tests)) retry_results.add( get_result('failures/expected/image.html', test_expectations.PASS, run_time=0.1), False, False) summary = test_run_results.summarize_results( self.port, expectations, initial_results, retry_results, enabled_pixel_tests_in_retry=True, only_include_failing=True) self.assertEquals(summary['num_regressions'], 0) self.assertEquals(summary['num_passes'], 1)