def test_rebaseline_updates_expectations_file(self): self._write(self.test_expectations_path, ( '# tags: [ Mac10.10 Mac Linux ]\n' '# tags: [ Debug ]\n' '# results: [ Failure ]\n' 'crbug.com/123 [ Debug Mac ] userscripts/first-test.html [ Failure ]\n' '[ Linux ] userscripts/first-test.html [ Failure ]\n')) self._setup_mock_build_data() test_baseline_set = TestBaselineSet(self.tool) test_baseline_set.add('userscripts/first-test.html', Build('MOCK Mac10.11')) self.command.rebaseline(self.options(), test_baseline_set) new_expectations = self._read(self.test_expectations_path) self.assertMultiLineEqual(new_expectations, ( '# tags: [ Mac10.10 Mac Linux ]\n' '# tags: [ Debug ]\n' '# results: [ Failure ]\n' 'crbug.com/123 [ Debug Mac10.10 ] userscripts/first-test.html [ Failure ]\n' '[ Linux ] userscripts/first-test.html [ Failure ]\n'))
def test_get_step_name(self): fetcher = TestResultsFetcher() fetcher.web = MockWeb( urls={ 'https://test-results.appspot.com/testfile?buildnumber=5&' 'callback=ADD_RESULTS&builder=foo&name=full_results.json': 'ADD_RESULTS(%s);' % (json.dumps([{ "TestType": "blink_web_tests (with patch)" }, { "TestType": "not_site_per_process_blink_web_tests (with patch)" }, { "TestType": "blink_web_tests (retry with patch)" }, { "TestType": "base_unittests (with patch)" }])) }) step_name = fetcher.get_layout_test_step_name(Build('foo', 5)) self.assertEqual(step_name, 'blink_web_tests (with patch)') self.assertLog([])
def test_results_directory(self): test_baseline_set = TestBaselineSet(self.tool) test_baseline_set.add('userscripts/first-test.html', Build('MOCK Win7')) self.command.rebaseline( self.options(optimize=False, results_directory='/tmp'), test_baseline_set) self.assertEqual(self.tool.executive.calls, [ [[ 'python', 'echo', 'copy-existing-baselines-internal', '--verbose', '--test', 'userscripts/first-test.html', '--suffixes', 'png,txt', '--port-name', 'test-win-win7', ]], [[ 'python', 'echo', 'rebaseline-test-internal', '--verbose', '--test', 'userscripts/first-test.html', '--suffixes', 'png,txt', '--port-name', 'test-win-win7', '--builder', 'MOCK Win7', '--results-directory', '/tmp', '--step-name', 'blink_web_tests (with patch)', ]], ])
def test_rebaseline_handles_slow_in_file(self): self._write(self.test_expectations_path, ('# tags: [ Linux Mac Mac10.10 Win ]\n' '# results: [ Failure Slow ]\n' '[ Linux ] userscripts/first-test.html [ Failure ]\n' '[ Mac ] userscripts/first-test.html [ Failure ]\n' '[ Win ] userscripts/first-test.html [ Failure Slow ]\n')) self._setup_mock_build_data() test_baseline_set = TestBaselineSet(self.tool) test_baseline_set.add('userscripts/first-test.html', Build('MOCK Mac10.11')) self.command.rebaseline(self.options(), test_baseline_set) new_expectations = self._read(self.test_expectations_path) self.assertMultiLineEqual( new_expectations, ('# tags: [ Linux Mac Mac10.10 Win ]\n' '# results: [ Failure Slow ]\n' '[ Linux ] userscripts/first-test.html [ Failure ]\n' '[ Mac10.10 ] userscripts/first-test.html [ Failure ]\n' '[ Win ] userscripts/first-test.html [ Failure Slow ]\n'))
def test_get_step_name(self): fetcher = TestResultsFetcher() fetcher.builders.step_name_for_builder = lambda builder: None fetcher.web = MockWeb( urls={ 'https://test-results.appspot.com/testfile?buildnumber=5&' 'callback=ADD_RESULTS&builder=foo&name=full_results.json': b'ADD_RESULTS(' + json.dumps([{ "TestType": "blink_web_tests (with patch)" }, { "TestType": "not_site_per_process_blink_web_tests (with patch)" }, { "TestType": "blink_web_tests (retry with patch)" }, { "TestType": "base_unittests (with patch)" }]).encode('utf8', 'replace') + b');' }) step_name = fetcher.get_layout_test_step_name(Build('foo', 5)) self.assertEqual(step_name, 'blink_web_tests (with patch)') self.assertLog([])
def test_try_job_results_skip_experimental_cq(self): git_cl = GitCL(MockHost()) git_cl._host.web = MockWeb(responses=[{ 'status_code': 200, 'body': SEARCHBUILDS_RESPONSE_PREFIX + b"""{ "builds": [ { "status": "SUCCESS", "builder": { "builder": "builder-a" }, "number": 111, "tags": [ {"key": "user_agent", "value": "cq"} ] }, { "status": "SUCCESS", "builder": { "builder": "builder-b" }, "number": 222, "tags": [ {"key": "user_agent", "value": "cq"}, {"key": "cq_experimental", "value": "true"} ] } ] }""" }]) self.assertEqual( # Only one build appears - builder-b is ignored because it is # experimental. git_cl.try_job_results(issue_number=None, cq_only=True), { Build('builder-a', 111): TryJobStatus('COMPLETED', 'SUCCESS'), })
def test_fetch_results_with_weird_step_name(self): fetcher = TestResultsFetcher() fetcher.web = MockWeb( urls={ 'https://test-results.appspot.com/testfile?buildnumber=123&' 'callback=ADD_RESULTS&builder=builder&name=full_results.json': 'ADD_RESULTS(%s);' % (json.dumps( [{ "TestType": "blink_web_tests on Intel GPU (with patch)" }, { "TestType": "base_unittests (with patch)" }])), 'https://test-results.appspot.com/data/layout_results/builder/123/' 'blink_web_tests%20on%20Intel%20GPU%20%28with%20patch%29/' 'layout-test-results/failing_results.json': json.dumps({'passed': True}), }) results = fetcher.fetch_results(Build('builder', 123)) self.assertEqual( results._results, # pylint: disable=protected-access {'passed': True}) self.assertLog([])
def execute(self, options, args, tool): self._tool = tool if not args: _log.error('Must list tests to rebaseline.') return if options.builders: builders_to_check = [] for builder_names in options.builders: builders_to_check += builder_names.split(',') else: builders_to_check = self._builders_to_pull_from() test_baseline_set = TestBaselineSet(tool) for builder in builders_to_check: for test_prefix in args: test_baseline_set.add(test_prefix, Build(builder)) _log.debug('Rebaselining: %s', test_baseline_set) self.rebaseline(options, test_baseline_set)
def test_rebaseline_handles_skips_in_file(self): # This test is like test_rebaseline_handles_platform_skips, except that the # Skip is in the same (generic) file rather than a platform file. In this case, # the Skip line should be left unmodified. Note that the first line is now # qualified as "[Linux Mac Win]"; if it was unqualified, it would conflict with # the second line. self._write( self.test_expectations_path, ('Bug(x) [ Linux Mac ] userscripts/first-test.html [ Failure ]\n' 'Bug(y) [ Win ] userscripts/first-test.html [ Skip ]\n')) self._setup_mock_build_data() test_baseline_set = TestBaselineSet(self.tool) test_baseline_set.add('userscripts/first-test.html', Build('MOCK Mac10.11')) self.command.rebaseline(self.options(), test_baseline_set) new_expectations = self._read(self.test_expectations_path) self.assertMultiLineEqual( new_expectations, ('Bug(x) [ Linux ] userscripts/first-test.html [ Failure ]\n' 'Bug(x) [ Mac10.10 ] userscripts/first-test.html [ Failure ]\n' 'Bug(y) [ Win ] userscripts/first-test.html [ Skip ]\n'))
def test_no_optimize(self): test_baseline_set = TestBaselineSet(self.tool) test_baseline_set.add('userscripts/first-test.html', Build('MOCK Win7')) self.command.rebaseline(self.options(optimize=False), test_baseline_set) self.assertEqual(self.tool.executive.calls, [[[ 'python', 'echo', 'copy-existing-baselines-internal', '--verbose', '--test', 'userscripts/first-test.html', '--suffixes', 'txt,png', '--port-name', 'test-win-win7', ]], [[ 'python', 'echo', 'rebaseline-test-internal', '--verbose', '--test', 'userscripts/first-test.html', '--suffixes', 'txt,png', '--port-name', 'test-win-win7', '--builder', 'MOCK Win7', '--step-name', 'webkit_layout_tests (with patch)', ]]])
def test_fill_in_missing_results_prefers_build_with_same_os_type(self): self.tool.builders = BuilderList({ 'MOCK Foo12': { 'port_name': 'foo-foo12', 'specifiers': ['Foo12', 'Release'], 'is_try_builder': True, }, 'MOCK Foo45': { 'port_name': 'foo-foo45', 'specifiers': ['Foo45', 'Release'], 'is_try_builder': True, }, 'MOCK Bar3': { 'port_name': 'bar-bar3', 'specifiers': ['Bar3', 'Release'], 'is_try_builder': True, }, 'MOCK Bar4': { 'port_name': 'bar-bar4', 'specifiers': ['Bar4', 'Release'], 'is_try_builder': True, }, }) test_baseline_set = TestBaselineSet(self.tool) test_baseline_set.add('one/flaky-fail.html', Build('MOCK Foo12', 100)) test_baseline_set.add('one/flaky-fail.html', Build('MOCK Bar4', 200)) self.command.fill_in_missing_results(test_baseline_set) self.assertEqual( sorted(test_baseline_set.build_port_pairs('one/flaky-fail.html')), [ (Build('MOCK Bar4', 200), 'bar-bar3'), (Build('MOCK Bar4', 200), 'bar-bar4'), (Build('MOCK Foo12', 100), 'foo-foo12'), (Build('MOCK Foo12', 100), 'foo-foo45'), ]) self.assertLog([ 'INFO: For one/flaky-fail.html:\n', 'INFO: Using "MOCK Foo12" build 100 for foo-foo45.\n', 'INFO: Using "MOCK Bar4" build 200 for bar-bar3.\n', ])
def test_get_step_name_without_build_number(self): fetcher = TestResultsFetcher() self.assertIsNone( fetcher.get_layout_test_step_name(Build('builder', None)))
def test_fetch_results_without_build_number(self): fetcher = TestResultsFetcher() self.assertIsNone(fetcher.fetch_results(Build('builder', None)))
def test_all_success_true(self): self.assertTrue( GitCL.all_success({ Build('some-builder', 1): TryJobStatus('COMPLETED', 'SUCCESS'), }))
def setUp(self): BaseTestCase.setUp(self) LoggingTestCase.setUp(self) builds = { Build('MOCK Try Win', 5000): TryJobStatus('COMPLETED', 'FAILURE'), Build('MOCK Try Mac', 4000): TryJobStatus('COMPLETED', 'FAILURE'), Build('MOCK Try Linux', 6000): TryJobStatus('COMPLETED', 'FAILURE'), } self.command.git_cl = MockGitCL(self.tool, builds) git = MockGit(filesystem=self.tool.filesystem, executive=self.tool.executive) git.changed_files = lambda **_: [ RELATIVE_WEB_TESTS + 'one/text-fail.html', RELATIVE_WEB_TESTS + 'one/flaky-fail.html', ] self.tool.git = lambda: git self.tool.builders = BuilderList({ 'MOCK Try Win': { 'port_name': 'test-win-win7', 'specifiers': ['Win7', 'Release'], 'is_try_builder': True, }, 'MOCK Try Linux': { 'port_name': 'test-linux-trusty', 'specifiers': ['Trusty', 'Release'], 'is_try_builder': True, }, 'MOCK Try Mac': { 'port_name': 'test-mac-mac10.11', 'specifiers': ['Mac10.11', 'Release'], 'is_try_builder': True, }, }) web_test_results = WebTestResults({ 'tests': { 'one': { 'crash.html': { 'expected': 'PASS', 'actual': 'CRASH', 'is_unexpected': True, 'artifacts': { 'crash_log': ['crash.log'] } }, 'expected-fail.html': { 'expected': 'FAIL', 'actual': 'FAIL', 'artifacts': { 'expected_text': ['expected-fail-expected.txt'], 'actual_text': ['expected-fail-actual.txt'] } }, 'flaky-fail.html': { 'expected': 'PASS', 'actual': 'PASS FAIL', 'is_unexpected': True, 'artifacts': { 'expected_audio': ['flaky-fail-expected.wav'], 'actual_audio': ['flaky-fail-actual.wav'] } }, 'missing.html': { 'expected': 'PASS', 'actual': 'FAIL', 'is_unexpected': True, 'artifacts': { 'actual_image': ['missing-actual.png'] }, 'is_missing_image': True }, 'slow-fail.html': { 'expected': 'SLOW', 'actual': 'FAIL', 'is_unexpected': True, 'artifacts': { 'actual_text': ['slow-fail-actual.txt'], 'expected_text': ['slow-fail-expected.txt'] } }, 'text-fail.html': { 'expected': 'PASS', 'actual': 'FAIL', 'is_unexpected': True, 'artifacts': { 'actual_text': ['text-fail-actual.txt'], 'expected_text': ['text-fail-expected.txt'] } }, 'unexpected-pass.html': { 'expected': 'FAIL', 'actual': 'PASS', 'is_unexpected': True }, }, 'two': { 'image-fail.html': { 'expected': 'PASS', 'actual': 'FAIL', 'is_unexpected': True, 'artifacts': { 'actual_image': ['image-fail-actual.png'], 'expected_image': ['image-fail-expected.png'] } } }, }, }) for build in builds: self.tool.results_fetcher.set_results(build, web_test_results) self.tool.results_fetcher.set_retry_sumary_json( build, json.dumps({ 'failures': [ 'one/flaky-fail.html', 'one/missing.html', 'one/slow-fail.html', 'one/text-fail.html', 'two/image-fail.html', ], 'ignored': [], })) # Write to the mock filesystem so that these tests are considered to exist. tests = [ 'one/flaky-fail.html', 'one/missing.html', 'one/slow-fail.html', 'one/text-fail.html', 'two/image-fail.html', ] for test in tests: path = self.mac_port.host.filesystem.join( self.mac_port.web_tests_dir(), test) self._write(path, 'contents') self.mac_port.host.filesystem.write_text_file( '/test.checkout/web_tests/external/wpt/MANIFEST.json', '{}')
def testUpdateTestExpectationsForWebview(self): host = self._setup_host() host.results_fetcher.set_results( Build('MOCK Android Pie', 123), WebTestResults({ 'tests': { 'abc.html': { 'expected': 'PASS', 'actual': 'CRASH TIMEOUT', 'is_unexpected': True, }, 'jkl.html': { 'expected': 'PASS', 'actual': 'FAIL', 'is_unexpected': True, }, 'cat.html': { 'expected': 'PASS', 'actual': 'CRASH CRASH TIMEOUT', 'is_unexpected': True, }, }, }, step_name='system_webview_wpt (with patch)'), step_name='system_webview_wpt (with patch)') updater = AndroidWPTExpectationsUpdater( host, ['-vvv', '--android-product', 'android_webview']) updater.git_cl = MockGitCL(host, { Build('MOCK Android Pie', 123): TryJobStatus('COMPLETED', 'FAILURE')}) # Run command updater.run() # Get new expectations content = host.filesystem.read_text_file( PRODUCTS_TO_EXPECTATION_FILE_PATHS['android_webview']) self.assertEqual( content, ('# results: [ Failure Crash Timeout]\n' '\n' '# Add untriaged failures in this block\n' 'crbug.com/1050754 external/wpt/abc.html [ Crash Failure Timeout ]\n' 'crbug.com/1050754 external/wpt/cat.html [ Crash Failure Timeout ]\n' 'crbug.com/1050754 external/wpt/def.html [ Crash ]\n' 'external/wpt/dog.html [ Crash Timeout ]\n' 'crbug.com/1050754 external/wpt/ghi.html [ Timeout ]\n' 'crbug.com/1111111 crbug.com/1050754' ' external/wpt/jkl.html [ Failure ]\n' 'crbug.com/6789043 external/wpt/van.html [ Failure ]\n' 'external/wpt/www.html [ Crash Failure ]\n' '\n' '# This comment will not be deleted\n' 'crbug.com/111111 external/wpt/hello_world.html [ Crash ]\n')) # check that chrome android's expectation file was not modified # since the same bot is used to update chrome android & webview # expectations self.assertEqual( host.filesystem.read_text_file( PRODUCTS_TO_EXPECTATION_FILE_PATHS['chrome_android']), self._raw_android_expectations) # Check logs logs = ''.join(self.logMessages()).lower() self.assertNotIn('weblayer', logs) self.assertNotIn('chrome', logs)
def testUpdateTestExpectationsForAll(self): host = self._setup_host() # Add results for Weblayer host.results_fetcher.set_results( Build('MOCK Android Weblayer - Pie', 123), WebTestResults({ 'tests': { 'abc.html': { 'expected': 'PASS', 'actual': 'CRASH TIMEOUT', 'is_unexpected': True, }, 'weblayer_only.html': { 'expected': 'PASS', 'actual': 'CRASH CRASH FAIL', 'is_unexpected': True, }, }, }, step_name='weblayer_shell_wpt (with patch)'), step_name='weblayer_shell_wpt (with patch)') # Add Results for Webview host.results_fetcher.set_results( Build('MOCK Android Pie', 101), WebTestResults({ 'tests': { 'cat.html': { 'expected': 'PASS', 'actual': 'CRASH FAIL TIMEOUT', 'is_unexpected': True, }, 'webview_only.html': { 'expected': 'PASS', 'actual': 'TIMEOUT', 'is_unexpected': True, }, }, }, step_name='system_webview_wpt (with patch)'), step_name='system_webview_wpt (with patch)') # Add Results for Chrome host.results_fetcher.set_results( Build('MOCK Android Pie', 101), WebTestResults({ 'tests': { 'jkl.html': { 'expected': 'PASS', 'actual': 'FAIL', 'is_unexpected': True, }, 'chrome_only.html': { 'expected': 'PASS', 'actual': 'CRASH CRASH TIMEOUT', 'is_unexpected': True, }, }, }, step_name='chrome_public_wpt (with patch)'), step_name='chrome_public_wpt (with patch)') updater = AndroidWPTExpectationsUpdater( host, ['-vvv', '--android-product', 'android_weblayer', '--android-product', 'chrome_android', '--android-product', 'android_webview']) updater.git_cl = MockGitCL(host, { Build('MOCK Android Weblayer - Pie', 123): TryJobStatus('COMPLETED', 'FAILURE'), Build('MOCK Android Pie', 101): TryJobStatus('COMPLETED', 'FAILURE')}) # Run command updater.run() # Check expectations for weblayer content = host.filesystem.read_text_file( PRODUCTS_TO_EXPECTATION_FILE_PATHS['android_weblayer']) self.assertEqual( content, ('# results: [ Failure Crash Timeout]\n' '\n' '# Add untriaged failures in this block\n' 'crbug.com/1050754 external/wpt/abc.html [ Crash Failure Timeout ]\n' 'crbug.com/1050754 external/wpt/cat.html [ Failure ]\n' 'crbug.com/1050754 external/wpt/def.html [ Crash ]\n' 'external/wpt/dog.html [ Crash Timeout ]\n' 'crbug.com/1050754 external/wpt/ghi.html [ Timeout ]\n' 'crbug.com/1111111 external/wpt/jkl.html [ Failure ]\n' 'crbug.com/6789043 external/wpt/van.html [ Failure ]\n' 'crbug.com/1050754 external/wpt/weblayer_only.html [ Failure Crash ]\n' 'external/wpt/www.html [ Crash Failure ]\n' '\n' '# This comment will not be deleted\n' 'crbug.com/111111 external/wpt/hello_world.html [ Crash ]\n')) # Check expectations for webview content = host.filesystem.read_text_file( PRODUCTS_TO_EXPECTATION_FILE_PATHS['android_webview']) self.assertEqual( content, ('# results: [ Failure Crash Timeout]\n' '\n' '# Add untriaged failures in this block\n' 'external/wpt/abc.html [ Failure ]\n' 'crbug.com/1050754 external/wpt/cat.html [ Crash Failure Timeout ]\n' 'crbug.com/1050754 external/wpt/def.html [ Crash ]\n' 'external/wpt/dog.html [ Crash Timeout ]\n' 'crbug.com/1050754 external/wpt/ghi.html [ Timeout ]\n' 'crbug.com/1111111 external/wpt/jkl.html [ Failure ]\n' 'crbug.com/6789043 external/wpt/van.html [ Failure ]\n' 'crbug.com/1050754 external/wpt/webview_only.html [ Timeout ]\n' 'external/wpt/www.html [ Crash Failure ]\n' '\n' '# This comment will not be deleted\n' 'crbug.com/111111 external/wpt/hello_world.html [ Crash ]\n')) # Check expectations chrome content = host.filesystem.read_text_file( PRODUCTS_TO_EXPECTATION_FILE_PATHS['chrome_android']) self.assertEqual( content, ('# results: [ Failure Crash Timeout]\n' '\n' '# Add untriaged failures in this block\n' 'external/wpt/abc.html [ Failure ]\n' 'crbug.com/1050754 external/wpt/cat.html [ Failure ]\n' 'crbug.com/1050754 external/wpt/chrome_only.html [ Crash Timeout ]\n' 'crbug.com/1050754 external/wpt/def.html [ Crash ]\n' 'external/wpt/dog.html [ Crash Timeout ]\n' 'crbug.com/1050754 external/wpt/ghi.html [ Timeout ]\n' 'crbug.com/1111111 crbug.com/1050754' ' external/wpt/jkl.html [ Failure ]\n' 'crbug.com/6789043 external/wpt/van.html [ Failure ]\n' 'external/wpt/www.html [ Crash Failure ]\n' '\n' '# This comment will not be deleted\n' 'crbug.com/111111 external/wpt/hello_world.html [ Crash ]\n'))
def test_filter_latest_jobs_higher_build_last(self): self.assertEqual( filter_latest_builds( [Build('foo', 3), Build('bar', 5), Build('foo', 5)]), [Build('bar', 5), Build('foo', 5)])
def test_latest_try_jobs_cq_only(self): git_cl = GitCL(MockHost()) git_cl._host.web = MockWeb(responses=[{ 'status_code': 200, 'body': SEARCHBUILDS_RESPONSE_PREFIX + b"""{ "builds": [ { "status": "SCHEDULED", "builder": { "builder": "cq-a" }, "tags": [ {"key": "user_agent", "value": "cq"} ] }, { "status": "SCHEDULED", "builder": { "builder": "cq-b" }, "tags": [ {"key": "user_agent", "value": "cq"}, {"key": "cq_experimental", "value": "false"} ] }, { "status": "SCHEDULED", "builder": { "builder": "cq-c" }, "tags": [ {"key": "user_agent", "value": "cq"}, {"key": "cq_experimental", "value": "false"} ] }, { "status": "SCHEDULED", "builder": { "builder": "cq-a-experimental" }, "tags": [ {"key": "user_agent", "value": "cq"}, {"key": "cq_experimental", "value": "true"} ] }, { "status": "SCHEDULED", "builder": { "builder": "cq-b-experimental" }, "tags": [ {"key": "user_agent", "value": "cq"}, {"key": "cq_experimental", "value": "true"} ] }, { "status": "SCHEDULED", "builder": { "builder": "other-a" }, "tags": [ {"key": "user_agent", "value": "git_cl_try"} ] }, { "status": "SCHEDULED", "builder": { "builder": "other-b" }, "tags": [ {"key": "user_agent", "value": "git_cl_try"}, {"key": "cq_experimental", "value": "false"} ] } ] }""" }]) self.assertEqual( git_cl.latest_try_jobs(cq_only=True), { Build('cq-a'): TryJobStatus('SCHEDULED'), Build('cq-b'): TryJobStatus('SCHEDULED'), Build('cq-c'): TryJobStatus('SCHEDULED'), })
def test_fetch_webdriver_results_without_master(self): fetcher = TestResultsFetcher() self.assertIsNone( fetcher.fetch_webdriver_test_results(Build('builder', 1), '')) self.assertLog( ['DEBUG: Builder name or build number or master is None\n'])
def test_filter_latest_jobs_no_build_number(self): self.assertEqual( filter_latest_builds([Build('foo', 3), Build('bar'), Build('bar')]), [Build('bar'), Build('foo', 3)])
def testCleanupAndUpdateTestExpectationsForAll(self): # Full integration test for expectations cleanup and update # using builder results. host = self._setup_host() # Add results for Weblayer host.results_fetcher.set_results( Build('MOCK Android Weblayer - Pie', 123), WebTestResults( { 'tests': { 'abc.html': { 'expected': 'PASS', 'actual': 'CRASH TIMEOUT', 'is_unexpected': True, }, 'weblayer_only.html': { 'expected': 'PASS', 'actual': 'CRASH CRASH FAIL', 'is_unexpected': True, }, 'disabled_weblayer_only.html': { 'expected': 'SKIP', 'actual': 'SKIP', 'is_unexpected': True, }, 'unexpected_pass.html': { 'expected': 'FAIL', 'actual': 'PASS', 'is_unexpected': True }, }, }, step_name=WEBLAYER_WPT_STEP + ' (with patch)'), step_name=WEBLAYER_WPT_STEP + ' (with patch)') # Add Results for Webview host.results_fetcher.set_results( Build('MOCK Android Pie', 101), WebTestResults( { 'tests': { 'cat.html': { 'expected': 'PASS', 'actual': 'CRASH FAIL TIMEOUT', 'is_unexpected': True, }, 'webview_only.html': { 'expected': 'PASS', 'actual': 'TIMEOUT', 'is_unexpected': True, }, 'disabled.html': { 'expected': 'SKIP', 'actual': 'SKIP', }, 'unexpected_pass.html': { 'expected': 'FAIL', 'actual': 'PASS', 'is_unexpected': True }, }, }, step_name=WEBVIEW_WPT_STEP + ' (with patch)'), step_name=WEBVIEW_WPT_STEP + ' (with patch)') # Add Results for Chrome host.results_fetcher.set_results( Build('MOCK Android Pie', 101), WebTestResults( { 'tests': { 'jkl.html': { 'expected': 'PASS', 'actual': 'FAIL', 'is_unexpected': True, }, 'chrome_only.html': { 'expected': 'PASS', 'actual': 'CRASH CRASH TIMEOUT', 'is_unexpected': True, }, 'disabled.html': { 'expected': 'SKIP', 'actual': 'SKIP', 'is_unexpected': True, }, 'unexpected_pass.html': { 'expected': 'FAIL', 'actual': 'PASS', 'is_unexpected': True }, }, }, step_name=CHROME_ANDROID_WPT_STEP + ' (with patch)'), step_name=CHROME_ANDROID_WPT_STEP + ' (with patch)') updater = AndroidWPTExpectationsUpdater(host, [ '-vvv', '--clean-up-test-expectations', '--clean-up-affected-tests-only', '--include-unexpected-pass', '--android-product', ANDROID_WEBLAYER, '--android-product', CHROME_ANDROID, '--android-product', ANDROID_WEBVIEW ]) def _git_command_return_val(cmd): if '--diff-filter=D' in cmd: return 'external/wpt/ghi.html' if '--diff-filter=R' in cmd: return 'C external/wpt/van.html external/wpt/wagon.html' return '' updater.git_cl = MockGitCL( host, { Build('MOCK Android Weblayer - Pie', 123): TryJobStatus('COMPLETED', 'FAILURE'), Build('MOCK Android Pie', 101): TryJobStatus('COMPLETED', 'FAILURE') }) updater.git.run = _git_command_return_val updater._relative_to_web_test_dir = lambda test_path: test_path # Run command updater.run() # Check expectations for weblayer content = host.filesystem.read_text_file( PRODUCTS_TO_EXPECTATION_FILE_PATHS[ANDROID_WEBLAYER]) self.assertEqual(content, ( '# results: [ Failure Crash Timeout]\n' '\n' '# Add untriaged failures in this block\n' 'crbug.com/1050754 external/wpt/abc.html [ Crash Failure Timeout ]\n' 'crbug.com/1050754 external/wpt/cat.html [ Failure ]\n' 'crbug.com/1050754 external/wpt/def.html [ Crash ]\n' 'external/wpt/dog.html [ Crash Timeout ]\n' 'crbug.com/1111111 external/wpt/jkl.html [ Failure ]\n' 'crbug.com/1050754 external/wpt/unexpected_pass.html [ Failure Pass ]\n' 'crbug.com/6789043 external/wpt/wagon.html [ Failure ]\n' 'crbug.com/1050754 external/wpt/weblayer_only.html [ Failure Crash ]\n' 'external/wpt/www.html [ Crash Failure ]\n' '\n' '# This comment will not be deleted\n' 'crbug.com/111111 external/wpt/hello_world.html [ Crash ]\n')) # Check expectations for webview content = host.filesystem.read_text_file( PRODUCTS_TO_EXPECTATION_FILE_PATHS[ANDROID_WEBVIEW]) self.assertEqual(content, ( '# results: [ Failure Crash Timeout]\n' '\n' '# Add untriaged failures in this block\n' 'external/wpt/abc.html [ Failure ]\n' 'crbug.com/1050754 external/wpt/cat.html [ Crash Failure Timeout ]\n' 'crbug.com/1050754 external/wpt/def.html [ Crash ]\n' 'external/wpt/dog.html [ Crash Timeout ]\n' 'crbug.com/1111111 external/wpt/jkl.html [ Failure ]\n' 'crbug.com/1050754 external/wpt/unexpected_pass.html [ Failure Pass ]\n' 'crbug.com/6789043 external/wpt/wagon.html [ Failure ]\n' 'crbug.com/1050754 external/wpt/webview_only.html [ Timeout ]\n' 'external/wpt/www.html [ Crash Failure ]\n' '\n' '# This comment will not be deleted\n' 'crbug.com/111111 external/wpt/hello_world.html [ Crash ]\n')) # Check expectations chrome content = host.filesystem.read_text_file( PRODUCTS_TO_EXPECTATION_FILE_PATHS[CHROME_ANDROID]) self.assertEqual(content, ( '# results: [ Failure Crash Timeout]\n' '\n' '# Add untriaged failures in this block\n' 'external/wpt/abc.html [ Failure ]\n' 'crbug.com/1050754 external/wpt/cat.html [ Failure ]\n' 'crbug.com/1050754 external/wpt/chrome_only.html [ Crash Timeout ]\n' 'crbug.com/1050754 external/wpt/def.html [ Crash ]\n' 'external/wpt/dog.html [ Crash Timeout ]\n' 'crbug.com/1111111 crbug.com/1050754' ' external/wpt/jkl.html [ Failure ]\n' 'crbug.com/1050754 external/wpt/unexpected_pass.html [ Failure Pass ]\n' 'crbug.com/6789043 external/wpt/wagon.html [ Failure ]\n' 'external/wpt/www.html [ Crash Failure ]\n' '\n' '# This comment will not be deleted\n' 'crbug.com/111111 external/wpt/hello_world.html [ Crash ]\n')) # Check disabled test file neverfix_content = host.filesystem.read_text_file( ANDROID_DISABLED_TESTS) self.assertEqual(neverfix_content, ( '# tags: [ android-weblayer android-webview chrome-android ]\n' '# results: [ Skip ]\n' '\n' '# Add untriaged disabled tests in this block\n' 'crbug.com/1050754 [ android-webview ] external/wpt/disabled.html [ Skip ]\n' 'crbug.com/1050754 [ chrome-android ] external/wpt/disabled.html [ Skip ]\n' 'crbug.com/1050754 [ android-weblayer ] external/wpt/disabled_weblayer_only.html [ Skip ]\n' ))
def test_run_single_platform_failure(self): """Tests the main run method in a case where one test fails on one platform.""" host = self.mock_host() # Fill in an initial value for TestExpectations expectations_path = \ host.port_factory.get().path_to_generic_test_expectations_file() host.filesystem.write_text_file( expectations_path, WPTExpectationsUpdater.MARKER_COMMENT + '\n') # Set up fake try job results. updater = WPTExpectationsUpdater(host) updater.git_cl = MockGitCL( updater.host, { Build('MOCK Try Mac10.10', 333): TryJobStatus('COMPLETED', 'FAILURE'), Build('MOCK Try Mac10.11', 111): TryJobStatus('COMPLETED', 'SUCCESS'), Build('MOCK Try Trusty', 222): TryJobStatus('COMPLETED', 'SUCCESS'), Build('MOCK Try Precise', 333): TryJobStatus('COMPLETED', 'SUCCESS'), Build('MOCK Try Win10', 444): TryJobStatus('COMPLETED', 'SUCCESS'), Build('MOCK Try Win7', 555): TryJobStatus('COMPLETED', 'SUCCESS'), }) # Set up failing results for one try bot. It shouldn't matter what # results are for the other builders since we shouldn't need to even # fetch results, since the try job status already tells us that all # of the tests passed. host.results_fetcher.set_results( Build('MOCK Try Mac10.10', 333), WebTestResults({ 'tests': { 'external': { 'wpt': { 'test': { 'path.html': { 'expected': 'PASS', 'actual': 'TIMEOUT', 'is_unexpected': True, } } } } } })) self.assertEqual(0, updater.run()) # Results are only fetched for failing builds. self.assertEqual(host.results_fetcher.fetched_builds, [ BuilderStep(Build('MOCK Try Mac10.10', 333), 'blink_web_tests (with patch)') ]) self.assertEqual( host.filesystem.read_text_file(expectations_path), '# ====== New tests from wpt-importer added here ======\n' 'crbug.com/626703 [ Mac10.10 ] external/wpt/test/path.html [ Timeout ]\n' )
def testCleanupAndUpdateTestExpectationsForAll(self): # Full integration test for expectations cleanup and update # using builder results. raw_android_expectations = ( '# results: [ Failure Crash Timeout]\n' '\n' 'crbug.com/1000754 external/wpt/foo1.html [ Failure ]\n' 'crbug.com/1000754 external/wpt/foo2.html [ Failure ]\n' 'crbug.com/1000754 external/wpt/bar.html [ Failure ]\n' '\n' '# Add untriaged failures in this block\n' '\n' '# This comment will not be deleted\n') host = self._setup_host(raw_android_expectations) # Add results for Weblayer host.results_fetcher.set_results( Build('MOCK Android Weblayer - Pie', 123), WebTestResults({ 'tests': { 'disabled_weblayer_only.html': { 'expected': 'SKIP', 'actual': 'SKIP', 'is_unexpected': True, }, 'bar.html': { 'expected': 'FAIL', 'actual': 'CRASH', 'is_unexpected': True }, }, }, step_name=WEBLAYER_WPT_STEP + ' (with patch)'), step_name=WEBLAYER_WPT_STEP + ' (with patch)') updater = AndroidWPTExpectationsUpdater( host, ['-vvv', '--clean-up-test-expectations', '--clean-up-affected-tests-only', '--include-unexpected-pass', '--android-product', ANDROID_WEBLAYER]) def _git_command_return_val(cmd): if '--diff-filter=D' in cmd: return 'external/wpt/foo2.html' if '--diff-filter=R' in cmd: return 'C\texternal/wpt/foo1.html\texternal/wpt/foo3.html' if '--diff-filter=M' in cmd: return 'external/wpt/bar.html' return '' updater.git_cl = MockGitCL(host, { Build('MOCK Android Weblayer - Pie', 123): TryJobStatus('COMPLETED', 'FAILURE')}) updater.git.run = _git_command_return_val updater._relative_to_web_test_dir = lambda test_path: test_path # Run command updater.run() # Check expectations for weblayer content = host.filesystem.read_text_file( PRODUCTS_TO_EXPECTATION_FILE_PATHS[ANDROID_WEBLAYER]) _new_expectations = ( '# results: [ Failure Crash Timeout]\n' '\n' 'crbug.com/1000754 external/wpt/foo3.html [ Failure ]\n' '\n' '# Add untriaged failures in this block\n' 'crbug.com/1050754 external/wpt/bar.html [ Crash ]\n' '\n' '# This comment will not be deleted\n') self.assertEqual(content, _new_expectations) # Check disabled test file neverfix_content = host.filesystem.read_text_file(ANDROID_DISABLED_TESTS) self.assertEqual( neverfix_content, ('# tags: [ android-weblayer android-webview chrome-android ]\n' '# results: [ Skip ]\n' '\n' '# Add untriaged disabled tests in this block\n' 'crbug.com/1050754 [ android-webview ] external/wpt/disabled.html [ Skip ]\n' 'crbug.com/1050754 [ android-weblayer ] external/wpt/disabled_weblayer_only.html [ Skip ]\n'))
def test_get_failing_results_dict_webdriver_failing_results_(self): host = self.mock_host() host.results_fetcher.set_results( Build('MOCK Try Trusty', 123), WebTestResults({ 'tests': { 'external': { 'wpt': { 'x': { 'failing-test.html': { 'expected': 'PASS', 'actual': 'IMAGE', 'is_unexpected': True, }, }, }, }, }, })) host.results_fetcher.set_webdriver_test_results( Build('MOCK Try Trusty', 123), "tryserver.blink", WebTestResults({ 'tests': { 'external': { 'wpt': { 'y': { 'webdriver-fail.html': { 'expected': 'PASS', 'actual': 'FAIL', 'is_unexpected': True, }, }, }, }, }, })) updater = WPTExpectationsUpdater(host) results_dict = updater.get_failing_results_dict( Build('MOCK Try Trusty', 123)) self.assertEqual(len(results_dict.keys()), 2) self.assertEqual( results_dict, { 'external/wpt/x/failing-test.html': { 'test-linux-trusty': SimpleTestResult( actual='IMAGE', expected='PASS', bug='crbug.com/626703', ), }, 'external/wpt/y/webdriver-fail.html': { 'test-linux-trusty': SimpleTestResult( actual='FAIL', expected='PASS', bug='crbug.com/626703', ), }, })
def testUpdateTestExpectationsForWeblayer(self): raw_android_expectations = ( '# results: [ Failure Crash Timeout]\n' '\n' 'crbug.com/1000754 external/wpt/foo.html [ Failure ]\n' '\n' '# Add untriaged failures in this block\n' 'crbug.com/1050754 external/wpt/bar.html [ Failure ]\n' '\n' '# This comment will not be deleted\n') host = self._setup_host(raw_android_expectations) host.results_fetcher.set_results( Build('MOCK Android Weblayer - Pie', 123), WebTestResults({ 'tests': { # A test result covered by default expectation 'new1.html': { 'expected': 'PASS', 'actual': 'FAIL', 'is_unexpected': True, }, # A test result covered by baseline 'new2.html': { 'expected': 'PASS', 'actual': 'FAIL', 'is_unexpected': True, }, # A new test case 'new3.html': { 'expected': 'PASS', 'actual': 'CRASH CRASH FAIL', 'is_unexpected': True, }, }, }, step_name=WEBLAYER_WPT_STEP + ' (with patch)'), step_name=WEBLAYER_WPT_STEP + ' (with patch)') updater = AndroidWPTExpectationsUpdater( host, ['-vvv', '--android-product', ANDROID_WEBLAYER, '--include-unexpected-pass']) updater.git_cl = MockGitCL(host, { Build('MOCK Android Weblayer - Pie', 123): TryJobStatus('COMPLETED', 'FAILURE')}) # Run command updater.run() # Get new expectations content = host.filesystem.read_text_file( PRODUCTS_TO_EXPECTATION_FILE_PATHS[ANDROID_WEBLAYER]) _new_expectations = ( '# results: [ Failure Crash Timeout]\n' '\n' 'crbug.com/1000754 external/wpt/foo.html [ Failure ]\n' '\n' '# Add untriaged failures in this block\n' 'crbug.com/1050754 external/wpt/bar.html [ Failure ]\n' 'crbug.com/1050754 external/wpt/new3.html [ Crash Failure ]\n' '\n' '# This comment will not be deleted\n') self.assertEqual(content, _new_expectations) # Check that ANDROID_DISABLED_TESTS expectation files were not changed self.assertEqual( self._raw_android_never_fix_tests, host.filesystem.read_text_file(ANDROID_DISABLED_TESTS))
def testUpdateTestExpectationsForWebview(self): host = self._setup_host() host.results_fetcher.set_results( Build('MOCK Android Pie', 123), WebTestResults( { 'tests': { 'abc.html': { 'expected': 'PASS', 'actual': 'CRASH TIMEOUT', 'is_unexpected': True, }, 'jkl.html': { 'expected': 'PASS', 'actual': 'FAIL', 'is_unexpected': True, }, 'cat.html': { 'expected': 'PASS', 'actual': 'CRASH CRASH TIMEOUT', 'is_unexpected': True, }, 'unexpected_pass.html': { 'expected': 'FAIL', 'actual': 'PASS', 'is_unexpected': True }, 'dog.html': { 'expected': 'SKIP', 'actual': 'SKIP', 'is_unexpected': True, }, }, }, step_name=WEBVIEW_WPT_STEP + ' (with patch)'), step_name=WEBVIEW_WPT_STEP + ' (with patch)') updater = AndroidWPTExpectationsUpdater(host, [ '-vvv', '--android-product', ANDROID_WEBVIEW, '--clean-up-test-expectations', '--clean-up-affected-tests-only', '--include-unexpected-pass' ]) updater.git_cl = MockGitCL(host, { Build('MOCK Android Pie', 123): TryJobStatus('COMPLETED', 'FAILURE') }) # Run command updater.run() # Get new expectations content = host.filesystem.read_text_file( PRODUCTS_TO_EXPECTATION_FILE_PATHS[ANDROID_WEBVIEW]) self.assertEqual(content, ( '# results: [ Failure Crash Timeout]\n' '\n' '# Add untriaged failures in this block\n' 'crbug.com/1050754 external/wpt/abc.html [ Crash Failure Timeout ]\n' 'crbug.com/1050754 external/wpt/cat.html [ Crash Failure Timeout ]\n' 'crbug.com/1050754 external/wpt/def.html [ Crash ]\n' 'external/wpt/dog.html [ Crash Timeout ]\n' 'crbug.com/1050754 external/wpt/ghi.html [ Timeout ]\n' 'crbug.com/1111111 crbug.com/1050754' ' external/wpt/jkl.html [ Failure ]\n' 'crbug.com/1050754 external/wpt/unexpected_pass.html [ Failure Pass ]\n' 'crbug.com/6789043 external/wpt/van.html [ Failure ]\n' 'external/wpt/www.html [ Crash Failure ]\n' '\n' '# This comment will not be deleted\n' 'crbug.com/111111 external/wpt/hello_world.html [ Crash ]\n')) neverfix_content = host.filesystem.read_text_file( ANDROID_DISABLED_TESTS) self.assertEqual(neverfix_content, ( '# tags: [ android-weblayer android-webview chrome-android ]\n' '# results: [ Skip ]\n' '\n' '# Add untriaged disabled tests in this block\n' 'crbug.com/1050754 [ android-webview ] external/wpt/disabled.html [ Skip ]\n' 'crbug.com/1050754 [ android-webview ] external/wpt/dog.html [ Skip ]\n' )) # check that chrome android's expectation file was not modified # since the same bot is used to update chrome android & webview # expectations self.assertEqual( host.filesystem.read_text_file( PRODUCTS_TO_EXPECTATION_FILE_PATHS[CHROME_ANDROID]), self._raw_android_expectations) # Check logs logs = ''.join(self.logMessages()).lower() self.assertNotIn(WEBLAYER_WPT_STEP, logs) self.assertNotIn(CHROME_ANDROID_WPT_STEP, logs) # Check that weblayer and chrome expectation files were not changed self.assertEqual( self._raw_android_expectations, host.filesystem.read_text_file( PRODUCTS_TO_EXPECTATION_FILE_PATHS[CHROME_ANDROID])) self.assertEqual( self._raw_android_expectations, host.filesystem.read_text_file( PRODUCTS_TO_EXPECTATION_FILE_PATHS[ANDROID_WEBLAYER]))
def test_has_failing_try_results_with_failing_results(self): self.assertTrue( GitCL.some_failed({ Build('some-builder', 1): TryJobStatus('COMPLETED', 'FAILURE'), }))