def _assert_expectations(self, test_data, expectations_string, only_ignore_very_flaky): results_json = self._results_json_from_test_data(test_data) expectations = bot_test_expectations.BotTestExpectations( results_json, BuilderList({}), set('test')) self.assertEqual(expectations.flakes_by_path(only_ignore_very_flaky), expectations_string)
def setUp(self): host = MockBlinkTool() host.port_factory = MockPortFactory(host) port = host.port_factory.get() base_dir = port.web_tests_dir() host.filesystem.write_text_file(base_dir + '/a/x.html', '<html>') host.filesystem.write_text_file(base_dir + '/a/y.html', '<html>') host.filesystem.write_text_file(base_dir + '/a/z.html', '<html>') host.builders = BuilderList({ 'MOCK Mac10.12': { 'port_name': 'test-mac-mac10.12', 'specifiers': ['Mac10.12', 'Release'] }, 'MOCK Trusty': { 'port_name': 'test-linux-trusty', 'specifiers': ['Trusty', 'Release'] }, 'MOCK Win10': { 'port_name': 'test-win-win10', 'specifiers': ['Win10', 'Release'] }, 'some-wpt-bot': { 'port_name': 'linux-trusty', 'specifiers': ['Trusty', 'Release'] }, }) self.host = host
def test_filtered_all_results_by_path(self): test_data = { 'buildNumbers': [1, 2 , 3, 4, 5, 6, 7], 'tests': { 'foo': { 'fail_filtered.html': {'results': [[4, 'P'], [1, 'F'], [1, 'C'], [1, 'P']]}, 'fail_not_filtered.html': {'results': [[3, 'P'], [2, 'F'], [2, 'P']]}, } } } filter_data = { 'buildNumbers': [3, 4, 5, 6, 8], 'num_failures_by_type' : { 'PASS' : [0, 0, 1, 1, 1, 0, 1] } } results_json = self._results_json_from_test_data(test_data) results_filter = self._results_filter_from_test_data(filter_data) expectations = bot_test_expectations.BotTestExpectations(results_json, BuilderList({}), set('test'), results_filter) results_by_path = expectations.all_results_by_path() expected_output = { 'foo/fail_filtered.html': ['PASS'], 'foo/fail_not_filtered.html': ['FAIL', 'PASS'], } self.assertEqual(results_by_path, expected_output)
def test_all_results_by_path(self): test_data = { 'tests': { 'foo': { 'multiple_pass.html': { 'results': [[4, 'P'], [1, 'P'], [2, 'P']] }, 'fail.html': { 'results': [[2, 'F']] }, 'all_types.html': { 'results': [[1, 'C'], [2, 'F'], [1, 'N'], [1, 'P'], [1, 'T'], [1, 'Y'], [10, 'X']] }, 'not_run.html': { 'results': [] }, } } } results_json = self._results_json_from_test_data(test_data) expectations = bot_test_expectations.BotTestExpectations( results_json, BuilderList({}), set('test')) results_by_path = expectations.all_results_by_path() expected_output = { 'foo/multiple_pass.html': ['PASS'], 'foo/fail.html': ['FAIL'], 'foo/all_types.html': ['CRASH', 'FAIL', 'PASS', 'TIMEOUT'] } self.assertEqual(results_by_path, expected_output)
def _setup_host(self): """Returns a mock host with fake values set up for testing.""" self.set_logging_level(logging.DEBUG) host = MockHost() host.port_factory = MockPortFactory(host) # Set up a fake list of try builders. host.builders = BuilderList({ 'MOCK Try Precise': { 'port_name': 'test-linux-precise', 'specifiers': ['Precise', 'Release'], 'is_try_builder': True, }, 'MOCK Android Weblayer - Pie': { 'port_name': 'test-android-pie', 'specifiers': ['Precise', 'Release', 'anDroid', 'android_Weblayer'], 'is_try_builder': True, }, 'MOCK Android Pie': { 'port_name': 'test-android-pie', 'specifiers': ['Precise', 'Release', 'anDroid', 'Android_Webview', 'Chrome_Android'], 'is_try_builder': True, }, }) # Write dummy expectations for path in PRODUCTS_TO_EXPECTATION_FILE_PATHS.values(): host.filesystem.write_text_file( path, self._raw_android_expectations) return host
def mock_host(self): """Returns a mock host with fake values set up for testing.""" host = MockHost() host.port_factory = MockPortFactory(host) # Set up a fake list of try builders. host.builders = BuilderList({ 'MOCK Try Mac10.10': { 'port_name': 'test-mac-mac10.10', 'specifiers': ['Mac10.10', 'Release'], 'is_try_builder': True, }, 'MOCK Try Mac10.11': { 'port_name': 'test-mac-mac10.11', 'specifiers': ['Mac10.11', 'Release'], 'is_try_builder': True, }, 'MOCK Try Trusty': { 'port_name': 'test-linux-trusty', 'specifiers': ['Trusty', 'Release'], 'is_try_builder': True, }, 'MOCK Try Precise': { 'port_name': 'test-linux-precise', 'specifiers': ['Precise', 'Release'], 'is_try_builder': True, }, 'MOCK Try Win10': { 'port_name': 'test-win-win10', 'specifiers': ['Win10', 'Release'], 'is_try_builder': True, }, 'MOCK Try Win7': { 'port_name': 'test-win-win7', 'specifiers': ['Win7', 'Release'], 'is_try_builder': True, }, }) # Write a dummy manifest file, describing what tests exist. host.filesystem.write_text_file( host.port_factory.get().layout_tests_dir() + '/external/wpt/MANIFEST.json', json.dumps({ 'items': { 'reftest': { 'reftest.html': [['/reftest.html', [['/reftest-ref.html', '==']], {}]] }, 'testharness': { 'test/path.html': [['/test/path.html', {}]], 'test/zzzz.html': [['/test/zzzz.html', {}]], }, 'manual': { 'x-manual.html': [['/x-manual.html', {}]], }, }, })) return host
def __init__(self, log_executive=False, web=None, git=None, os_name=None, os_version=None, time_return_val=123): super(MockHost, self).__init__( log_executive=log_executive, os_name=os_name, os_version=os_version, time_return_val=time_return_val) add_unit_tests_to_mock_filesystem(self.filesystem) self._add_base_manifest_to_mock_filesystem(self.filesystem) self.web = web or MockWeb() self._git = git self.buildbot = MockBuildBot() # Note: We're using a real PortFactory here. Tests which don't wish to depend # on the list of known ports should override this with a MockPortFactory. self.port_factory = PortFactory(self) self.builders = BuilderList({ 'Fake Test Win10': { 'port_name': 'win-win10', 'specifiers': ['Win10', 'Release'] }, 'Fake Test Linux': { 'port_name': 'linux-trusty', 'specifiers': ['Trusty', 'Release'] }, 'Fake Test Linux (dbg)': { 'port_name': 'linux-trusty', 'specifiers': ['Trusty', 'Debug'] }, 'Fake Test Mac10.12': { 'port_name': 'mac-mac10.12', 'specifiers': ['Mac10.12', 'Release'], 'is_try_builder': True, }, 'fake_blink_try_linux': { 'port_name': 'linux-trusty', 'specifiers': ['Trusty', 'Release'], 'is_try_builder': True, }, 'fake_blink_try_win': { 'port_name': 'win-win10', 'specifiers': ['Win10', 'Release'], 'is_try_builder': True, }, 'android_blink_rel': { 'bucket': 'master.tryserver.chromium.android', 'port_name': 'android-kitkat', 'specifiers': ['KitKat', 'Release'], 'is_try_builder': True, }, })
def fake_builder_list(self): return BuilderList({ 'Dummy builder name': { 'master': 'dummy.master', 'port_name': 'dummy-port', 'specifiers': ['dummy', 'release'], }, })
def _define_builders(self, builders_dict): """Defines the available builders for the test. Args: builders_dict: A dictionary containing builder names to their attributes, see BuilderList.__init__ for the format. """ self._host.builders = BuilderList(builders_dict)
def sample_builder_list(): return BuilderList({ 'some-wpt-bot': { 'port_name': 'port-c', 'specifiers': ['C', 'Release'], 'is_try_builder': True, }, 'Blink A': { 'port_name': 'port-a', 'specifiers': ['A', 'Release'] }, 'Blink B': { 'port_name': 'port-b', 'specifiers': ['B', 'Release'] }, 'Blink B (dbg)': { 'port_name': 'port-b', 'specifiers': ['B', 'Debug'] }, 'Blink C (dbg)': { 'port_name': 'port-c', 'specifiers': ['C', 'Release'] }, 'Try A': { 'port_name': 'port-a', 'specifiers': ['A', 'Release'], 'is_try_builder': True }, 'Try B': { 'port_name': 'port-b', 'specifiers': ['B', 'Release'], 'is_try_builder': True }, 'CQ Try A': { 'bucket': 'bucket.a', 'port_name': 'port-a', 'specifiers': ['A', 'Release'], 'is_try_builder': True, 'is_cq_builder': True }, 'CQ Try B': { 'bucket': 'bucket.b', 'port_name': 'port-b', 'specifiers': ['B', 'Release'], 'is_try_builder': True, 'is_cq_builder': True }, 'CQ Try C': { 'bucket': 'bucket.c', 'port_name': 'port-c', 'specifiers': ['c', 'Release'], 'is_try_builder': True, 'is_cq_builder': True, 'master': "luci", 'has_webdriver_tests': True }, })
def test_get_from_builder_name(self): host = MockHost() host.builders = BuilderList({ 'My Fake Mac10.12 Builder': { 'port_name': 'mac-mac10.12', 'specifiers': ['Mac10.12', 'Release'], } }) self.assertEqual(factory.PortFactory(host).get_from_builder_name('My Fake Mac10.12 Builder').name(), 'mac-mac10.12')
def fake_builders_list(): return BuilderList({ 'foo-builder': { 'port_name': 'dummy-port', 'specifiers': ['Linux', 'Release'] }, 'bar-builder': { 'port_name': 'dummy-port', 'specifiers': ['Mac', 'Debug'] }, })
def _assert_is_flaky(self, results_string, should_be_flaky, only_ignore_very_flaky, expected=None): results_json = self._results_json_from_test_data({}) expectations = bot_test_expectations.BotTestExpectations(results_json, BuilderList({}), set('test')) results_entry = self._results_from_string(results_string) if expected: results_entry[bot_test_expectations.ResultsJSON.EXPECTATIONS_KEY] = expected num_actual_results = len(expectations._flaky_types_in_results(results_entry, only_ignore_very_flaky)) if should_be_flaky: self.assertGreater(num_actual_results, 1) else: self.assertLessEqual(num_actual_results, 1)
def fake_builder_list(self): return BuilderList({ 'Dummy builder name': { 'master': 'dummy.master', 'port_name': 'dummy-port', 'specifiers': ['dummy', 'release'], }, 'Dummy tryserver builder name': { 'master': 'tryserver.dummy.master', 'port_name': 'dummy-port', 'specifiers': ['dummy', 'release'], "is_try_builder": True }, })
def setUp(self): self.host = MockHost() self.fs = MockFileSystem() self.host.filesystem = self.fs # TODO(robertma): Even though we have mocked the builder list (and hence # all_port_names), we are still relying on the knowledge of currently # configured ports and their fallback order. Ideally, we should improve # MockPortFactory and use it. self.host.builders = BuilderList({ 'Fake Test Win10': { 'port_name': 'win-win10', 'specifiers': ['Win10', 'Release'] }, 'Fake Test Linux': { 'port_name': 'linux-trusty', 'specifiers': ['Trusty', 'Release'] }, 'Fake Test Mac10.15': { 'port_name': 'mac-mac10.15', 'specifiers': ['Mac10.15', 'Release'] }, 'Fake Test Mac10.14': { 'port_name': 'mac-mac10.14', 'specifiers': ['Mac10.14', 'Release'] }, 'Fake Test Mac10.13': { 'port_name': 'mac-mac10.13', 'specifiers': ['Mac10.13', 'Release'] }, 'Fake Test Mac10.12': { 'port_name': 'mac-mac10.12', 'specifiers': ['Mac10.12', 'Release'] }, 'Fake Test Mac10.11': { 'port_name': 'mac-mac10.11', 'specifiers': ['Mac10.11', 'Release'] }, 'Fake Test Mac10.10': { 'port_name': 'mac-mac10.10', 'specifiers': ['Mac10.10', 'Release'] }, }) # Note: this is a pre-assumption of the tests in this file. If this # assertion fails, port configurations are likely changed, and the # tests need to be adjusted accordingly. self.assertEqual(sorted(self.host.port_factory.all_port_names()), [ 'linux-trusty', 'mac-mac10.10', 'mac-mac10.11', 'mac-mac10.12', 'mac-mac10.13', 'mac-mac10.14', 'mac-mac10.15', 'win-win10' ])
def __init__(self): SystemHost.__init__(self) self.web = web.Web() self._git = None # Everything below this line is WebKit-specific and belongs on a higher-level object. self.buildbot = BuildBot() # FIXME: Unfortunately Port objects are currently the central-dispatch objects of the NRWT world. # In order to instantiate a port correctly, we have to pass it at least an executive, user, git, and filesystem # so for now we just pass along the whole Host object. # FIXME: PortFactory doesn't belong on this Host object if Port is going to have a Host (circular dependency). self.port_factory = PortFactory(self) self.builders = BuilderList.load_default_builder_list(self.filesystem)
def _setup_host(self): """Returns a mock host with fake values set up for testing.""" self.set_logging_level(logging.DEBUG) host = MockHost() host.port_factory = MockPortFactory(host) host.executive._output = '' # Set up a fake list of try builders. host.builders = BuilderList({ 'MOCK Try Precise': { 'port_name': 'test-linux-precise', 'specifiers': ['Precise', 'Release'], 'is_try_builder': True, }, 'MOCK Android Weblayer - Pie': { 'port_name': 'test-android-pie', 'specifiers': ['Precise', 'Release', 'anDroid', 'android_Weblayer'], 'is_try_builder': True, }, 'MOCK Android Pie': { 'port_name': 'test-android-pie', 'specifiers': ['Precise', 'Release', 'anDroid', 'Android_Webview', 'Chrome_Android'], 'is_try_builder': True, }, }) host.filesystem.write_text_file( host.port_factory.get().web_tests_dir() + '/external/' + BASE_MANIFEST_NAME, json.dumps({ 'items': { 'testharness': { 'ghi.html': ['abcdef123', [None, {}]], 'van.html': ['abcdef123', [None, {}]], }, }, })) # Write dummy expectations for path in PRODUCTS_TO_EXPECTATION_FILE_PATHS.values(): host.filesystem.write_text_file( path, self._raw_android_expectations) host.filesystem.write_text_file( ANDROID_DISABLED_TESTS, self._raw_android_never_fix_tests) return host
def sample_builder_list(): return BuilderList({ 'Blink A': { 'port_name': 'port-a', 'specifiers': ['A', 'Release'] }, 'Blink B': { 'port_name': 'port-b', 'specifiers': ['B', 'Release'] }, 'Blink B (dbg)': { 'port_name': 'port-b', 'specifiers': ['B', 'Debug'] }, 'Blink C (dbg)': { 'port_name': 'port-c', 'specifiers': ['C', 'Release'] }, 'Try A': { 'port_name': 'port-a', 'specifiers': ['A', 'Release'], 'is_try_builder': True }, 'Try B': { 'port_name': 'port-b', 'specifiers': ['B', 'Release'], 'is_try_builder': True }, 'CQ Try A': { 'bucket': 'bucket.a', 'port_name': 'port-a', 'specifiers': ['A', 'Release'], 'is_try_builder': True }, 'CQ Try B': { 'bucket': 'bucket.b', 'port_name': 'port-b', 'specifiers': ['B', 'Release'], 'is_try_builder': True }, })
def setUp(self): self.tool = MockBlinkTool() self.command = self.command_constructor() self.command._tool = self.tool # pylint: disable=protected-access self.tool.builders = BuilderList({ 'MOCK Mac10.10 (dbg)': {'port_name': 'test-mac-mac10.10', 'specifiers': ['Mac10.10', 'Debug']}, 'MOCK Mac10.10': {'port_name': 'test-mac-mac10.10', 'specifiers': ['Mac10.10', 'Release']}, 'MOCK Mac10.11 (dbg)': {'port_name': 'test-mac-mac10.11', 'specifiers': ['Mac10.11', 'Debug']}, 'MOCK Mac10.11 ASAN': {'port_name': 'test-mac-mac10.11', 'specifiers': ['Mac10.11', 'Release']}, 'MOCK Mac10.11': {'port_name': 'test-mac-mac10.11', 'specifiers': ['Mac10.11', 'Release']}, 'MOCK Precise': {'port_name': 'test-linux-precise', 'specifiers': ['Precise', 'Release']}, 'MOCK Trusty': {'port_name': 'test-linux-trusty', 'specifiers': ['Trusty', 'Release']}, 'MOCK Win10': {'port_name': 'test-win-win10', 'specifiers': ['Win10', 'Release']}, 'MOCK Win7 (dbg)': {'port_name': 'test-win-win7', 'specifiers': ['Win7', 'Debug']}, 'MOCK Win7 (dbg)(1)': {'port_name': 'test-win-win7', 'specifiers': ['Win7', 'Debug']}, 'MOCK Win7 (dbg)(2)': {'port_name': 'test-win-win7', 'specifiers': ['Win7', 'Debug']}, 'MOCK Win7': {'port_name': 'test-win-win7', 'specifiers': ['Win7', 'Release']}, }) self.mac_port = self.tool.port_factory.get_from_builder_name('MOCK Mac10.11') self.test_expectations_path = self.mac_port.path_to_generic_test_expectations_file() # This file must exist for Port classes to function properly. self._write('VirtualTestSuites', '[]') # Create some dummy tests (note _setup_mock_build_data uses the same test names). self._write('userscripts/first-test.html', 'Dummy test contents') self._write('userscripts/second-test.html', 'Dummy test contents') # In AbstractParallelRebaselineCommand._rebaseline_commands, a default port # object is gotten using self.tool.port_factory.get(), which is used to get # test paths -- and the layout tests directory may be different for the "test" # ports and real ports. Since only "test" ports are used in this class, # we can make the default port also a "test" port. self.original_port_factory_get = self.tool.port_factory.get test_port = self.tool.port_factory.get('test') def get_test_port(port_name=None, options=None, **kwargs): if not port_name: return test_port return self.original_port_factory_get(port_name, options, **kwargs) self.tool.port_factory.get = get_test_port
def test_fill_in_missing_results_prefers_build_with_same_os_type(self): self.tool.builders = BuilderList({ 'MOCK Foo12': { 'port_name': 'foo-foo12', 'specifiers': ['Foo12', 'Release'], 'is_try_builder': True, }, 'MOCK Foo45': { 'port_name': 'foo-foo45', 'specifiers': ['Foo45', 'Release'], 'is_try_builder': True, }, 'MOCK Bar3': { 'port_name': 'bar-bar3', 'specifiers': ['Bar3', 'Release'], 'is_try_builder': True, }, 'MOCK Bar4': { 'port_name': 'bar-bar4', 'specifiers': ['Bar4', 'Release'], 'is_try_builder': True, }, }) test_baseline_set = TestBaselineSet(self.tool) test_baseline_set.add('one/flaky-fail.html', Build('MOCK Foo12', 100)) test_baseline_set.add('one/flaky-fail.html', Build('MOCK Bar4', 200)) self.command.fill_in_missing_results(test_baseline_set) self.assertEqual( test_baseline_set.build_port_pairs('one/flaky-fail.html'), [ (Build('MOCK Foo12', 100), 'foo-foo12'), (Build('MOCK Bar4', 200), 'bar-bar4'), (Build('MOCK Foo12', 100), 'foo-foo45'), (Build('MOCK Bar4', 200), 'bar-bar3'), ]) self.assertLog([ 'INFO: For one/flaky-fail.html:\n', 'INFO: Using "MOCK Foo12" build 100 for foo-foo45.\n', 'INFO: Using "MOCK Bar4" build 200 for bar-bar3.\n', ])
def _assert_unexpected_results(self, test_data, expectations_string): results_json = self._results_json_from_test_data(test_data) expectations = bot_test_expectations.BotTestExpectations( results_json, BuilderList({}), set('test')) self.assertEqual(expectations.unexpected_results_by_path(), expectations_string)
def setUp(self): BaseTestCase.setUp(self) LoggingTestCase.setUp(self) builds = { Build('MOCK Try Win', 5000): TryJobStatus('COMPLETED', 'FAILURE'), Build('MOCK Try Mac', 4000): TryJobStatus('COMPLETED', 'FAILURE'), Build('MOCK Try Linux', 6000): TryJobStatus('COMPLETED', 'FAILURE'), } self.command.git_cl = MockGitCL(self.tool, builds) git = MockGit(filesystem=self.tool.filesystem, executive=self.tool.executive) git.changed_files = lambda **_: [ 'third_party/WebKit/LayoutTests/one/text-fail.html', 'third_party/WebKit/LayoutTests/one/flaky-fail.html', ] self.tool.git = lambda: git self.tool.builders = BuilderList({ 'MOCK Try Win': { 'port_name': 'test-win-win7', 'specifiers': ['Win7', 'Release'], 'is_try_builder': True, }, 'MOCK Try Linux': { 'port_name': 'test-linux-trusty', 'specifiers': ['Trusty', 'Release'], 'is_try_builder': True, }, 'MOCK Try Mac': { 'port_name': 'test-mac-mac10.11', 'specifiers': ['Mac10.11', 'Release'], 'is_try_builder': True, }, }) layout_test_results = LayoutTestResults({ 'tests': { 'one': { 'crash.html': { 'expected': 'PASS', 'actual': 'CRASH', 'is_unexpected': True }, 'expected-fail.html': { 'expected': 'FAIL', 'actual': 'IMAGE+TEXT' }, 'flaky-fail.html': { 'expected': 'PASS', 'actual': 'PASS TEXT', 'is_unexpected': True }, 'missing.html': { 'expected': 'PASS', 'actual': 'MISSING', 'is_unexpected': True }, 'slow-fail.html': { 'expected': 'SLOW', 'actual': 'TEXT', 'is_unexpected': True }, 'text-fail.html': { 'expected': 'PASS', 'actual': 'TEXT', 'is_unexpected': True }, 'unexpected-pass.html': { 'expected': 'FAIL', 'actual': 'PASS', 'is_unexpected': True }, }, 'two': { 'image-fail.html': { 'expected': 'PASS', 'actual': 'IMAGE', 'is_unexpected': True } }, }, }) for build in builds: self.tool.buildbot.set_results(build, layout_test_results) self.tool.buildbot.set_retry_sumary_json( build, json.dumps({ 'failures': [ 'one/flaky-fail.html', 'one/missing.html', 'one/slow-fail.html', 'one/text-fail.html', 'two/image-fail.html', ], 'ignored': [], })) # Write to the mock filesystem so that these tests are considered to exist. tests = [ 'one/flaky-fail.html', 'one/missing.html', 'one/slow-fail.html', 'one/text-fail.html', 'two/image-fail.html', ] for test in tests: path = self.mac_port.host.filesystem.join( self.mac_port.layout_tests_dir(), test) self._write(path, 'contents') self.mac_port.host.filesystem.write_text_file( '/test.checkout/LayoutTests/external/wpt/MANIFEST.json', '{}')
def test_constructor_validates_list(self): with self.assertRaises(AssertionError): BuilderList({'Blink A': {}}) with self.assertRaises(AssertionError): BuilderList({'Blink A': {'port_name': 'port-a', 'specifiers': []}})
def __init__(self): self.web = Web() self.builders = BuilderList.load_default_builder_list(FileSystem())
def setUp(self): BaseTestCase.setUp(self) LoggingTestCase.setUp(self) builds = { Build('MOCK Try Win', 5000): TryJobStatus('COMPLETED', 'FAILURE'), Build('MOCK Try Mac', 4000): TryJobStatus('COMPLETED', 'FAILURE'), Build('MOCK Try Linux', 6000): TryJobStatus('COMPLETED', 'FAILURE'), } self.command.git_cl = MockGitCL(self.tool, builds) git = MockGit(filesystem=self.tool.filesystem, executive=self.tool.executive) git.changed_files = lambda **_: [ RELATIVE_WEB_TESTS + 'one/text-fail.html', RELATIVE_WEB_TESTS + 'one/flaky-fail.html', ] self.tool.git = lambda: git self.tool.builders = BuilderList({ 'MOCK Try Win': { 'port_name': 'test-win-win7', 'specifiers': ['Win7', 'Release'], 'is_try_builder': True, }, 'MOCK Try Linux': { 'port_name': 'test-linux-trusty', 'specifiers': ['Trusty', 'Release'], 'is_try_builder': True, }, 'MOCK Try Mac': { 'port_name': 'test-mac-mac10.11', 'specifiers': ['Mac10.11', 'Release'], 'is_try_builder': True, }, }) web_test_results = WebTestResults({ 'tests': { 'one': { 'crash.html': { 'expected': 'PASS', 'actual': 'CRASH', 'is_unexpected': True, 'artifacts': { 'crash_log': ['crash.log'] } }, 'expected-fail.html': { 'expected': 'FAIL', 'actual': 'FAIL', 'artifacts': { 'expected_text': ['expected-fail-expected.txt'], 'actual_text': ['expected-fail-actual.txt'] } }, 'flaky-fail.html': { 'expected': 'PASS', 'actual': 'PASS FAIL', 'is_unexpected': True, 'artifacts': { 'expected_audio': ['flaky-fail-expected.wav'], 'actual_audio': ['flaky-fail-actual.wav'] } }, 'missing.html': { 'expected': 'PASS', 'actual': 'FAIL', 'is_unexpected': True, 'artifacts': { 'actual_image': ['missing-actual.png'] }, 'is_missing_image': True }, 'slow-fail.html': { 'expected': 'SLOW', 'actual': 'FAIL', 'is_unexpected': True, 'artifacts': { 'actual_text': ['slow-fail-actual.txt'], 'expected_text': ['slow-fail-expected.txt'] } }, 'text-fail.html': { 'expected': 'PASS', 'actual': 'FAIL', 'is_unexpected': True, 'artifacts': { 'actual_text': ['text-fail-actual.txt'], 'expected_text': ['text-fail-expected.txt'] } }, 'unexpected-pass.html': { 'expected': 'FAIL', 'actual': 'PASS', 'is_unexpected': True }, }, 'two': { 'image-fail.html': { 'expected': 'PASS', 'actual': 'FAIL', 'is_unexpected': True, 'artifacts': { 'actual_image': ['image-fail-actual.png'], 'expected_image': ['image-fail-expected.png'] } } }, }, }) for build in builds: self.tool.results_fetcher.set_results(build, web_test_results) self.tool.results_fetcher.set_retry_sumary_json( build, json.dumps({ 'failures': [ 'one/flaky-fail.html', 'one/missing.html', 'one/slow-fail.html', 'one/text-fail.html', 'two/image-fail.html', ], 'ignored': [], })) # Write to the mock filesystem so that these tests are considered to exist. tests = [ 'one/flaky-fail.html', 'one/missing.html', 'one/slow-fail.html', 'one/text-fail.html', 'two/image-fail.html', ] for test in tests: path = self.mac_port.host.filesystem.join( self.mac_port.web_tests_dir(), test) self._write(path, 'contents') self.mac_port.host.filesystem.write_text_file( '/test.checkout/web_tests/external/wpt/MANIFEST.json', '{}')