def _assert_expectations(self, test_data, expectations_string,
only_ignore_very_flaky):
results_json = self._results_json_from_test_data(test_data)
expectations = bot_test_expectations.BotTestExpectations(
results_json, BuilderList({}), set('test'))
self.assertEqual(expectations.flakes_by_path(only_ignore_very_flaky),
expectations_string)
def setUp(self):
host = MockBlinkTool()
host.port_factory = MockPortFactory(host)
port = host.port_factory.get()
base_dir = port.web_tests_dir()
host.filesystem.write_text_file(base_dir + '/a/x.html', '<html>')
host.filesystem.write_text_file(base_dir + '/a/y.html', '<html>')
host.filesystem.write_text_file(base_dir + '/a/z.html', '<html>')
host.builders = BuilderList({
'MOCK Mac10.12': {
'port_name': 'test-mac-mac10.12',
'specifiers': ['Mac10.12', 'Release']
},
'MOCK Trusty': {
'port_name': 'test-linux-trusty',
'specifiers': ['Trusty', 'Release']
},
'MOCK Win10': {
'port_name': 'test-win-win10',
'specifiers': ['Win10', 'Release']
},
'some-wpt-bot': {
'port_name': 'linux-trusty',
'specifiers': ['Trusty', 'Release']
},
})
self.host = host
def test_filtered_all_results_by_path(self):
test_data = {
'buildNumbers': [1, 2 , 3, 4, 5, 6, 7],
'tests': {
'foo': {
'fail_filtered.html': {'results': [[4, 'P'], [1, 'F'], [1, 'C'], [1, 'P']]},
'fail_not_filtered.html': {'results': [[3, 'P'], [2, 'F'], [2, 'P']]},
}
}
}
filter_data = {
'buildNumbers': [3, 4, 5, 6, 8],
'num_failures_by_type' : {
'PASS' : [0, 0, 1, 1, 1, 0, 1]
}
}
results_json = self._results_json_from_test_data(test_data)
results_filter = self._results_filter_from_test_data(filter_data)
expectations = bot_test_expectations.BotTestExpectations(results_json, BuilderList({}), set('test'), results_filter)
results_by_path = expectations.all_results_by_path()
expected_output = {
'foo/fail_filtered.html': ['PASS'],
'foo/fail_not_filtered.html': ['FAIL', 'PASS'],
}
self.assertEqual(results_by_path, expected_output)
def test_all_results_by_path(self):
test_data = {
'tests': {
'foo': {
'multiple_pass.html': {
'results': [[4, 'P'], [1, 'P'], [2, 'P']]
},
'fail.html': {
'results': [[2, 'F']]
},
'all_types.html': {
'results': [[1, 'C'], [2, 'F'], [1, 'N'], [1, 'P'],
[1, 'T'], [1, 'Y'], [10, 'X']]
},
'not_run.html': {
'results': []
},
}
}
}
results_json = self._results_json_from_test_data(test_data)
expectations = bot_test_expectations.BotTestExpectations(
results_json, BuilderList({}), set('test'))
results_by_path = expectations.all_results_by_path()
expected_output = {
'foo/multiple_pass.html': ['PASS'],
'foo/fail.html': ['FAIL'],
'foo/all_types.html': ['CRASH', 'FAIL', 'PASS', 'TIMEOUT']
}
self.assertEqual(results_by_path, expected_output)
def _setup_host(self):
"""Returns a mock host with fake values set up for testing."""
self.set_logging_level(logging.DEBUG)
host = MockHost()
host.port_factory = MockPortFactory(host)
# Set up a fake list of try builders.
host.builders = BuilderList({
'MOCK Try Precise': {
'port_name': 'test-linux-precise',
'specifiers': ['Precise', 'Release'],
'is_try_builder': True,
},
'MOCK Android Weblayer - Pie': {
'port_name': 'test-android-pie',
'specifiers': ['Precise', 'Release',
'anDroid', 'android_Weblayer'],
'is_try_builder': True,
},
'MOCK Android Pie': {
'port_name': 'test-android-pie',
'specifiers': ['Precise', 'Release', 'anDroid',
'Android_Webview', 'Chrome_Android'],
'is_try_builder': True,
},
})
# Write dummy expectations
for path in PRODUCTS_TO_EXPECTATION_FILE_PATHS.values():
host.filesystem.write_text_file(
path, self._raw_android_expectations)
return host
def mock_host(self):
"""Returns a mock host with fake values set up for testing."""
host = MockHost()
host.port_factory = MockPortFactory(host)
# Set up a fake list of try builders.
host.builders = BuilderList({
'MOCK Try Mac10.10': {
'port_name': 'test-mac-mac10.10',
'specifiers': ['Mac10.10', 'Release'],
'is_try_builder': True,
},
'MOCK Try Mac10.11': {
'port_name': 'test-mac-mac10.11',
'specifiers': ['Mac10.11', 'Release'],
'is_try_builder': True,
},
'MOCK Try Trusty': {
'port_name': 'test-linux-trusty',
'specifiers': ['Trusty', 'Release'],
'is_try_builder': True,
},
'MOCK Try Precise': {
'port_name': 'test-linux-precise',
'specifiers': ['Precise', 'Release'],
'is_try_builder': True,
},
'MOCK Try Win10': {
'port_name': 'test-win-win10',
'specifiers': ['Win10', 'Release'],
'is_try_builder': True,
},
'MOCK Try Win7': {
'port_name': 'test-win-win7',
'specifiers': ['Win7', 'Release'],
'is_try_builder': True,
},
})
# Write a dummy manifest file, describing what tests exist.
host.filesystem.write_text_file(
host.port_factory.get().layout_tests_dir() +
'/external/wpt/MANIFEST.json',
json.dumps({
'items': {
'reftest': {
'reftest.html':
[['/reftest.html', [['/reftest-ref.html', '==']], {}]]
},
'testharness': {
'test/path.html': [['/test/path.html', {}]],
'test/zzzz.html': [['/test/zzzz.html', {}]],
},
'manual': {
'x-manual.html': [['/x-manual.html', {}]],
},
},
}))
return host
def __init__(self,
log_executive=False,
web=None,
git=None,
os_name=None,
os_version=None,
time_return_val=123):
super(MockHost, self).__init__(
log_executive=log_executive,
os_name=os_name,
os_version=os_version,
time_return_val=time_return_val)
add_unit_tests_to_mock_filesystem(self.filesystem)
self._add_base_manifest_to_mock_filesystem(self.filesystem)
self.web = web or MockWeb()
self._git = git
self.buildbot = MockBuildBot()
# Note: We're using a real PortFactory here. Tests which don't wish to depend
# on the list of known ports should override this with a MockPortFactory.
self.port_factory = PortFactory(self)
self.builders = BuilderList({
'Fake Test Win10': {
'port_name': 'win-win10',
'specifiers': ['Win10', 'Release']
},
'Fake Test Linux': {
'port_name': 'linux-trusty',
'specifiers': ['Trusty', 'Release']
},
'Fake Test Linux (dbg)': {
'port_name': 'linux-trusty',
'specifiers': ['Trusty', 'Debug']
},
'Fake Test Mac10.12': {
'port_name': 'mac-mac10.12',
'specifiers': ['Mac10.12', 'Release'],
'is_try_builder': True,
},
'fake_blink_try_linux': {
'port_name': 'linux-trusty',
'specifiers': ['Trusty', 'Release'],
'is_try_builder': True,
},
'fake_blink_try_win': {
'port_name': 'win-win10',
'specifiers': ['Win10', 'Release'],
'is_try_builder': True,
},
'android_blink_rel': {
'bucket': 'master.tryserver.chromium.android',
'port_name': 'android-kitkat',
'specifiers': ['KitKat', 'Release'],
'is_try_builder': True,
},
})
def fake_builder_list(self):
return BuilderList({
'Dummy builder name': {
'master': 'dummy.master',
'port_name': 'dummy-port',
'specifiers': ['dummy', 'release'],
},
})
def _define_builders(self, builders_dict):
"""Defines the available builders for the test.
Args:
builders_dict: A dictionary containing builder names to their
attributes, see BuilderList.__init__ for the format.
"""
self._host.builders = BuilderList(builders_dict)
def sample_builder_list():
return BuilderList({
'some-wpt-bot': {
'port_name': 'port-c',
'specifiers': ['C', 'Release'],
'is_try_builder': True,
},
'Blink A': {
'port_name': 'port-a',
'specifiers': ['A', 'Release']
},
'Blink B': {
'port_name': 'port-b',
'specifiers': ['B', 'Release']
},
'Blink B (dbg)': {
'port_name': 'port-b',
'specifiers': ['B', 'Debug']
},
'Blink C (dbg)': {
'port_name': 'port-c',
'specifiers': ['C', 'Release']
},
'Try A': {
'port_name': 'port-a',
'specifiers': ['A', 'Release'],
'is_try_builder': True
},
'Try B': {
'port_name': 'port-b',
'specifiers': ['B', 'Release'],
'is_try_builder': True
},
'CQ Try A': {
'bucket': 'bucket.a',
'port_name': 'port-a',
'specifiers': ['A', 'Release'],
'is_try_builder': True,
'is_cq_builder': True
},
'CQ Try B': {
'bucket': 'bucket.b',
'port_name': 'port-b',
'specifiers': ['B', 'Release'],
'is_try_builder': True,
'is_cq_builder': True
},
'CQ Try C': {
'bucket': 'bucket.c',
'port_name': 'port-c',
'specifiers': ['c', 'Release'],
'is_try_builder': True,
'is_cq_builder': True,
'master': "luci",
'has_webdriver_tests': True
},
})
def test_get_from_builder_name(self):
host = MockHost()
host.builders = BuilderList({
'My Fake Mac10.12 Builder': {
'port_name': 'mac-mac10.12',
'specifiers': ['Mac10.12', 'Release'],
}
})
self.assertEqual(factory.PortFactory(host).get_from_builder_name('My Fake Mac10.12 Builder').name(),
'mac-mac10.12')
def fake_builders_list():
return BuilderList({
'foo-builder': {
'port_name': 'dummy-port',
'specifiers': ['Linux', 'Release']
},
'bar-builder': {
'port_name': 'dummy-port',
'specifiers': ['Mac', 'Debug']
},
})
def _assert_is_flaky(self, results_string, should_be_flaky, only_ignore_very_flaky, expected=None):
results_json = self._results_json_from_test_data({})
expectations = bot_test_expectations.BotTestExpectations(results_json, BuilderList({}), set('test'))
results_entry = self._results_from_string(results_string)
if expected:
results_entry[bot_test_expectations.ResultsJSON.EXPECTATIONS_KEY] = expected
num_actual_results = len(expectations._flaky_types_in_results(results_entry, only_ignore_very_flaky))
if should_be_flaky:
self.assertGreater(num_actual_results, 1)
else:
self.assertLessEqual(num_actual_results, 1)
def fake_builder_list(self):
return BuilderList({
'Dummy builder name': {
'master': 'dummy.master',
'port_name': 'dummy-port',
'specifiers': ['dummy', 'release'],
},
'Dummy tryserver builder name': {
'master': 'tryserver.dummy.master',
'port_name': 'dummy-port',
'specifiers': ['dummy', 'release'],
"is_try_builder": True
},
})
def setUp(self):
self.host = MockHost()
self.fs = MockFileSystem()
self.host.filesystem = self.fs
# TODO(robertma): Even though we have mocked the builder list (and hence
# all_port_names), we are still relying on the knowledge of currently
# configured ports and their fallback order. Ideally, we should improve
# MockPortFactory and use it.
self.host.builders = BuilderList({
'Fake Test Win10': {
'port_name': 'win-win10',
'specifiers': ['Win10', 'Release']
},
'Fake Test Linux': {
'port_name': 'linux-trusty',
'specifiers': ['Trusty', 'Release']
},
'Fake Test Mac10.15': {
'port_name': 'mac-mac10.15',
'specifiers': ['Mac10.15', 'Release']
},
'Fake Test Mac10.14': {
'port_name': 'mac-mac10.14',
'specifiers': ['Mac10.14', 'Release']
},
'Fake Test Mac10.13': {
'port_name': 'mac-mac10.13',
'specifiers': ['Mac10.13', 'Release']
},
'Fake Test Mac10.12': {
'port_name': 'mac-mac10.12',
'specifiers': ['Mac10.12', 'Release']
},
'Fake Test Mac10.11': {
'port_name': 'mac-mac10.11',
'specifiers': ['Mac10.11', 'Release']
},
'Fake Test Mac10.10': {
'port_name': 'mac-mac10.10',
'specifiers': ['Mac10.10', 'Release']
},
})
# Note: this is a pre-assumption of the tests in this file. If this
# assertion fails, port configurations are likely changed, and the
# tests need to be adjusted accordingly.
self.assertEqual(sorted(self.host.port_factory.all_port_names()), [
'linux-trusty', 'mac-mac10.10', 'mac-mac10.11', 'mac-mac10.12',
'mac-mac10.13', 'mac-mac10.14', 'mac-mac10.15', 'win-win10'
])
def __init__(self):
SystemHost.__init__(self)
self.web = web.Web()
self._git = None
# Everything below this line is WebKit-specific and belongs on a higher-level object.
self.buildbot = BuildBot()
# FIXME: Unfortunately Port objects are currently the central-dispatch objects of the NRWT world.
# In order to instantiate a port correctly, we have to pass it at least an executive, user, git, and filesystem
# so for now we just pass along the whole Host object.
# FIXME: PortFactory doesn't belong on this Host object if Port is going to have a Host (circular dependency).
self.port_factory = PortFactory(self)
self.builders = BuilderList.load_default_builder_list(self.filesystem)
def _setup_host(self):
"""Returns a mock host with fake values set up for testing."""
self.set_logging_level(logging.DEBUG)
host = MockHost()
host.port_factory = MockPortFactory(host)
host.executive._output = ''
# Set up a fake list of try builders.
host.builders = BuilderList({
'MOCK Try Precise': {
'port_name': 'test-linux-precise',
'specifiers': ['Precise', 'Release'],
'is_try_builder': True,
},
'MOCK Android Weblayer - Pie': {
'port_name': 'test-android-pie',
'specifiers': ['Precise', 'Release',
'anDroid', 'android_Weblayer'],
'is_try_builder': True,
},
'MOCK Android Pie': {
'port_name': 'test-android-pie',
'specifiers': ['Precise', 'Release', 'anDroid',
'Android_Webview', 'Chrome_Android'],
'is_try_builder': True,
},
})
host.filesystem.write_text_file(
host.port_factory.get().web_tests_dir() + '/external/' +
BASE_MANIFEST_NAME,
json.dumps({
'items': {
'testharness': {
'ghi.html': ['abcdef123', [None, {}]],
'van.html': ['abcdef123', [None, {}]],
},
},
}))
# Write dummy expectations
for path in PRODUCTS_TO_EXPECTATION_FILE_PATHS.values():
host.filesystem.write_text_file(
path, self._raw_android_expectations)
host.filesystem.write_text_file(
ANDROID_DISABLED_TESTS, self._raw_android_never_fix_tests)
return host
def sample_builder_list():
return BuilderList({
'Blink A': {
'port_name': 'port-a',
'specifiers': ['A', 'Release']
},
'Blink B': {
'port_name': 'port-b',
'specifiers': ['B', 'Release']
},
'Blink B (dbg)': {
'port_name': 'port-b',
'specifiers': ['B', 'Debug']
},
'Blink C (dbg)': {
'port_name': 'port-c',
'specifiers': ['C', 'Release']
},
'Try A': {
'port_name': 'port-a',
'specifiers': ['A', 'Release'],
'is_try_builder': True
},
'Try B': {
'port_name': 'port-b',
'specifiers': ['B', 'Release'],
'is_try_builder': True
},
'CQ Try A': {
'bucket': 'bucket.a',
'port_name': 'port-a',
'specifiers': ['A', 'Release'],
'is_try_builder': True
},
'CQ Try B': {
'bucket': 'bucket.b',
'port_name': 'port-b',
'specifiers': ['B', 'Release'],
'is_try_builder': True
},
})
def setUp(self):
self.tool = MockBlinkTool()
self.command = self.command_constructor()
self.command._tool = self.tool # pylint: disable=protected-access
self.tool.builders = BuilderList({
'MOCK Mac10.10 (dbg)': {'port_name': 'test-mac-mac10.10', 'specifiers': ['Mac10.10', 'Debug']},
'MOCK Mac10.10': {'port_name': 'test-mac-mac10.10', 'specifiers': ['Mac10.10', 'Release']},
'MOCK Mac10.11 (dbg)': {'port_name': 'test-mac-mac10.11', 'specifiers': ['Mac10.11', 'Debug']},
'MOCK Mac10.11 ASAN': {'port_name': 'test-mac-mac10.11', 'specifiers': ['Mac10.11', 'Release']},
'MOCK Mac10.11': {'port_name': 'test-mac-mac10.11', 'specifiers': ['Mac10.11', 'Release']},
'MOCK Precise': {'port_name': 'test-linux-precise', 'specifiers': ['Precise', 'Release']},
'MOCK Trusty': {'port_name': 'test-linux-trusty', 'specifiers': ['Trusty', 'Release']},
'MOCK Win10': {'port_name': 'test-win-win10', 'specifiers': ['Win10', 'Release']},
'MOCK Win7 (dbg)': {'port_name': 'test-win-win7', 'specifiers': ['Win7', 'Debug']},
'MOCK Win7 (dbg)(1)': {'port_name': 'test-win-win7', 'specifiers': ['Win7', 'Debug']},
'MOCK Win7 (dbg)(2)': {'port_name': 'test-win-win7', 'specifiers': ['Win7', 'Debug']},
'MOCK Win7': {'port_name': 'test-win-win7', 'specifiers': ['Win7', 'Release']},
})
self.mac_port = self.tool.port_factory.get_from_builder_name('MOCK Mac10.11')
self.test_expectations_path = self.mac_port.path_to_generic_test_expectations_file()
# This file must exist for Port classes to function properly.
self._write('VirtualTestSuites', '[]')
# Create some dummy tests (note _setup_mock_build_data uses the same test names).
self._write('userscripts/first-test.html', 'Dummy test contents')
self._write('userscripts/second-test.html', 'Dummy test contents')
# In AbstractParallelRebaselineCommand._rebaseline_commands, a default port
# object is gotten using self.tool.port_factory.get(), which is used to get
# test paths -- and the layout tests directory may be different for the "test"
# ports and real ports. Since only "test" ports are used in this class,
# we can make the default port also a "test" port.
self.original_port_factory_get = self.tool.port_factory.get
test_port = self.tool.port_factory.get('test')
def get_test_port(port_name=None, options=None, **kwargs):
if not port_name:
return test_port
return self.original_port_factory_get(port_name, options, **kwargs)
self.tool.port_factory.get = get_test_port
def test_fill_in_missing_results_prefers_build_with_same_os_type(self):
self.tool.builders = BuilderList({
'MOCK Foo12': {
'port_name': 'foo-foo12',
'specifiers': ['Foo12', 'Release'],
'is_try_builder': True,
},
'MOCK Foo45': {
'port_name': 'foo-foo45',
'specifiers': ['Foo45', 'Release'],
'is_try_builder': True,
},
'MOCK Bar3': {
'port_name': 'bar-bar3',
'specifiers': ['Bar3', 'Release'],
'is_try_builder': True,
},
'MOCK Bar4': {
'port_name': 'bar-bar4',
'specifiers': ['Bar4', 'Release'],
'is_try_builder': True,
},
})
test_baseline_set = TestBaselineSet(self.tool)
test_baseline_set.add('one/flaky-fail.html', Build('MOCK Foo12', 100))
test_baseline_set.add('one/flaky-fail.html', Build('MOCK Bar4', 200))
self.command.fill_in_missing_results(test_baseline_set)
self.assertEqual(
test_baseline_set.build_port_pairs('one/flaky-fail.html'),
[
(Build('MOCK Foo12', 100), 'foo-foo12'),
(Build('MOCK Bar4', 200), 'bar-bar4'),
(Build('MOCK Foo12', 100), 'foo-foo45'),
(Build('MOCK Bar4', 200), 'bar-bar3'),
])
self.assertLog([
'INFO: For one/flaky-fail.html:\n',
'INFO: Using "MOCK Foo12" build 100 for foo-foo45.\n',
'INFO: Using "MOCK Bar4" build 200 for bar-bar3.\n',
])
def _assert_unexpected_results(self, test_data, expectations_string):
results_json = self._results_json_from_test_data(test_data)
expectations = bot_test_expectations.BotTestExpectations(
results_json, BuilderList({}), set('test'))
self.assertEqual(expectations.unexpected_results_by_path(),
expectations_string)
def setUp(self):
BaseTestCase.setUp(self)
LoggingTestCase.setUp(self)
builds = {
Build('MOCK Try Win', 5000): TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Try Mac', 4000): TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Try Linux', 6000):
TryJobStatus('COMPLETED', 'FAILURE'),
}
self.command.git_cl = MockGitCL(self.tool, builds)
git = MockGit(filesystem=self.tool.filesystem,
executive=self.tool.executive)
git.changed_files = lambda **_: [
'third_party/WebKit/LayoutTests/one/text-fail.html',
'third_party/WebKit/LayoutTests/one/flaky-fail.html',
]
self.tool.git = lambda: git
self.tool.builders = BuilderList({
'MOCK Try Win': {
'port_name': 'test-win-win7',
'specifiers': ['Win7', 'Release'],
'is_try_builder': True,
},
'MOCK Try Linux': {
'port_name': 'test-linux-trusty',
'specifiers': ['Trusty', 'Release'],
'is_try_builder': True,
},
'MOCK Try Mac': {
'port_name': 'test-mac-mac10.11',
'specifiers': ['Mac10.11', 'Release'],
'is_try_builder': True,
},
})
layout_test_results = LayoutTestResults({
'tests': {
'one': {
'crash.html': {
'expected': 'PASS',
'actual': 'CRASH',
'is_unexpected': True
},
'expected-fail.html': {
'expected': 'FAIL',
'actual': 'IMAGE+TEXT'
},
'flaky-fail.html': {
'expected': 'PASS',
'actual': 'PASS TEXT',
'is_unexpected': True
},
'missing.html': {
'expected': 'PASS',
'actual': 'MISSING',
'is_unexpected': True
},
'slow-fail.html': {
'expected': 'SLOW',
'actual': 'TEXT',
'is_unexpected': True
},
'text-fail.html': {
'expected': 'PASS',
'actual': 'TEXT',
'is_unexpected': True
},
'unexpected-pass.html': {
'expected': 'FAIL',
'actual': 'PASS',
'is_unexpected': True
},
},
'two': {
'image-fail.html': {
'expected': 'PASS',
'actual': 'IMAGE',
'is_unexpected': True
}
},
},
})
for build in builds:
self.tool.buildbot.set_results(build, layout_test_results)
self.tool.buildbot.set_retry_sumary_json(
build,
json.dumps({
'failures': [
'one/flaky-fail.html',
'one/missing.html',
'one/slow-fail.html',
'one/text-fail.html',
'two/image-fail.html',
],
'ignored': [],
}))
# Write to the mock filesystem so that these tests are considered to exist.
tests = [
'one/flaky-fail.html',
'one/missing.html',
'one/slow-fail.html',
'one/text-fail.html',
'two/image-fail.html',
]
for test in tests:
path = self.mac_port.host.filesystem.join(
self.mac_port.layout_tests_dir(), test)
self._write(path, 'contents')
self.mac_port.host.filesystem.write_text_file(
'/test.checkout/LayoutTests/external/wpt/MANIFEST.json', '{}')
def test_constructor_validates_list(self):
with self.assertRaises(AssertionError):
BuilderList({'Blink A': {}})
with self.assertRaises(AssertionError):
BuilderList({'Blink A': {'port_name': 'port-a', 'specifiers': []}})
def __init__(self):
self.web = Web()
self.builders = BuilderList.load_default_builder_list(FileSystem())
def setUp(self):
BaseTestCase.setUp(self)
LoggingTestCase.setUp(self)
builds = {
Build('MOCK Try Win', 5000): TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Try Mac', 4000): TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Try Linux', 6000):
TryJobStatus('COMPLETED', 'FAILURE'),
}
self.command.git_cl = MockGitCL(self.tool, builds)
git = MockGit(filesystem=self.tool.filesystem,
executive=self.tool.executive)
git.changed_files = lambda **_: [
RELATIVE_WEB_TESTS + 'one/text-fail.html',
RELATIVE_WEB_TESTS + 'one/flaky-fail.html',
]
self.tool.git = lambda: git
self.tool.builders = BuilderList({
'MOCK Try Win': {
'port_name': 'test-win-win7',
'specifiers': ['Win7', 'Release'],
'is_try_builder': True,
},
'MOCK Try Linux': {
'port_name': 'test-linux-trusty',
'specifiers': ['Trusty', 'Release'],
'is_try_builder': True,
},
'MOCK Try Mac': {
'port_name': 'test-mac-mac10.11',
'specifiers': ['Mac10.11', 'Release'],
'is_try_builder': True,
},
})
web_test_results = WebTestResults({
'tests': {
'one': {
'crash.html': {
'expected': 'PASS',
'actual': 'CRASH',
'is_unexpected': True,
'artifacts': {
'crash_log': ['crash.log']
}
},
'expected-fail.html': {
'expected': 'FAIL',
'actual': 'FAIL',
'artifacts': {
'expected_text': ['expected-fail-expected.txt'],
'actual_text': ['expected-fail-actual.txt']
}
},
'flaky-fail.html': {
'expected': 'PASS',
'actual': 'PASS FAIL',
'is_unexpected': True,
'artifacts': {
'expected_audio': ['flaky-fail-expected.wav'],
'actual_audio': ['flaky-fail-actual.wav']
}
},
'missing.html': {
'expected': 'PASS',
'actual': 'FAIL',
'is_unexpected': True,
'artifacts': {
'actual_image': ['missing-actual.png']
},
'is_missing_image': True
},
'slow-fail.html': {
'expected': 'SLOW',
'actual': 'FAIL',
'is_unexpected': True,
'artifacts': {
'actual_text': ['slow-fail-actual.txt'],
'expected_text': ['slow-fail-expected.txt']
}
},
'text-fail.html': {
'expected': 'PASS',
'actual': 'FAIL',
'is_unexpected': True,
'artifacts': {
'actual_text': ['text-fail-actual.txt'],
'expected_text': ['text-fail-expected.txt']
}
},
'unexpected-pass.html': {
'expected': 'FAIL',
'actual': 'PASS',
'is_unexpected': True
},
},
'two': {
'image-fail.html': {
'expected': 'PASS',
'actual': 'FAIL',
'is_unexpected': True,
'artifacts': {
'actual_image': ['image-fail-actual.png'],
'expected_image': ['image-fail-expected.png']
}
}
},
},
})
for build in builds:
self.tool.results_fetcher.set_results(build, web_test_results)
self.tool.results_fetcher.set_retry_sumary_json(
build,
json.dumps({
'failures': [
'one/flaky-fail.html',
'one/missing.html',
'one/slow-fail.html',
'one/text-fail.html',
'two/image-fail.html',
],
'ignored': [],
}))
# Write to the mock filesystem so that these tests are considered to exist.
tests = [
'one/flaky-fail.html',
'one/missing.html',
'one/slow-fail.html',
'one/text-fail.html',
'two/image-fail.html',
]
for test in tests:
path = self.mac_port.host.filesystem.join(
self.mac_port.web_tests_dir(), test)
self._write(path, 'contents')
self.mac_port.host.filesystem.write_text_file(
'/test.checkout/web_tests/external/wpt/MANIFEST.json', '{}')
