def test_run_commit_queue_for_cl_pass(self):
host = MockHost()
host.filesystem.write_text_file(
'/mock-checkout/third_party/WebKit/LayoutTests/W3CImportExpectations',
'')
importer = TestImporter(host)
# Only the latest job for each builder is counted.
importer.git_cl = MockGitCL(host,
results={
Build('cq-builder-a', 120):
TryJobStatus('COMPLETED', 'FAILURE'),
Build('cq-builder-a', 123):
TryJobStatus('COMPLETED', 'SUCCESS'),
})
success = importer.run_commit_queue_for_cl()
self.assertTrue(success)
self.assertLog([
'INFO: Triggering CQ try jobs.\n',
'INFO: CQ appears to have passed; trying to commit.\n',
'INFO: Update completed.\n',
])
self.assertEqual(importer.git_cl.calls, [
['git', 'cl', 'try'],
['git', 'cl', 'upload', '-f', '--send-mail'],
['git', 'cl', 'set-commit'],
])
def test_run_commit_queue_for_cl_fails(self):
host = MockHost()
host.filesystem.write_text_file(
'/mock-checkout/third_party/WebKit/LayoutTests/W3CImportExpectations',
'')
importer = TestImporter(host)
importer.git_cl = MockGitCL(host,
results={
Build('cq-builder-a', 120):
TryJobStatus('COMPLETED', 'SUCCESS'),
Build('cq-builder-a', 123):
TryJobStatus('COMPLETED', 'FAILURE'),
Build('cq-builder-b', 200):
TryJobStatus('COMPLETED', 'SUCCESS'),
})
importer.fetch_new_expectations_and_baselines = lambda: None
success = importer.run_commit_queue_for_cl()
self.assertFalse(success)
self.assertLog([
'INFO: Triggering CQ try jobs.\n',
'ERROR: CQ appears to have failed; aborting.\n',
])
self.assertEqual(importer.git_cl.calls, [
['git', 'cl', 'try'],
['git', 'cl', 'set-close'],
])
def test_run_single_platform_failure(self):
"""Tests the main run method in a case where one test fails on one platform."""
host = self.mock_host()
# Fill in an initial value for TestExpectations
expectations_path = host.port_factory.get(
).path_to_generic_test_expectations_file()
host.filesystem.write_text_file(expectations_path,
MARKER_COMMENT + '\n')
# Set up fake try job results.
updater = WPTExpectationsUpdater(host)
updater.git_cl = MockGitCL(
updater.host, {
Build('MOCK Try Mac10.10', 333):
TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Try Mac10.11', 111):
TryJobStatus('COMPLETED', 'SUCCESS'),
Build('MOCK Try Trusty', 222):
TryJobStatus('COMPLETED', 'SUCCESS'),
Build('MOCK Try Precise', 333):
TryJobStatus('COMPLETED', 'SUCCESS'),
Build('MOCK Try Win10', 444):
TryJobStatus('COMPLETED', 'SUCCESS'),
Build('MOCK Try Win7', 555):
TryJobStatus('COMPLETED', 'SUCCESS'),
})
# Set up failing results for one try bot. It shouldn't matter what
# results are for the other builders since we shouldn't need to even
# fetch results, since the try job status already tells us that all
# of the tests passed.
host.buildbot.set_results(
Build('MOCK Try Mac10.10', 333),
LayoutTestResults({
'tests': {
'external': {
'wpt': {
'test': {
'path.html': {
'expected': 'PASS',
'actual': 'TIMEOUT',
'is_unexpected': True,
}
}
}
}
}
}))
self.assertEqual(0, updater.run(args=[]))
# Results are only fetched for failing builds.
self.assertEqual(host.buildbot.fetched_builds,
[Build('MOCK Try Mac10.10', 333)])
self.assertEqual(
host.filesystem.read_text_file(expectations_path),
'# ====== New tests from wpt-importer added here ======\n'
'crbug.com/626703 [ Mac10.10 ] external/wpt/test/path.html [ Timeout ]\n'
)
def test_run_no_try_results(self):
updater = WPTExpectationsUpdater(self.mock_host())
updater.git_cl = MockGitCL(updater.host, {})
with self.assertRaises(ScriptError) as e:
updater.run(args=[])
self.assertEqual(e.exception.message,
'No try job information was collected.')
def test_update(self):
host = MockHost()
filesystem = host.filesystem
finder = PathFinder(filesystem)
flag_expectations_file = finder.path_from_layout_tests(
'FlagExpectations', 'foo')
filesystem.write_text_file(
flag_expectations_file,
'something/pass-unexpectedly-mac.html [ Fail ]')
self._setup_mock_results(host.buildbot)
cmd = ['update', '--flag=--foo']
TryFlag(cmd, host, MockGitCL(host, self.mock_try_results)).run()
def results_url(build):
return '%s/%s/%s/layout-test-results/results.html' % (
'https://test-results.appspot.com/data/layout_results',
build.builder_name, build.build_number)
self.assertEqual(
host.stdout.getvalue(), '\n'.join([
'Fetching results...',
'-- Linux: %s' % results_url(self.linux_build),
'-- Mac: %s' % results_url(self.mac_build),
'-- Win: %s' % results_url(self.win_build), '',
'### 1 unexpected passes:', '',
'Bug(none) [ Mac ] something/pass-unexpectedly-mac.html [ Pass ]',
'', '### 2 unexpected failures:', '',
'Bug(none) something/fail-everywhere.html [ Failure ]',
'Bug(none) [ Linux Win ] something/fail-win-and-linux.html [ Failure ]',
''
]))
def test_run_commit_queue_for_cl_closed_cl(self):
host = MockHost()
host.filesystem.write_text_file(
'/mock-checkout/third_party/WebKit/LayoutTests/W3CImportExpectations',
'')
importer = TestImporter(host)
importer.git_cl = MockGitCL(
host,
results=CLStatus(
status='closed',
try_job_results={
Build('cq-builder-a', 120):
TryJobStatus('COMPLETED', 'SUCCESS'),
Build('cq-builder-b', 200):
TryJobStatus('COMPLETED', 'SUCCESS'),
},
))
success = importer.run_commit_queue_for_cl()
self.assertFalse(success)
self.assertLog([
'INFO: Triggering CQ try jobs.\n',
'ERROR: The CL was closed; aborting.\n',
])
self.assertEqual(importer.git_cl.calls, [
['git', 'cl', 'try'],
])
def test_update(self):
host = MockHost()
try_jobs = {
self.linux_build: TryJobStatus('COMPLETED', 'SUCCESS'),
self.win_build: TryJobStatus('COMPLETED', 'SUCCESS'),
self.mac_build: TryJobStatus('COMPLETED', 'SUCCESS')
}
self._setup_mock_results(host.buildbot)
TryFlag(['update'], host, MockGitCL(host, try_jobs)).run()
def results_url(build):
return '%s/%s/%s/layout-test-results/results.html' % (
'https://storage.googleapis.com/chromium-layout-test-archives',
build.builder_name, build.build_number)
self.assertEqual(
host.stdout.getvalue(), '\n'.join([
'Fetching results...',
'-- Linux: %s' % results_url(self.linux_build),
'-- Mac: %s' % results_url(self.mac_build),
'-- Win: %s' % results_url(self.win_build), '',
'### 1 unexpected passes:', '',
'Bug(none) [ Mac ] something/pass-unexpectedly-mac.html [ Pass ]',
'', '### 2 unexpected failures:', '',
'Bug(none) something/fail-everywhere.html [ Failure ]',
'Bug(none) [ Linux Win ] something/fail-win-and-linux.html [ Failure ]',
''
]))
def test_run_commit_queue_for_cl_only_checks_non_blink_bots(self):
host = MockHost()
host.filesystem.write_text_file(
'/mock-checkout/third_party/WebKit/LayoutTests/W3CImportExpectations',
'')
host.builders = BuilderList({
'fakeos_blink_rel': {
'port_name': 'test-fakeos',
'specifiers': ['FakeOS', 'Release'],
'is_try_builder': True,
}
})
importer = TestImporter(host)
importer.git_cl = MockGitCL(host,
results={
Build('fakeos_blink_rel', 123):
TryJobStatus('COMPLETED', 'FAILURE'),
Build('cq-builder-b', 200):
TryJobStatus('COMPLETED', 'SUCCESS'),
})
importer.fetch_new_expectations_and_baselines = lambda: None
success = importer.run_commit_queue_for_cl()
self.assertTrue(success)
self.assertLog([
'INFO: Triggering CQ try jobs.\n',
'INFO: CQ appears to have passed; trying to commit.\n',
'INFO: Update completed.\n',
])
self.assertEqual(importer.git_cl.calls, [
['git', 'cl', 'try'],
['git', 'cl', 'upload', '-f', '--send-mail'],
['git', 'cl', 'set-commit'],
])
def test_update_expectations_for_cl_all_jobs_pass(self):
host = MockHost()
host.filesystem.write_text_file(
'/mock-checkout/third_party/WebKit/LayoutTests/W3CImportExpectations', '')
importer = TestImporter(host)
importer.git_cl = MockGitCL(host, results={
Build('builder-a', 123): TryJobStatus('COMPLETED', 'SUCCESS'),
})
success = importer.update_expectations_for_cl()
self.assertTrue(success)
def test_execute_no_try_jobs_started_and_no_trigger_jobs(self):
# If there are no try jobs started yet and --no-trigger-jobs is passed,
# then we just abort immediately.
self.command.git_cl = MockGitCL(self.tool, {})
exit_code = self.command.execute(
self.command_options(trigger_jobs=False), [], self.tool)
self.assertEqual(exit_code, 1)
self.assertLog([
'INFO: No finished try jobs.\n',
'INFO: Aborted: no try jobs and --no-trigger-jobs passed.\n',
])
def test_execute_no_try_jobs_started_triggers_jobs(self):
# If there are no try jobs started yet, by default the tool will
# trigger new try jobs.
self.command.git_cl = MockGitCL(self.tool, {})
exit_code = self.command.execute(self.command_options(), [], self.tool)
self.assertEqual(exit_code, 1)
self.assertLog([
'INFO: No finished try jobs.\n', 'INFO: Triggering try jobs:\n',
'INFO: MOCK Try Linux\n', 'INFO: MOCK Try Mac\n',
'INFO: MOCK Try Win\n',
'INFO: Once all pending try jobs have finished, please re-run\n'
'webkit-patch rebaseline-cl to fetch new baselines.\n'
])
def test_run_commit_queue_for_cl_timeout(self):
# This simulates the case where we time out while waiting for try jobs.
host = MockHost()
importer = TestImporter(host)
importer.git_cl = MockGitCL(host, time_out=True)
success = importer.run_commit_queue_for_cl()
self.assertFalse(success)
self.assertLog([
'INFO: Triggering CQ try jobs.\n',
'ERROR: Timed out waiting for CQ; aborting.\n'
])
self.assertEqual(importer.git_cl.calls,
[['git', 'cl', 'try'], ['git', 'cl', 'set-close']])
def test_update_expectations_for_cl_no_results(self):
host = MockHost()
host.filesystem.write_text_file(
'/mock-checkout/third_party/WebKit/LayoutTests/W3CImportExpectations', '')
importer = TestImporter(host)
importer.git_cl = MockGitCL(host, results={})
success = importer.update_expectations_for_cl()
self.assertFalse(success)
self.assertLog([
'INFO: Triggering try jobs for updating expectations.\n',
'ERROR: No initial try job results, aborting.\n',
])
self.assertEqual(importer.git_cl.calls[-1], ['git', 'cl', 'set-close'])
def test_update_irrelevant_unexpected_pass(self):
host = MockHost()
filesystem = host.filesystem
finder = PathFinder(filesystem)
flag_expectations_file = finder.path_from_layout_tests(
'FlagExpectations', 'foo')
self._setup_mock_results(host.buildbot)
cmd = ['update', '--flag=--foo']
# Unexpected passes that don't have flag-specific failure expectations
# should not be reported.
filesystem.write_text_file(flag_expectations_file, '')
TryFlag(cmd, host, MockGitCL(host, self.mock_try_results)).run()
self.assertTrue('### 0 unexpected passes' in host.stdout.getvalue())
def test_update_expectations_for_cl_all_jobs_pass(self):
host = MockHost()
host.filesystem.write_text_file(
'/mock-checkout/third_party/WebKit/LayoutTests/W3CImportExpectations', '')
importer = TestImporter(host)
importer.git_cl = MockGitCL(host, status='lgtm', try_job_results={
Build('builder-a', 123): TryJobStatus('COMPLETED', 'SUCCESS'),
})
success = importer.update_expectations_for_cl()
self.assertLog([
'INFO: Triggering try jobs for updating expectations.\n',
'INFO: All jobs finished.\n',
])
self.assertTrue(success)
def test_update_expectations_for_cl_closed_cl(self):
host = MockHost()
host.filesystem.write_text_file(
'/mock-checkout/third_party/WebKit/LayoutTests/W3CImportExpectations', '')
importer = TestImporter(host)
importer.git_cl = MockGitCL(host, status='closed', try_job_results={
Build('builder-a', 123): TryJobStatus('COMPLETED', 'SUCCESS'),
})
success = importer.update_expectations_for_cl()
self.assertFalse(success)
self.assertLog([
'INFO: Triggering try jobs for updating expectations.\n',
'ERROR: The CL was closed, aborting.\n',
])
def test_update_expectations_for_cl_fail_but_no_changes(self):
host = MockHost()
host.filesystem.write_text_file(
'/mock-checkout/third_party/WebKit/LayoutTests/W3CImportExpectations', '')
importer = TestImporter(host)
importer.git_cl = MockGitCL(host, results={
Build('builder-a', 123): TryJobStatus('COMPLETED', 'FAILURE'),
})
importer.fetch_new_expectations_and_baselines = lambda: None
success = importer.update_expectations_for_cl()
self.assertTrue(success)
self.assertLog([
'INFO: Triggering try jobs for updating expectations.\n',
])
def test_run_commit_queue_for_cl_timeout(self):
host = MockHost()
importer = TestImporter(host)
# The simulates the case where importer.git_cl.wait_for_try_jobs returns
# None, which would normally happen if we time out waiting for results.
importer.git_cl = MockGitCL(host, results=None)
success = importer.run_commit_queue_for_cl()
self.assertFalse(success)
self.assertLog([
'INFO: Triggering CQ try jobs.\n',
'ERROR: Timed out waiting for CQ; aborting.\n'
])
self.assertEqual(importer.git_cl.calls,
[['git', 'cl', 'try'], ['git', 'cl', 'set-close']])
def test_execute_with_canceled_job(self):
builds = {
Build('MOCK Try Win', 5000): TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Try Mac', 4000): TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Try Linux', 6000): TryJobStatus('COMPLETED', 'CANCELED'),
}
self.command.git_cl = MockGitCL(self.tool, builds)
exit_code = self.command.execute(self.command_options(), [], self.tool)
self.assertEqual(exit_code, 1)
self.assertLog([
'INFO: Finished try jobs found for all try bots.\n',
'INFO: There are some builders with no results:\n',
'INFO: MOCK Try Linux\n',
'INFO: Would you like to continue?\n',
'INFO: Aborting.\n',
])
def _run_trigger_test(self, regenerate):
host = MockHost()
git = host.git()
git_cl = MockGitCL(host)
finder = PathFinder(host.filesystem)
flag_file = finder.path_from_layout_tests(
'additional-driver-flag.setting')
flag_expectations_file = finder.path_from_layout_tests(
'FlagExpectations', 'foo')
cmd = ['trigger', '--flag=--foo']
if regenerate:
cmd.append('--regenerate')
TryFlag(cmd, host, git_cl).run()
expected_added_paths = {flag_file}
expected_commits = [[
'Flag try job: force --foo for run-webkit-tests.'
]]
if regenerate:
expected_added_paths.add(flag_expectations_file)
expected_commits.append(
['Flag try job: clear expectations for --foo.'])
self.assertEqual(git.added_paths, expected_added_paths)
self.assertEqual(git.local_commits(), expected_commits)
self.assertEqual(
git_cl.calls,
[[
'git', 'cl', 'upload', '--bypass-hooks', '-f', '-m',
'Flag try job for --foo.'
],
[
'git', 'cl', 'try', '-m', 'tryserver.chromium.linux', '-b',
'linux_chromium_rel_ng'
],
[
'git', 'cl', 'try', '-m', 'tryserver.chromium.mac', '-b',
'mac_chromium_rel_ng'
],
[
'git', 'cl', 'try', '-m', 'tryserver.chromium.win', '-b',
'win7_chromium_rel_ng'
]])
def test_execute_with_passing_jobs(self):
builds = {
Build('MOCK Try Win', 5000): TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Try Mac', 4000): TryJobStatus('COMPLETED', 'SUCCESS'),
Build('MOCK Try Linux', 6000): TryJobStatus('COMPLETED', 'SUCCESS'),
}
self.command.git_cl = MockGitCL(self.tool, builds)
exit_code = self.command.execute(self.command_options(), [], self.tool)
self.assertEqual(exit_code, 0)
self.assertLog([
'INFO: Finished try jobs found for all try bots.\n',
'INFO: Rebaselining one/flaky-fail.html\n',
'INFO: Rebaselining one/missing.html\n',
'INFO: Rebaselining one/slow-fail.html\n',
'INFO: Rebaselining one/text-fail.html\n',
'INFO: Rebaselining two/image-fail.html\n'
])
def test_execute_one_missing_build(self):
builds = {
Build('MOCK Try Win', 5000): TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Try Mac', 4000): TryJobStatus('COMPLETED', 'FAILURE'),
}
self.command.git_cl = MockGitCL(self.tool, builds)
exit_code = self.command.execute(self.command_options(), [], self.tool)
self.assertEqual(exit_code, 1)
self.assertLog([
'INFO: Finished try jobs:\n',
'INFO: MOCK Try Mac\n',
'INFO: MOCK Try Win\n',
'INFO: Triggering try jobs:\n',
'INFO: MOCK Try Linux\n',
'INFO: Once all pending try jobs have finished, please re-run\n'
'webkit-patch rebaseline-cl to fetch new baselines.\n',
])
def test_execute_with_no_trigger_jobs_option(self):
builds = {
Build('MOCK Try Win', 5000): TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Try Mac', 4000): TryJobStatus('COMPLETED', 'FAILURE'),
}
self.command.git_cl = MockGitCL(self.tool, builds)
exit_code = self.command.execute(
self.command_options(trigger_jobs=False), [], self.tool)
self.assertEqual(exit_code, 1)
self.assertLog([
'INFO: Finished try jobs:\n',
'INFO: MOCK Try Mac\n',
'INFO: MOCK Try Win\n',
'INFO: There are some builders with no results:\n',
'INFO: MOCK Try Linux\n',
'INFO: Would you like to continue?\n',
'INFO: Aborting.\n',
])
def test_execute_with_no_trigger_jobs_option(self):
builds = {
Build('MOCK Try Win', 5000): TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Try Mac', 4000): TryJobStatus('COMPLETED', 'FAILURE'),
}
self.command.git_cl = MockGitCL(self.tool, builds)
exit_code = self.command.execute(
self.command_options(trigger_jobs=False), [], self.tool)
self.assertEqual(exit_code, 1)
self.assertLog([
'INFO: Finished try jobs:\n', 'INFO: MOCK Try Mac\n',
'INFO: MOCK Try Win\n',
'INFO: There are some builders with no results:\n',
'INFO: MOCK Try Linux\n',
'INFO: Would you like to try to fill in missing results with\n'
'available results? This assumes that layout test results\n'
'for the platforms with missing results are the same as\n'
'results on other platforms.\n', 'INFO: Aborting.\n'
])
def test_execute_with_unfinished_jobs(self):
builds = {
Build('MOCK Try Win', 5000): TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Try Mac', 4000): TryJobStatus('STARTED'),
Build('MOCK Try Linux', 6000): TryJobStatus('SCHEDULED'),
}
self.command.git_cl = MockGitCL(self.tool, builds)
exit_code = self.command.execute(self.command_options(), [], self.tool)
self.assertEqual(exit_code, 1)
self.assertLog([
'INFO: Finished try jobs:\n',
'INFO: MOCK Try Win\n',
'INFO: Scheduled or started try jobs:\n',
'INFO: MOCK Try Linux\n',
'INFO: MOCK Try Mac\n',
'INFO: There are some builders with no results:\n',
'INFO: MOCK Try Linux\n',
'INFO: MOCK Try Mac\n',
'INFO: Would you like to continue?\n',
'INFO: Aborting.\n',
])
def test_run_no_try_results(self):
updater = WPTExpectationsUpdater(self.mock_host())
updater.git_cl = MockGitCL(updater.host, {})
self.assertEqual(1, updater.run(args=[]))
self.assertLog(['ERROR: No try job information was collected.\n'])
def test_invalid_action(self):
host = MockHost()
cmd = ['invalid', '--flag=--foo']
TryFlag(cmd, host, MockGitCL(host)).run()
self.assertEqual(host.stderr.getvalue(),
'specify "trigger" or "update"\n')
def test_run_no_issue_number(self):
updater = WPTExpectationsUpdater(self.mock_host())
updater.git_cl = MockGitCL(updater.host, issue_number='None')
with self.assertRaises(ScriptError) as e:
updater.run(args=[])
self.assertEqual(e.exception.message, 'No issue on current branch.')
def test_execute_with_no_issue_number_aborts(self):
# If the user hasn't uploaded a CL, an error message is printed.
self.command.git_cl = MockGitCL(self.tool, issue_number='None')
exit_code = self.command.execute(self.command_options(), [], self.tool)
self.assertEqual(exit_code, 1)
self.assertLog(['ERROR: No issue number for current branch.\n'])
def setUp(self):
BaseTestCase.setUp(self)
LoggingTestCase.setUp(self)
builds = {
Build('MOCK Try Win', 5000): TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Try Mac', 4000): TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Try Linux', 6000):
TryJobStatus('COMPLETED', 'FAILURE'),
}
self.command.git_cl = MockGitCL(self.tool, builds)
git = MockGit(filesystem=self.tool.filesystem,
executive=self.tool.executive)
git.changed_files = lambda **_: [
'third_party/WebKit/LayoutTests/one/text-fail.html',
'third_party/WebKit/LayoutTests/one/flaky-fail.html',
]
self.tool.git = lambda: git
self.tool.builders = BuilderList({
'MOCK Try Win': {
'port_name': 'test-win-win7',
'specifiers': ['Win7', 'Release'],
'is_try_builder': True,
},
'MOCK Try Linux': {
'port_name': 'test-linux-trusty',
'specifiers': ['Trusty', 'Release'],
'is_try_builder': True,
},
'MOCK Try Mac': {
'port_name': 'test-mac-mac10.11',
'specifiers': ['Mac10.11', 'Release'],
'is_try_builder': True,
},
})
layout_test_results = LayoutTestResults({
'tests': {
'one': {
'crash.html': {
'expected': 'PASS',
'actual': 'CRASH',
'is_unexpected': True
},
'expected-fail.html': {
'expected': 'FAIL',
'actual': 'IMAGE+TEXT'
},
'flaky-fail.html': {
'expected': 'PASS',
'actual': 'PASS TEXT',
'is_unexpected': True
},
'missing.html': {
'expected': 'PASS',
'actual': 'MISSING',
'is_unexpected': True
},
'slow-fail.html': {
'expected': 'SLOW',
'actual': 'TEXT',
'is_unexpected': True
},
'text-fail.html': {
'expected': 'PASS',
'actual': 'TEXT',
'is_unexpected': True
},
'unexpected-pass.html': {
'expected': 'FAIL',
'actual': 'PASS',
'is_unexpected': True
},
},
'two': {
'image-fail.html': {
'expected': 'PASS',
'actual': 'IMAGE',
'is_unexpected': True
}
},
},
})
for build in builds:
self.tool.buildbot.set_results(build, layout_test_results)
self.tool.buildbot.set_retry_sumary_json(
build,
json.dumps({
'failures': [
'one/flaky-fail.html',
'one/missing.html',
'one/slow-fail.html',
'one/text-fail.html',
'two/image-fail.html',
],
'ignored': [],
}))
# Write to the mock filesystem so that these tests are considered to exist.
tests = [
'one/flaky-fail.html',
'one/missing.html',
'one/slow-fail.html',
'one/text-fail.html',
'two/image-fail.html',
]
for test in tests:
path = self.mac_port.host.filesystem.join(
self.mac_port.layout_tests_dir(), test)
self._write(path, 'contents')
self.mac_port.host.filesystem.write_text_file(
'/test.checkout/LayoutTests/external/wpt/MANIFEST.json', '{}')