def test_run_commit_queue_for_cl_fail_cq(self):
host = MockHost()
host.filesystem.write_text_file(
MOCK_WEB_TESTS + 'W3CImportExpectations', '')
importer = TestImporter(host)
importer.git_cl = MockGitCL(host,
status='lgtm',
try_job_results={
Build('cq-builder-a', 120):
TryJobStatus('COMPLETED', 'SUCCESS'),
Build('cq-builder-a', 123):
TryJobStatus('COMPLETED', 'FAILURE'),
Build('cq-builder-b', 200):
TryJobStatus('COMPLETED', 'SUCCESS'),
})
importer.fetch_new_expectations_and_baselines = lambda: None
success = importer.run_commit_queue_for_cl()
self.assertFalse(success)
self.assertLog([
'INFO: Triggering CQ try jobs.\n',
'INFO: All jobs finished.\n',
'ERROR: CQ appears to have failed; aborting.\n',
])
self.assertEqual(importer.git_cl.calls, [
['git', 'cl', 'try'],
['git', 'cl', 'set-close'],
])
def test_run_no_try_results(self):
updater = WPTExpectationsUpdater(self.mock_host())
updater.git_cl = MockGitCL(updater.host, {})
with self.assertRaises(ScriptError) as e:
updater.run(args=[])
self.assertEqual(e.exception.message,
'No try job information was collected.')
def test_run_single_platform_failure(self):
"""Tests the main run method in a case where one test fails on one platform."""
host = self.mock_host()
# Fill in an initial value for TestExpectations
expectations_path = host.port_factory.get(
).path_to_generic_test_expectations_file()
host.filesystem.write_text_file(expectations_path,
MARKER_COMMENT + '\n')
# Set up fake try job results.
updater = WPTExpectationsUpdater(host)
updater.git_cl = MockGitCL(
updater.host, {
Build('MOCK Try Mac10.10', 333):
TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Try Mac10.11', 111):
TryJobStatus('COMPLETED', 'SUCCESS'),
Build('MOCK Try Trusty', 222):
TryJobStatus('COMPLETED', 'SUCCESS'),
Build('MOCK Try Precise', 333):
TryJobStatus('COMPLETED', 'SUCCESS'),
Build('MOCK Try Win10', 444):
TryJobStatus('COMPLETED', 'SUCCESS'),
Build('MOCK Try Win7', 555):
TryJobStatus('COMPLETED', 'SUCCESS'),
})
# Set up failing results for one try bot. It shouldn't matter what
# results are for the other builders since we shouldn't need to even
# fetch results, since the try job status already tells us that all
# of the tests passed.
host.results_fetcher.set_results(
Build('MOCK Try Mac10.10', 333),
WebTestResults({
'tests': {
'external': {
'wpt': {
'test': {
'path.html': {
'expected': 'PASS',
'actual': 'TIMEOUT',
'is_unexpected': True,
}
}
}
}
}
}))
self.assertEqual(0, updater.run(args=[]))
# Results are only fetched for failing builds.
self.assertEqual(host.results_fetcher.fetched_builds,
[Build('MOCK Try Mac10.10', 333)])
self.assertEqual(
host.filesystem.read_text_file(expectations_path),
'# ====== New tests from wpt-importer added here ======\n'
'crbug.com/626703 [ Mac10.10 ] external/wpt/test/path.html [ Timeout ]\n'
)
def test_update(self):
host = MockHost()
filesystem = host.filesystem
finder = PathFinder(filesystem)
flag_expectations_file = finder.path_from_web_tests(
'FlagExpectations', 'foo')
filesystem.write_text_file(
flag_expectations_file,
'something/pass-unexpectedly-mac.html [ Fail ]')
self._setup_mock_results(host.buildbot)
cmd = ['update', '--flag=--foo']
TryFlag(cmd, host, MockGitCL(host, self.mock_try_results)).run()
def results_url(build):
return '%s/%s/%s/%s/layout-test-results/results.html' % (
'https://test-results.appspot.com/data/layout_results',
build.builder_name, build.build_number,
'webkit_layout_tests%20%28with%20patch%29')
self.assertEqual(
host.stdout.getvalue(), '\n'.join([
'Fetching results...',
'-- Linux: %s' % results_url(self.linux_build),
'-- Mac: %s' % results_url(self.mac_build),
'-- Win: %s' % results_url(self.win_build), '',
'### 1 unexpected passes:', '',
'Bug(none) [ Mac ] something/pass-unexpectedly-mac.html [ Pass ]',
'', '### 2 unexpected failures:', '',
'Bug(none) something/fail-everywhere.html [ Failure ]',
'Bug(none) [ Linux Win ] something/fail-win-and-linux.html [ Failure ]',
''
]))
def test_run_commit_queue_for_cl_fail_to_land(self):
host = self.mock_host()
host.filesystem.write_text_file(
MOCK_WEB_TESTS + 'W3CImportExpectations', '')
importer = self._get_test_importer(host)
# Only the latest job for each builder is counted.
importer.git_cl = MockGitCL(host,
status='lgtm',
try_job_results={
Build('cq-builder-a', 120):
TryJobStatus('COMPLETED', 'FAILURE'),
Build('cq-builder-a', 123):
TryJobStatus('COMPLETED', 'SUCCESS'),
})
importer.git_cl.wait_for_closed_status = lambda: False
success = importer.run_commit_queue_for_cl()
self.assertFalse(success)
self.assertLog([
'INFO: Triggering CQ try jobs.\n',
'INFO: All jobs finished.\n',
'INFO: CQ appears to have passed; trying to commit.\n',
'ERROR: Cannot submit CL; aborting.\n',
])
self.assertEqual(importer.git_cl.calls, [
['git', 'cl', 'try'],
['git', 'cl', 'upload', '-f', '--send-mail'],
['git', 'cl', 'set-commit'],
['git', 'cl', 'set-close'],
])
def test_run_commit_queue_for_cl_pass(self):
host = self.mock_host()
host.filesystem.write_text_file(
MOCK_WEB_TESTS + 'W3CImportExpectations', '')
importer = self._get_test_importer(host)
# Only the latest job for each builder is counted.
importer.git_cl = MockGitCL(host,
status='lgtm',
try_job_results={
Build('cq-builder-a', 120):
TryJobStatus('COMPLETED', 'FAILURE'),
Build('cq-builder-a', 123):
TryJobStatus('COMPLETED', 'SUCCESS'),
})
success = importer.run_commit_queue_for_cl()
self.assertTrue(success)
self.assertLog([
'INFO: Triggering CQ try jobs.\n',
'INFO: All jobs finished.\n',
'INFO: CQ appears to have passed; sending to the rubber-stamper '
'bot for CR+1 and commit.\n',
'INFO: If the rubber-stamper bot rejects the CL, you either need '
'to modify the benign file patterns, or manually CR+1 and land the '
'import yourself if it touches code files. See https://chromium.'
'googlesource.com/infra/infra/+/refs/heads/master/go/src/infra/'
'appengine/rubber-stamper/README.md\n',
'INFO: Update completed.\n',
])
self.assertEqual(importer.git_cl.calls, [
['git', 'cl', 'try'],
[
'git', 'cl', 'upload', '-f', '--send-mail',
'--enable-auto-submit', '--reviewers', RUBBER_STAMPER_BOT
],
])
def test_run_commit_queue_for_cl_pass(self):
host = MockHost()
host.filesystem.write_text_file(
MOCK_WEB_TESTS + 'W3CImportExpectations', '')
importer = TestImporter(host)
# Only the latest job for each builder is counted.
importer.git_cl = MockGitCL(host,
status='lgtm',
try_job_results={
Build('cq-builder-a', 120):
TryJobStatus('COMPLETED', 'FAILURE'),
Build('cq-builder-a', 123):
TryJobStatus('COMPLETED', 'SUCCESS'),
})
success = importer.run_commit_queue_for_cl()
self.assertTrue(success)
self.assertLog([
'INFO: Triggering CQ try jobs.\n',
'INFO: All jobs finished.\n',
'INFO: CQ appears to have passed; trying to commit.\n',
'INFO: Update completed.\n',
])
self.assertEqual(importer.git_cl.calls, [
['git', 'cl', 'try'],
['git', 'cl', 'upload', '-f', '--send-mail'],
['git', 'cl', 'set-commit'],
])
def test_execute_no_try_jobs_started_and_no_trigger_jobs(self):
# If there are no try jobs started yet and --no-trigger-jobs is passed,
# then we just abort immediately.
self.command.git_cl = MockGitCL(self.tool, {})
exit_code = self.command.execute(
self.command_options(trigger_jobs=False), [], self.tool)
self.assertEqual(exit_code, 1)
self.assertLog([
'INFO: No finished try jobs.\n',
'INFO: Aborted: no try jobs and --no-trigger-jobs passed.\n',
])
def test_update_expectations_for_cl_no_results(self):
host = MockHost()
host.filesystem.write_text_file(MOCK_WEB_TESTS + 'W3CImportExpectations', '')
importer = TestImporter(host)
importer.git_cl = MockGitCL(host, time_out=True)
success = importer.update_expectations_for_cl()
self.assertFalse(success)
self.assertLog([
'INFO: Triggering try jobs for updating expectations.\n',
'ERROR: No initial try job results, aborting.\n',
])
self.assertEqual(importer.git_cl.calls[-1], ['git', 'cl', 'set-close'])
def test_submit_cl_timeout_and_already_merged(self):
# Here we simulate a case where we timeout waiting for the CQ to submit a
# CL because we miss the notification that it was merged. We then get an
# error when trying to close the CL because it's already been merged.
host = self.mock_host()
host.filesystem.write_text_file(
MOCK_WEB_TESTS + 'W3CImportExpectations', '')
importer = self._get_test_importer(host)
# Define some error text that looks like a typical ScriptError.
git_error_text = (
'This is a git Script Error\n'
'...there is usually a stack trace here with some calls\n'
'...and maybe other calls\n'
'And finally, there is the exception:\n'
'GerritError: Conflict: change is merged\n')
importer.git_cl = MockGitCL(
host,
status='lgtm',
git_error_output={'set-close': git_error_text},
# Only the latest job for each builder is counted.
try_job_results={
Build('cq-builder-a', 120):
TryJobStatus('COMPLETED', 'FAILURE'),
Build('cq-builder-a', 123):
TryJobStatus('COMPLETED', 'SUCCESS')
})
importer._need_sheriff_attention = lambda: False
importer.git_cl.wait_for_closed_status = lambda timeout_seconds: False
success = importer.run_commit_queue_for_cl()
# Since the CL is already merged, we absorb the error and treat it as success.
self.assertTrue(success)
self.assertLog([
'INFO: Triggering CQ try jobs.\n',
'INFO: All jobs finished.\n',
'INFO: CQ appears to have passed; sending to the rubber-stamper '
'bot for CR+1 and commit.\n',
'INFO: If the rubber-stamper bot rejects the CL, you either need '
'to modify the benign file patterns, or manually CR+1 and land the '
'import yourself if it touches code files. See https://chromium.'
'googlesource.com/infra/infra/+/refs/heads/main/go/src/infra/'
'appengine/rubber-stamper/README.md\n',
'ERROR: Cannot submit CL; aborting.\n',
'ERROR: CL is already merged; treating as success.\n',
])
self.assertEqual(importer.git_cl.calls, [
['git', 'cl', 'try'],
[
'git', 'cl', 'upload', '-f', '--send-mail',
'--enable-auto-submit', '--reviewers', RUBBER_STAMPER_BOT
],
['git', 'cl', 'set-close'],
])
def test_run_commit_queue_for_cl_timeout(self):
# This simulates the case where we time out while waiting for try jobs.
host = MockHost()
importer = TestImporter(host)
importer.git_cl = MockGitCL(host, time_out=True)
success = importer.run_commit_queue_for_cl()
self.assertFalse(success)
self.assertLog([
'INFO: Triggering CQ try jobs.\n',
'ERROR: Timed out waiting for CQ; aborting.\n'
])
self.assertEqual(importer.git_cl.calls,
[['git', 'cl', 'try'], ['git', 'cl', 'set-close']])
def test_update_expectations_for_cl_all_jobs_pass(self):
host = MockHost()
host.filesystem.write_text_file(MOCK_WEB_TESTS + 'W3CImportExpectations', '')
importer = TestImporter(host)
importer.git_cl = MockGitCL(host, status='lgtm', try_job_results={
Build('builder-a', 123): TryJobStatus('COMPLETED', 'SUCCESS'),
})
success = importer.update_expectations_for_cl()
self.assertLog([
'INFO: Triggering try jobs for updating expectations.\n',
'INFO: All jobs finished.\n',
])
self.assertTrue(success)
def test_update_expectations_for_cl_closed_cl(self):
host = MockHost()
host.filesystem.write_text_file(MOCK_WEB_TESTS + 'W3CImportExpectations', '')
importer = TestImporter(host)
importer.git_cl = MockGitCL(host, status='closed', try_job_results={
Build('builder-a', 123): TryJobStatus('COMPLETED', 'SUCCESS'),
})
success = importer.update_expectations_for_cl()
self.assertFalse(success)
self.assertLog([
'INFO: Triggering try jobs for updating expectations.\n',
'ERROR: The CL was closed, aborting.\n',
])
def test_execute_no_try_jobs_started_triggers_jobs(self):
# If there are no try jobs started yet, by default the tool will
# trigger new try jobs.
self.command.git_cl = MockGitCL(self.tool, {})
exit_code = self.command.execute(self.command_options(), [], self.tool)
self.assertEqual(exit_code, 1)
self.assertLog([
'INFO: No finished try jobs.\n', 'INFO: Triggering try jobs:\n',
'INFO: MOCK Try Linux\n', 'INFO: MOCK Try Mac\n',
'INFO: MOCK Try Win\n',
'INFO: Once all pending try jobs have finished, please re-run\n'
'blink_tool.py rebaseline-cl to fetch new baselines.\n'
])
def test_update_irrelevant_unexpected_pass(self):
host = MockHost()
filesystem = host.filesystem
finder = PathFinder(filesystem)
flag_expectations_file = finder.path_from_web_tests(
'FlagExpectations', 'foo')
self._setup_mock_results(host.buildbot)
cmd = ['update', '--flag=--foo']
# Unexpected passes that don't have flag-specific failure expectations
# should not be reported.
filesystem.write_text_file(flag_expectations_file, '')
TryFlag(cmd, host, MockGitCL(host, self.mock_try_results)).run()
self.assertTrue('### 0 unexpected passes' in host.stdout.getvalue())
def test_update_expectations_for_cl_fail_but_no_changes(self):
host = MockHost()
host.filesystem.write_text_file(MOCK_WEB_TESTS + 'W3CImportExpectations', '')
importer = TestImporter(host)
importer.git_cl = MockGitCL(host, status='lgtm', try_job_results={
Build('builder-a', 123): TryJobStatus('COMPLETED', 'FAILURE'),
})
importer.fetch_new_expectations_and_baselines = lambda: None
success = importer.update_expectations_for_cl()
self.assertTrue(success)
self.assertLog([
'INFO: Triggering try jobs for updating expectations.\n',
'INFO: All jobs finished.\n',
])
def test_execute_with_canceled_job(self):
builds = {
Build('MOCK Try Win', 5000): TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Try Mac', 4000): TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Try Linux', 6000): TryJobStatus('COMPLETED', 'CANCELED'),
}
self.command.git_cl = MockGitCL(self.tool, builds)
exit_code = self.command.execute(self.command_options(), [], self.tool)
self.assertEqual(exit_code, 1)
self.assertLog([
'INFO: Finished try jobs found for all try bots.\n',
'INFO: There are some builders with no results:\n',
'INFO: MOCK Try Linux\n',
'INFO: Would you like to continue?\n',
'INFO: Aborting.\n',
])
def _run_trigger_test(self, regenerate):
host = MockHost()
git = host.git()
git_cl = MockGitCL(host)
finder = PathFinder(host.filesystem)
flag_file = finder.path_from_layout_tests(
'additional-driver-flag.setting')
flag_expectations_file = finder.path_from_layout_tests(
'FlagExpectations', 'foo')
cmd = ['trigger', '--flag=--foo']
if regenerate:
cmd.append('--regenerate')
TryFlag(cmd, host, git_cl).run()
expected_added_paths = {flag_file}
expected_commits = [[
'Flag try job: force --foo for run_web_tests.py.'
]]
if regenerate:
expected_added_paths.add(flag_expectations_file)
expected_commits.append(
['Flag try job: clear expectations for --foo.'])
self.assertEqual(git.added_paths, expected_added_paths)
self.assertEqual(git.local_commits(), expected_commits)
self.assertEqual(
git_cl.calls,
[[
'git', 'cl', 'upload', '--bypass-hooks', '-f', '-m',
'Flag try job for --foo.'
],
[
'git', 'cl', 'try', '-B', 'luci.chromium.try', '-b',
'linux_chromium_rel_ng'
],
[
'git', 'cl', 'try', '-B', 'master.tryserver.chromium.mac',
'-b', 'mac_chromium_rel_ng'
],
[
'git', 'cl', 'try', '-B', 'master.tryserver.chromium.win',
'-b', 'win7_chromium_rel_ng'
]])
def test_execute_one_missing_build(self):
builds = {
Build('MOCK Try Win', 5000): TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Try Mac', 4000): TryJobStatus('COMPLETED', 'FAILURE'),
}
self.command.git_cl = MockGitCL(self.tool, builds)
exit_code = self.command.execute(self.command_options(), [], self.tool)
self.assertEqual(exit_code, 1)
self.assertLog([
'INFO: Finished try jobs:\n',
'INFO: MOCK Try Mac\n',
'INFO: MOCK Try Win\n',
'INFO: Triggering try jobs:\n',
'INFO: MOCK Try Linux\n',
'INFO: Once all pending try jobs have finished, please re-run\n'
'blink_tool.py rebaseline-cl to fetch new baselines.\n',
])
def test_execute_with_passing_jobs(self):
builds = {
Build('MOCK Try Win', 5000): TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Try Mac', 4000): TryJobStatus('COMPLETED', 'SUCCESS'),
Build('MOCK Try Linux', 6000): TryJobStatus('COMPLETED', 'SUCCESS'),
}
self.command.git_cl = MockGitCL(self.tool, builds)
exit_code = self.command.execute(self.command_options(), [], self.tool)
self.assertEqual(exit_code, 0)
self.assertLog([
'INFO: Finished try jobs found for all try bots.\n',
'INFO: Rebaselining one/flaky-fail.html\n',
'INFO: Rebaselining one/missing.html\n',
'INFO: Rebaselining one/slow-fail.html\n',
'INFO: Rebaselining one/text-fail.html\n',
'INFO: Rebaselining two/image-fail.html\n'
])
def test_run_commit_queue_for_cl_closed_cl(self):
host = MockHost()
host.filesystem.write_text_file(MOCK_WEB_TESTS + 'W3CImportExpectations', '')
importer = TestImporter(host)
importer.git_cl = MockGitCL(host, status='closed', try_job_results={
Build('cq-builder-a', 120): TryJobStatus('COMPLETED', 'SUCCESS'),
Build('cq-builder-b', 200): TryJobStatus('COMPLETED', 'SUCCESS'),
})
success = importer.run_commit_queue_for_cl()
self.assertFalse(success)
self.assertLog([
'INFO: Triggering CQ try jobs.\n',
'ERROR: The CL was closed; aborting.\n',
])
self.assertEqual(importer.git_cl.calls, [
['git', 'cl', 'try'],
])
def test_execute_with_no_trigger_jobs_option(self):
builds = {
Build('MOCK Try Win', 5000): TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Try Mac', 4000): TryJobStatus('COMPLETED', 'FAILURE'),
}
self.command.git_cl = MockGitCL(self.tool, builds)
exit_code = self.command.execute(
self.command_options(trigger_jobs=False), [], self.tool)
self.assertEqual(exit_code, 1)
self.assertLog([
'INFO: Finished try jobs:\n',
'INFO: MOCK Try Mac\n',
'INFO: MOCK Try Win\n',
'INFO: There are some builders with no results:\n',
'INFO: MOCK Try Linux\n',
'INFO: Would you like to continue?\n',
'INFO: Aborting.\n',
])
def test_submit_cl_timeout_and_already_merged(self):
# Here we simulate a case where we timeout waiting for the CQ to submit a
# CL because we miss the notification that it was merged. We then get an
# error when trying to close the CL because it's already been merged.
host = MockHost()
host.filesystem.write_text_file(
MOCK_WEB_TESTS + 'W3CImportExpectations', '')
importer = TestImporter(host)
# Define some error text that looks like a typical ScriptError.
git_error_text = (
'This is a git Script Error\n'
'...there is usually a stack trace here with some calls\n'
'...and maybe other calls\n'
'And finally, there is the exception:\n'
'GerritError: Conflict: change is merged\n')
importer.git_cl = MockGitCL(
host,
status='lgtm',
git_error_output={'set-close': git_error_text},
# Only the latest job for each builder is counted.
try_job_results={
Build('cq-builder-a', 120):
TryJobStatus('COMPLETED', 'FAILURE'),
Build('cq-builder-a', 123):
TryJobStatus('COMPLETED', 'SUCCESS')
})
importer.git_cl.wait_for_closed_status = lambda: False
success = importer.run_commit_queue_for_cl()
# Since the CL is already merged, we absorb the error and treat it as success.
self.assertTrue(success)
self.assertLog([
'INFO: Triggering CQ try jobs.\n',
'INFO: All jobs finished.\n',
'INFO: CQ appears to have passed; trying to commit.\n',
'ERROR: Cannot submit CL; aborting.\n',
'ERROR: CL is already merged; treating as success.\n',
])
self.assertEqual(importer.git_cl.calls, [
['git', 'cl', 'try'],
['git', 'cl', 'upload', '-f', '--send-mail'],
['git', 'cl', 'set-commit'],
['git', 'cl', 'set-close'],
])
def test_execute_with_unfinished_jobs(self):
builds = {
Build('MOCK Try Win', 5000): TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Try Mac', 4000): TryJobStatus('STARTED'),
Build('MOCK Try Linux', 6000): TryJobStatus('SCHEDULED'),
}
self.command.git_cl = MockGitCL(self.tool, builds)
exit_code = self.command.execute(self.command_options(), [], self.tool)
self.assertEqual(exit_code, 1)
self.assertLog([
'INFO: Finished try jobs:\n',
'INFO: MOCK Try Win\n',
'INFO: Scheduled or started try jobs:\n',
'INFO: MOCK Try Linux\n',
'INFO: MOCK Try Mac\n',
'INFO: There are some builders with no results:\n',
'INFO: MOCK Try Linux\n',
'INFO: MOCK Try Mac\n',
'INFO: Would you like to continue?\n',
'INFO: Aborting.\n',
])
def test_invalid_action(self):
host = MockHost()
cmd = ['invalid', '--flag=--foo']
TryFlag(cmd, host, MockGitCL(host)).run()
self.assertEqual(host.stderr.getvalue(),
'specify "trigger" or "update"\n')
def testUpdateTestExpectationsForWeblayer(self):
raw_android_expectations = (
'# results: [ Failure Crash Timeout]\n'
'\n'
'crbug.com/1000754 external/wpt/foo.html [ Failure ]\n'
'\n'
'# Add untriaged failures in this block\n'
'crbug.com/1050754 external/wpt/bar.html [ Failure ]\n'
'\n'
'# This comment will not be deleted\n')
host = self._setup_host(raw_android_expectations)
host.results_fetcher.set_results(
Build('MOCK Android Weblayer - Pie', 123),
WebTestResults({
'tests': {
# A test result covered by default expectation
'new1.html': {
'expected': 'PASS',
'actual': 'FAIL',
'is_unexpected': True,
},
# A test result covered by baseline
'new2.html': {
'expected': 'PASS',
'actual': 'FAIL',
'is_unexpected': True,
},
# A new test case
'new3.html': {
'expected': 'PASS',
'actual': 'CRASH CRASH FAIL',
'is_unexpected': True,
},
},
}, step_name=WEBLAYER_WPT_STEP + ' (with patch)'),
step_name=WEBLAYER_WPT_STEP + ' (with patch)')
updater = AndroidWPTExpectationsUpdater(
host, ['-vvv', '--android-product', ANDROID_WEBLAYER,
'--include-unexpected-pass'])
updater.git_cl = MockGitCL(host, {
Build('MOCK Android Weblayer - Pie', 123):
TryJobStatus('COMPLETED', 'FAILURE')})
# Run command
updater.run()
# Get new expectations
content = host.filesystem.read_text_file(
PRODUCTS_TO_EXPECTATION_FILE_PATHS[ANDROID_WEBLAYER])
_new_expectations = (
'# results: [ Failure Crash Timeout]\n'
'\n'
'crbug.com/1000754 external/wpt/foo.html [ Failure ]\n'
'\n'
'# Add untriaged failures in this block\n'
'crbug.com/1050754 external/wpt/bar.html [ Failure ]\n'
'crbug.com/1050754 external/wpt/new3.html [ Crash Failure ]\n'
'\n'
'# This comment will not be deleted\n')
self.assertEqual(content, _new_expectations)
# Check that ANDROID_DISABLED_TESTS expectation files were not changed
self.assertEqual(
self._raw_android_never_fix_tests,
host.filesystem.read_text_file(ANDROID_DISABLED_TESTS))
def setUp(self):
BaseTestCase.setUp(self)
LoggingTestCase.setUp(self)
builds = {
Build('MOCK Try Win', 5000): TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Try Mac', 4000): TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Try Linux', 6000):
TryJobStatus('COMPLETED', 'FAILURE'),
}
self.command.git_cl = MockGitCL(self.tool, builds)
git = MockGit(filesystem=self.tool.filesystem,
executive=self.tool.executive)
git.changed_files = lambda **_: [
'third_party/WebKit/LayoutTests/one/text-fail.html',
'third_party/WebKit/LayoutTests/one/flaky-fail.html',
]
self.tool.git = lambda: git
self.tool.builders = BuilderList({
'MOCK Try Win': {
'port_name': 'test-win-win7',
'specifiers': ['Win7', 'Release'],
'is_try_builder': True,
},
'MOCK Try Linux': {
'port_name': 'test-linux-trusty',
'specifiers': ['Trusty', 'Release'],
'is_try_builder': True,
},
'MOCK Try Mac': {
'port_name': 'test-mac-mac10.11',
'specifiers': ['Mac10.11', 'Release'],
'is_try_builder': True,
},
})
layout_test_results = LayoutTestResults({
'tests': {
'one': {
'crash.html': {
'expected': 'PASS',
'actual': 'CRASH',
'is_unexpected': True
},
'expected-fail.html': {
'expected': 'FAIL',
'actual': 'IMAGE+TEXT'
},
'flaky-fail.html': {
'expected': 'PASS',
'actual': 'PASS TEXT',
'is_unexpected': True
},
'missing.html': {
'expected': 'PASS',
'actual': 'MISSING',
'is_unexpected': True
},
'slow-fail.html': {
'expected': 'SLOW',
'actual': 'TEXT',
'is_unexpected': True
},
'text-fail.html': {
'expected': 'PASS',
'actual': 'TEXT',
'is_unexpected': True
},
'unexpected-pass.html': {
'expected': 'FAIL',
'actual': 'PASS',
'is_unexpected': True
},
},
'two': {
'image-fail.html': {
'expected': 'PASS',
'actual': 'IMAGE',
'is_unexpected': True
}
},
},
})
for build in builds:
self.tool.buildbot.set_results(build, layout_test_results)
self.tool.buildbot.set_retry_sumary_json(
build,
json.dumps({
'failures': [
'one/flaky-fail.html',
'one/missing.html',
'one/slow-fail.html',
'one/text-fail.html',
'two/image-fail.html',
],
'ignored': [],
}))
# Write to the mock filesystem so that these tests are considered to exist.
tests = [
'one/flaky-fail.html',
'one/missing.html',
'one/slow-fail.html',
'one/text-fail.html',
'two/image-fail.html',
]
for test in tests:
path = self.mac_port.host.filesystem.join(
self.mac_port.layout_tests_dir(), test)
self._write(path, 'contents')
self.mac_port.host.filesystem.write_text_file(
'/test.checkout/LayoutTests/external/wpt/MANIFEST.json', '{}')
def test_execute_with_no_issue_number_aborts(self):
# If the user hasn't uploaded a CL, an error message is printed.
self.command.git_cl = MockGitCL(self.tool, issue_number='None')
exit_code = self.command.execute(self.command_options(), [], self.tool)
self.assertEqual(exit_code, 1)
self.assertLog(['ERROR: No issue number for current branch.\n'])
def setUp(self):
BaseTestCase.setUp(self)
LoggingTestCase.setUp(self)
builds = {
Build('MOCK Try Win', 5000): TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Try Mac', 4000): TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Try Linux', 6000):
TryJobStatus('COMPLETED', 'FAILURE'),
}
self.command.git_cl = MockGitCL(self.tool, builds)
git = MockGit(filesystem=self.tool.filesystem,
executive=self.tool.executive)
git.changed_files = lambda **_: [
RELATIVE_WEB_TESTS + 'one/text-fail.html',
RELATIVE_WEB_TESTS + 'one/flaky-fail.html',
]
self.tool.git = lambda: git
self.tool.builders = BuilderList({
'MOCK Try Win': {
'port_name': 'test-win-win7',
'specifiers': ['Win7', 'Release'],
'is_try_builder': True,
},
'MOCK Try Linux': {
'port_name': 'test-linux-trusty',
'specifiers': ['Trusty', 'Release'],
'is_try_builder': True,
},
'MOCK Try Mac': {
'port_name': 'test-mac-mac10.11',
'specifiers': ['Mac10.11', 'Release'],
'is_try_builder': True,
},
})
web_test_results = WebTestResults({
'tests': {
'one': {
'crash.html': {
'expected': 'PASS',
'actual': 'CRASH',
'is_unexpected': True,
'artifacts': {
'crash_log': ['crash.log']
}
},
'expected-fail.html': {
'expected': 'FAIL',
'actual': 'FAIL',
'artifacts': {
'expected_text': ['expected-fail-expected.txt'],
'actual_text': ['expected-fail-actual.txt']
}
},
'flaky-fail.html': {
'expected': 'PASS',
'actual': 'PASS FAIL',
'is_unexpected': True,
'artifacts': {
'expected_audio': ['flaky-fail-expected.wav'],
'actual_audio': ['flaky-fail-actual.wav']
}
},
'missing.html': {
'expected': 'PASS',
'actual': 'FAIL',
'is_unexpected': True,
'artifacts': {
'actual_image': ['missing-actual.png']
},
'is_missing_image': True
},
'slow-fail.html': {
'expected': 'SLOW',
'actual': 'FAIL',
'is_unexpected': True,
'artifacts': {
'actual_text': ['slow-fail-actual.txt'],
'expected_text': ['slow-fail-expected.txt']
}
},
'text-fail.html': {
'expected': 'PASS',
'actual': 'FAIL',
'is_unexpected': True,
'artifacts': {
'actual_text': ['text-fail-actual.txt'],
'expected_text': ['text-fail-expected.txt']
}
},
'unexpected-pass.html': {
'expected': 'FAIL',
'actual': 'PASS',
'is_unexpected': True
},
},
'two': {
'image-fail.html': {
'expected': 'PASS',
'actual': 'FAIL',
'is_unexpected': True,
'artifacts': {
'actual_image': ['image-fail-actual.png'],
'expected_image': ['image-fail-expected.png']
}
}
},
},
})
for build in builds:
self.tool.results_fetcher.set_results(build, web_test_results)
self.tool.results_fetcher.set_retry_sumary_json(
build,
json.dumps({
'failures': [
'one/flaky-fail.html',
'one/missing.html',
'one/slow-fail.html',
'one/text-fail.html',
'two/image-fail.html',
],
'ignored': [],
}))
# Write to the mock filesystem so that these tests are considered to exist.
tests = [
'one/flaky-fail.html',
'one/missing.html',
'one/slow-fail.html',
'one/text-fail.html',
'two/image-fail.html',
]
for test in tests:
path = self.mac_port.host.filesystem.join(
self.mac_port.web_tests_dir(), test)
self._write(path, 'contents')
self.mac_port.host.filesystem.write_text_file(
'/test.checkout/web_tests/external/wpt/MANIFEST.json', '{}')
def test_run_no_issue_number(self):
updater = WPTExpectationsUpdater(self.mock_host())
updater.git_cl = MockGitCL(updater.host, issue_number='None')
with self.assertRaises(ScriptError) as e:
updater.run(args=[])
self.assertEqual(e.exception.message, 'No issue on current branch.')
