def test_rebaseline_updates_expectations_file(self):
self._write(self.test_expectations_path, (
'# tags: [ Mac10.10 Mac Linux ]\n'
'# tags: [ Debug ]\n'
'# results: [ Failure ]\n'
'crbug.com/123 [ Debug Mac ] userscripts/first-test.html [ Failure ]\n'
'[ Linux ] userscripts/first-test.html [ Failure ]\n'))
self._setup_mock_build_data()
test_baseline_set = TestBaselineSet(self.tool)
test_baseline_set.add('userscripts/first-test.html',
Build('MOCK Mac10.11'))
self.command.rebaseline(self.options(), test_baseline_set)
new_expectations = self._read(self.test_expectations_path)
self.assertMultiLineEqual(new_expectations, (
'# tags: [ Mac10.10 Mac Linux ]\n'
'# tags: [ Debug ]\n'
'# results: [ Failure ]\n'
'crbug.com/123 [ Debug Mac10.10 ] userscripts/first-test.html [ Failure ]\n'
'[ Linux ] userscripts/first-test.html [ Failure ]\n'))
def test_get_step_name(self):
fetcher = TestResultsFetcher()
fetcher.web = MockWeb(
urls={
'https://test-results.appspot.com/testfile?buildnumber=5&'
'callback=ADD_RESULTS&builder=foo&name=full_results.json':
'ADD_RESULTS(%s);' %
(json.dumps([{
"TestType": "blink_web_tests (with patch)"
}, {
"TestType":
"not_site_per_process_blink_web_tests (with patch)"
}, {
"TestType": "blink_web_tests (retry with patch)"
}, {
"TestType": "base_unittests (with patch)"
}]))
})
step_name = fetcher.get_layout_test_step_name(Build('foo', 5))
self.assertEqual(step_name, 'blink_web_tests (with patch)')
self.assertLog([])
def test_results_directory(self):
test_baseline_set = TestBaselineSet(self.tool)
test_baseline_set.add('userscripts/first-test.html',
Build('MOCK Win7'))
self.command.rebaseline(
self.options(optimize=False, results_directory='/tmp'),
test_baseline_set)
self.assertEqual(self.tool.executive.calls, [
[[
'python',
'echo',
'copy-existing-baselines-internal',
'--verbose',
'--test',
'userscripts/first-test.html',
'--suffixes',
'png,txt',
'--port-name',
'test-win-win7',
]],
[[
'python',
'echo',
'rebaseline-test-internal',
'--verbose',
'--test',
'userscripts/first-test.html',
'--suffixes',
'png,txt',
'--port-name',
'test-win-win7',
'--builder',
'MOCK Win7',
'--results-directory',
'/tmp',
'--step-name',
'blink_web_tests (with patch)',
]],
])
def test_rebaseline_handles_slow_in_file(self):
self._write(self.test_expectations_path,
('# tags: [ Linux Mac Mac10.10 Win ]\n'
'# results: [ Failure Slow ]\n'
'[ Linux ] userscripts/first-test.html [ Failure ]\n'
'[ Mac ] userscripts/first-test.html [ Failure ]\n'
'[ Win ] userscripts/first-test.html [ Failure Slow ]\n'))
self._setup_mock_build_data()
test_baseline_set = TestBaselineSet(self.tool)
test_baseline_set.add('userscripts/first-test.html',
Build('MOCK Mac10.11'))
self.command.rebaseline(self.options(), test_baseline_set)
new_expectations = self._read(self.test_expectations_path)
self.assertMultiLineEqual(
new_expectations,
('# tags: [ Linux Mac Mac10.10 Win ]\n'
'# results: [ Failure Slow ]\n'
'[ Linux ] userscripts/first-test.html [ Failure ]\n'
'[ Mac10.10 ] userscripts/first-test.html [ Failure ]\n'
'[ Win ] userscripts/first-test.html [ Failure Slow ]\n'))
def test_get_step_name(self):
fetcher = TestResultsFetcher()
fetcher.builders.step_name_for_builder = lambda builder: None
fetcher.web = MockWeb(
urls={
'https://test-results.appspot.com/testfile?buildnumber=5&'
'callback=ADD_RESULTS&builder=foo&name=full_results.json':
b'ADD_RESULTS(' +
json.dumps([{
"TestType": "blink_web_tests (with patch)"
}, {
"TestType":
"not_site_per_process_blink_web_tests (with patch)"
}, {
"TestType": "blink_web_tests (retry with patch)"
}, {
"TestType": "base_unittests (with patch)"
}]).encode('utf8', 'replace') + b');'
})
step_name = fetcher.get_layout_test_step_name(Build('foo', 5))
self.assertEqual(step_name, 'blink_web_tests (with patch)')
self.assertLog([])
def test_try_job_results_skip_experimental_cq(self):
git_cl = GitCL(MockHost())
git_cl._host.web = MockWeb(responses=[{
'status_code':
200,
'body':
SEARCHBUILDS_RESPONSE_PREFIX + b"""{
"builds": [
{
"status": "SUCCESS",
"builder": {
"builder": "builder-a"
},
"number": 111,
"tags": [
{"key": "user_agent", "value": "cq"}
]
},
{
"status": "SUCCESS",
"builder": {
"builder": "builder-b"
},
"number": 222,
"tags": [
{"key": "user_agent", "value": "cq"},
{"key": "cq_experimental", "value": "true"}
]
}
]
}"""
}])
self.assertEqual(
# Only one build appears - builder-b is ignored because it is
# experimental.
git_cl.try_job_results(issue_number=None, cq_only=True),
{
Build('builder-a', 111): TryJobStatus('COMPLETED', 'SUCCESS'),
})
def test_fetch_results_with_weird_step_name(self):
fetcher = TestResultsFetcher()
fetcher.web = MockWeb(
urls={
'https://test-results.appspot.com/testfile?buildnumber=123&'
'callback=ADD_RESULTS&builder=builder&name=full_results.json':
'ADD_RESULTS(%s);' % (json.dumps(
[{
"TestType": "blink_web_tests on Intel GPU (with patch)"
}, {
"TestType": "base_unittests (with patch)"
}])),
'https://test-results.appspot.com/data/layout_results/builder/123/'
'blink_web_tests%20on%20Intel%20GPU%20%28with%20patch%29/'
'layout-test-results/failing_results.json':
json.dumps({'passed': True}),
})
results = fetcher.fetch_results(Build('builder', 123))
self.assertEqual(
results._results, # pylint: disable=protected-access
{'passed': True})
self.assertLog([])
def execute(self, options, args, tool):
self._tool = tool
if not args:
_log.error('Must list tests to rebaseline.')
return
if options.builders:
builders_to_check = []
for builder_names in options.builders:
builders_to_check += builder_names.split(',')
else:
builders_to_check = self._builders_to_pull_from()
test_baseline_set = TestBaselineSet(tool)
for builder in builders_to_check:
for test_prefix in args:
test_baseline_set.add(test_prefix, Build(builder))
_log.debug('Rebaselining: %s', test_baseline_set)
self.rebaseline(options, test_baseline_set)
def test_rebaseline_handles_skips_in_file(self):
# This test is like test_rebaseline_handles_platform_skips, except that the
# Skip is in the same (generic) file rather than a platform file. In this case,
# the Skip line should be left unmodified. Note that the first line is now
# qualified as "[Linux Mac Win]"; if it was unqualified, it would conflict with
# the second line.
self._write(
self.test_expectations_path,
('Bug(x) [ Linux Mac ] userscripts/first-test.html [ Failure ]\n'
'Bug(y) [ Win ] userscripts/first-test.html [ Skip ]\n'))
self._setup_mock_build_data()
test_baseline_set = TestBaselineSet(self.tool)
test_baseline_set.add('userscripts/first-test.html',
Build('MOCK Mac10.11'))
self.command.rebaseline(self.options(), test_baseline_set)
new_expectations = self._read(self.test_expectations_path)
self.assertMultiLineEqual(
new_expectations,
('Bug(x) [ Linux ] userscripts/first-test.html [ Failure ]\n'
'Bug(x) [ Mac10.10 ] userscripts/first-test.html [ Failure ]\n'
'Bug(y) [ Win ] userscripts/first-test.html [ Skip ]\n'))
def test_no_optimize(self):
test_baseline_set = TestBaselineSet(self.tool)
test_baseline_set.add('userscripts/first-test.html',
Build('MOCK Win7'))
self.command.rebaseline(self.options(optimize=False),
test_baseline_set)
self.assertEqual(self.tool.executive.calls,
[[[
'python',
'echo',
'copy-existing-baselines-internal',
'--verbose',
'--test',
'userscripts/first-test.html',
'--suffixes',
'txt,png',
'--port-name',
'test-win-win7',
]],
[[
'python',
'echo',
'rebaseline-test-internal',
'--verbose',
'--test',
'userscripts/first-test.html',
'--suffixes',
'txt,png',
'--port-name',
'test-win-win7',
'--builder',
'MOCK Win7',
'--step-name',
'webkit_layout_tests (with patch)',
]]])
def test_fill_in_missing_results_prefers_build_with_same_os_type(self):
self.tool.builders = BuilderList({
'MOCK Foo12': {
'port_name': 'foo-foo12',
'specifiers': ['Foo12', 'Release'],
'is_try_builder': True,
},
'MOCK Foo45': {
'port_name': 'foo-foo45',
'specifiers': ['Foo45', 'Release'],
'is_try_builder': True,
},
'MOCK Bar3': {
'port_name': 'bar-bar3',
'specifiers': ['Bar3', 'Release'],
'is_try_builder': True,
},
'MOCK Bar4': {
'port_name': 'bar-bar4',
'specifiers': ['Bar4', 'Release'],
'is_try_builder': True,
},
})
test_baseline_set = TestBaselineSet(self.tool)
test_baseline_set.add('one/flaky-fail.html', Build('MOCK Foo12', 100))
test_baseline_set.add('one/flaky-fail.html', Build('MOCK Bar4', 200))
self.command.fill_in_missing_results(test_baseline_set)
self.assertEqual(
sorted(test_baseline_set.build_port_pairs('one/flaky-fail.html')),
[
(Build('MOCK Bar4', 200), 'bar-bar3'),
(Build('MOCK Bar4', 200), 'bar-bar4'),
(Build('MOCK Foo12', 100), 'foo-foo12'),
(Build('MOCK Foo12', 100), 'foo-foo45'),
])
self.assertLog([
'INFO: For one/flaky-fail.html:\n',
'INFO: Using "MOCK Foo12" build 100 for foo-foo45.\n',
'INFO: Using "MOCK Bar4" build 200 for bar-bar3.\n',
])
def test_get_step_name_without_build_number(self):
fetcher = TestResultsFetcher()
self.assertIsNone(
fetcher.get_layout_test_step_name(Build('builder', None)))
def test_fetch_results_without_build_number(self):
fetcher = TestResultsFetcher()
self.assertIsNone(fetcher.fetch_results(Build('builder', None)))
def test_all_success_true(self):
self.assertTrue(
GitCL.all_success({
Build('some-builder', 1):
TryJobStatus('COMPLETED', 'SUCCESS'),
}))
def setUp(self):
BaseTestCase.setUp(self)
LoggingTestCase.setUp(self)
builds = {
Build('MOCK Try Win', 5000): TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Try Mac', 4000): TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Try Linux', 6000):
TryJobStatus('COMPLETED', 'FAILURE'),
}
self.command.git_cl = MockGitCL(self.tool, builds)
git = MockGit(filesystem=self.tool.filesystem,
executive=self.tool.executive)
git.changed_files = lambda **_: [
RELATIVE_WEB_TESTS + 'one/text-fail.html',
RELATIVE_WEB_TESTS + 'one/flaky-fail.html',
]
self.tool.git = lambda: git
self.tool.builders = BuilderList({
'MOCK Try Win': {
'port_name': 'test-win-win7',
'specifiers': ['Win7', 'Release'],
'is_try_builder': True,
},
'MOCK Try Linux': {
'port_name': 'test-linux-trusty',
'specifiers': ['Trusty', 'Release'],
'is_try_builder': True,
},
'MOCK Try Mac': {
'port_name': 'test-mac-mac10.11',
'specifiers': ['Mac10.11', 'Release'],
'is_try_builder': True,
},
})
web_test_results = WebTestResults({
'tests': {
'one': {
'crash.html': {
'expected': 'PASS',
'actual': 'CRASH',
'is_unexpected': True,
'artifacts': {
'crash_log': ['crash.log']
}
},
'expected-fail.html': {
'expected': 'FAIL',
'actual': 'FAIL',
'artifacts': {
'expected_text': ['expected-fail-expected.txt'],
'actual_text': ['expected-fail-actual.txt']
}
},
'flaky-fail.html': {
'expected': 'PASS',
'actual': 'PASS FAIL',
'is_unexpected': True,
'artifacts': {
'expected_audio': ['flaky-fail-expected.wav'],
'actual_audio': ['flaky-fail-actual.wav']
}
},
'missing.html': {
'expected': 'PASS',
'actual': 'FAIL',
'is_unexpected': True,
'artifacts': {
'actual_image': ['missing-actual.png']
},
'is_missing_image': True
},
'slow-fail.html': {
'expected': 'SLOW',
'actual': 'FAIL',
'is_unexpected': True,
'artifacts': {
'actual_text': ['slow-fail-actual.txt'],
'expected_text': ['slow-fail-expected.txt']
}
},
'text-fail.html': {
'expected': 'PASS',
'actual': 'FAIL',
'is_unexpected': True,
'artifacts': {
'actual_text': ['text-fail-actual.txt'],
'expected_text': ['text-fail-expected.txt']
}
},
'unexpected-pass.html': {
'expected': 'FAIL',
'actual': 'PASS',
'is_unexpected': True
},
},
'two': {
'image-fail.html': {
'expected': 'PASS',
'actual': 'FAIL',
'is_unexpected': True,
'artifacts': {
'actual_image': ['image-fail-actual.png'],
'expected_image': ['image-fail-expected.png']
}
}
},
},
})
for build in builds:
self.tool.results_fetcher.set_results(build, web_test_results)
self.tool.results_fetcher.set_retry_sumary_json(
build,
json.dumps({
'failures': [
'one/flaky-fail.html',
'one/missing.html',
'one/slow-fail.html',
'one/text-fail.html',
'two/image-fail.html',
],
'ignored': [],
}))
# Write to the mock filesystem so that these tests are considered to exist.
tests = [
'one/flaky-fail.html',
'one/missing.html',
'one/slow-fail.html',
'one/text-fail.html',
'two/image-fail.html',
]
for test in tests:
path = self.mac_port.host.filesystem.join(
self.mac_port.web_tests_dir(), test)
self._write(path, 'contents')
self.mac_port.host.filesystem.write_text_file(
'/test.checkout/web_tests/external/wpt/MANIFEST.json', '{}')
def testUpdateTestExpectationsForWebview(self):
host = self._setup_host()
host.results_fetcher.set_results(
Build('MOCK Android Pie', 123),
WebTestResults({
'tests': {
'abc.html': {
'expected': 'PASS',
'actual': 'CRASH TIMEOUT',
'is_unexpected': True,
},
'jkl.html': {
'expected': 'PASS',
'actual': 'FAIL',
'is_unexpected': True,
},
'cat.html': {
'expected': 'PASS',
'actual': 'CRASH CRASH TIMEOUT',
'is_unexpected': True,
},
},
}, step_name='system_webview_wpt (with patch)'),
step_name='system_webview_wpt (with patch)')
updater = AndroidWPTExpectationsUpdater(
host, ['-vvv', '--android-product', 'android_webview'])
updater.git_cl = MockGitCL(host, {
Build('MOCK Android Pie', 123):
TryJobStatus('COMPLETED', 'FAILURE')})
# Run command
updater.run()
# Get new expectations
content = host.filesystem.read_text_file(
PRODUCTS_TO_EXPECTATION_FILE_PATHS['android_webview'])
self.assertEqual(
content,
('# results: [ Failure Crash Timeout]\n'
'\n'
'# Add untriaged failures in this block\n'
'crbug.com/1050754 external/wpt/abc.html [ Crash Failure Timeout ]\n'
'crbug.com/1050754 external/wpt/cat.html [ Crash Failure Timeout ]\n'
'crbug.com/1050754 external/wpt/def.html [ Crash ]\n'
'external/wpt/dog.html [ Crash Timeout ]\n'
'crbug.com/1050754 external/wpt/ghi.html [ Timeout ]\n'
'crbug.com/1111111 crbug.com/1050754'
' external/wpt/jkl.html [ Failure ]\n'
'crbug.com/6789043 external/wpt/van.html [ Failure ]\n'
'external/wpt/www.html [ Crash Failure ]\n'
'\n'
'# This comment will not be deleted\n'
'crbug.com/111111 external/wpt/hello_world.html [ Crash ]\n'))
# check that chrome android's expectation file was not modified
# since the same bot is used to update chrome android & webview
# expectations
self.assertEqual(
host.filesystem.read_text_file(
PRODUCTS_TO_EXPECTATION_FILE_PATHS['chrome_android']),
self._raw_android_expectations)
# Check logs
logs = ''.join(self.logMessages()).lower()
self.assertNotIn('weblayer', logs)
self.assertNotIn('chrome', logs)
def testUpdateTestExpectationsForAll(self):
host = self._setup_host()
# Add results for Weblayer
host.results_fetcher.set_results(
Build('MOCK Android Weblayer - Pie', 123),
WebTestResults({
'tests': {
'abc.html': {
'expected': 'PASS',
'actual': 'CRASH TIMEOUT',
'is_unexpected': True,
},
'weblayer_only.html': {
'expected': 'PASS',
'actual': 'CRASH CRASH FAIL',
'is_unexpected': True,
},
},
}, step_name='weblayer_shell_wpt (with patch)'),
step_name='weblayer_shell_wpt (with patch)')
# Add Results for Webview
host.results_fetcher.set_results(
Build('MOCK Android Pie', 101),
WebTestResults({
'tests': {
'cat.html': {
'expected': 'PASS',
'actual': 'CRASH FAIL TIMEOUT',
'is_unexpected': True,
},
'webview_only.html': {
'expected': 'PASS',
'actual': 'TIMEOUT',
'is_unexpected': True,
},
},
}, step_name='system_webview_wpt (with patch)'),
step_name='system_webview_wpt (with patch)')
# Add Results for Chrome
host.results_fetcher.set_results(
Build('MOCK Android Pie', 101),
WebTestResults({
'tests': {
'jkl.html': {
'expected': 'PASS',
'actual': 'FAIL',
'is_unexpected': True,
},
'chrome_only.html': {
'expected': 'PASS',
'actual': 'CRASH CRASH TIMEOUT',
'is_unexpected': True,
},
},
}, step_name='chrome_public_wpt (with patch)'),
step_name='chrome_public_wpt (with patch)')
updater = AndroidWPTExpectationsUpdater(
host, ['-vvv',
'--android-product', 'android_weblayer',
'--android-product', 'chrome_android',
'--android-product', 'android_webview'])
updater.git_cl = MockGitCL(host, {
Build('MOCK Android Weblayer - Pie', 123):
TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Android Pie', 101):
TryJobStatus('COMPLETED', 'FAILURE')})
# Run command
updater.run()
# Check expectations for weblayer
content = host.filesystem.read_text_file(
PRODUCTS_TO_EXPECTATION_FILE_PATHS['android_weblayer'])
self.assertEqual(
content,
('# results: [ Failure Crash Timeout]\n'
'\n'
'# Add untriaged failures in this block\n'
'crbug.com/1050754 external/wpt/abc.html [ Crash Failure Timeout ]\n'
'crbug.com/1050754 external/wpt/cat.html [ Failure ]\n'
'crbug.com/1050754 external/wpt/def.html [ Crash ]\n'
'external/wpt/dog.html [ Crash Timeout ]\n'
'crbug.com/1050754 external/wpt/ghi.html [ Timeout ]\n'
'crbug.com/1111111 external/wpt/jkl.html [ Failure ]\n'
'crbug.com/6789043 external/wpt/van.html [ Failure ]\n'
'crbug.com/1050754 external/wpt/weblayer_only.html [ Failure Crash ]\n'
'external/wpt/www.html [ Crash Failure ]\n'
'\n'
'# This comment will not be deleted\n'
'crbug.com/111111 external/wpt/hello_world.html [ Crash ]\n'))
# Check expectations for webview
content = host.filesystem.read_text_file(
PRODUCTS_TO_EXPECTATION_FILE_PATHS['android_webview'])
self.assertEqual(
content,
('# results: [ Failure Crash Timeout]\n'
'\n'
'# Add untriaged failures in this block\n'
'external/wpt/abc.html [ Failure ]\n'
'crbug.com/1050754 external/wpt/cat.html [ Crash Failure Timeout ]\n'
'crbug.com/1050754 external/wpt/def.html [ Crash ]\n'
'external/wpt/dog.html [ Crash Timeout ]\n'
'crbug.com/1050754 external/wpt/ghi.html [ Timeout ]\n'
'crbug.com/1111111 external/wpt/jkl.html [ Failure ]\n'
'crbug.com/6789043 external/wpt/van.html [ Failure ]\n'
'crbug.com/1050754 external/wpt/webview_only.html [ Timeout ]\n'
'external/wpt/www.html [ Crash Failure ]\n'
'\n'
'# This comment will not be deleted\n'
'crbug.com/111111 external/wpt/hello_world.html [ Crash ]\n'))
# Check expectations chrome
content = host.filesystem.read_text_file(
PRODUCTS_TO_EXPECTATION_FILE_PATHS['chrome_android'])
self.assertEqual(
content,
('# results: [ Failure Crash Timeout]\n'
'\n'
'# Add untriaged failures in this block\n'
'external/wpt/abc.html [ Failure ]\n'
'crbug.com/1050754 external/wpt/cat.html [ Failure ]\n'
'crbug.com/1050754 external/wpt/chrome_only.html [ Crash Timeout ]\n'
'crbug.com/1050754 external/wpt/def.html [ Crash ]\n'
'external/wpt/dog.html [ Crash Timeout ]\n'
'crbug.com/1050754 external/wpt/ghi.html [ Timeout ]\n'
'crbug.com/1111111 crbug.com/1050754'
' external/wpt/jkl.html [ Failure ]\n'
'crbug.com/6789043 external/wpt/van.html [ Failure ]\n'
'external/wpt/www.html [ Crash Failure ]\n'
'\n'
'# This comment will not be deleted\n'
'crbug.com/111111 external/wpt/hello_world.html [ Crash ]\n'))
def test_filter_latest_jobs_higher_build_last(self):
self.assertEqual(
filter_latest_builds(
[Build('foo', 3), Build('bar', 5), Build('foo', 5)]),
[Build('bar', 5), Build('foo', 5)])
def test_latest_try_jobs_cq_only(self):
git_cl = GitCL(MockHost())
git_cl._host.web = MockWeb(responses=[{
'status_code':
200,
'body':
SEARCHBUILDS_RESPONSE_PREFIX + b"""{
"builds": [
{
"status": "SCHEDULED",
"builder": {
"builder": "cq-a"
},
"tags": [
{"key": "user_agent", "value": "cq"}
]
},
{
"status": "SCHEDULED",
"builder": {
"builder": "cq-b"
},
"tags": [
{"key": "user_agent", "value": "cq"},
{"key": "cq_experimental", "value": "false"}
]
},
{
"status": "SCHEDULED",
"builder": {
"builder": "cq-c"
},
"tags": [
{"key": "user_agent", "value": "cq"},
{"key": "cq_experimental", "value": "false"}
]
},
{
"status": "SCHEDULED",
"builder": {
"builder": "cq-a-experimental"
},
"tags": [
{"key": "user_agent", "value": "cq"},
{"key": "cq_experimental", "value": "true"}
]
},
{
"status": "SCHEDULED",
"builder": {
"builder": "cq-b-experimental"
},
"tags": [
{"key": "user_agent", "value": "cq"},
{"key": "cq_experimental", "value": "true"}
]
},
{
"status": "SCHEDULED",
"builder": {
"builder": "other-a"
},
"tags": [
{"key": "user_agent", "value": "git_cl_try"}
]
},
{
"status": "SCHEDULED",
"builder": {
"builder": "other-b"
},
"tags": [
{"key": "user_agent", "value": "git_cl_try"},
{"key": "cq_experimental", "value": "false"}
]
}
]
}"""
}])
self.assertEqual(
git_cl.latest_try_jobs(cq_only=True), {
Build('cq-a'): TryJobStatus('SCHEDULED'),
Build('cq-b'): TryJobStatus('SCHEDULED'),
Build('cq-c'): TryJobStatus('SCHEDULED'),
})
def test_fetch_webdriver_results_without_master(self):
fetcher = TestResultsFetcher()
self.assertIsNone(
fetcher.fetch_webdriver_test_results(Build('builder', 1), ''))
self.assertLog(
['DEBUG: Builder name or build number or master is None\n'])
def test_filter_latest_jobs_no_build_number(self):
self.assertEqual(
filter_latest_builds([Build('foo', 3),
Build('bar'),
Build('bar')]),
[Build('bar'), Build('foo', 3)])
def testCleanupAndUpdateTestExpectationsForAll(self):
# Full integration test for expectations cleanup and update
# using builder results.
host = self._setup_host()
# Add results for Weblayer
host.results_fetcher.set_results(
Build('MOCK Android Weblayer - Pie', 123),
WebTestResults(
{
'tests': {
'abc.html': {
'expected': 'PASS',
'actual': 'CRASH TIMEOUT',
'is_unexpected': True,
},
'weblayer_only.html': {
'expected': 'PASS',
'actual': 'CRASH CRASH FAIL',
'is_unexpected': True,
},
'disabled_weblayer_only.html': {
'expected': 'SKIP',
'actual': 'SKIP',
'is_unexpected': True,
},
'unexpected_pass.html': {
'expected': 'FAIL',
'actual': 'PASS',
'is_unexpected': True
},
},
},
step_name=WEBLAYER_WPT_STEP + ' (with patch)'),
step_name=WEBLAYER_WPT_STEP + ' (with patch)')
# Add Results for Webview
host.results_fetcher.set_results(
Build('MOCK Android Pie', 101),
WebTestResults(
{
'tests': {
'cat.html': {
'expected': 'PASS',
'actual': 'CRASH FAIL TIMEOUT',
'is_unexpected': True,
},
'webview_only.html': {
'expected': 'PASS',
'actual': 'TIMEOUT',
'is_unexpected': True,
},
'disabled.html': {
'expected': 'SKIP',
'actual': 'SKIP',
},
'unexpected_pass.html': {
'expected': 'FAIL',
'actual': 'PASS',
'is_unexpected': True
},
},
},
step_name=WEBVIEW_WPT_STEP + ' (with patch)'),
step_name=WEBVIEW_WPT_STEP + ' (with patch)')
# Add Results for Chrome
host.results_fetcher.set_results(
Build('MOCK Android Pie', 101),
WebTestResults(
{
'tests': {
'jkl.html': {
'expected': 'PASS',
'actual': 'FAIL',
'is_unexpected': True,
},
'chrome_only.html': {
'expected': 'PASS',
'actual': 'CRASH CRASH TIMEOUT',
'is_unexpected': True,
},
'disabled.html': {
'expected': 'SKIP',
'actual': 'SKIP',
'is_unexpected': True,
},
'unexpected_pass.html': {
'expected': 'FAIL',
'actual': 'PASS',
'is_unexpected': True
},
},
},
step_name=CHROME_ANDROID_WPT_STEP + ' (with patch)'),
step_name=CHROME_ANDROID_WPT_STEP + ' (with patch)')
updater = AndroidWPTExpectationsUpdater(host, [
'-vvv', '--clean-up-test-expectations',
'--clean-up-affected-tests-only', '--include-unexpected-pass',
'--android-product', ANDROID_WEBLAYER, '--android-product',
CHROME_ANDROID, '--android-product', ANDROID_WEBVIEW
])
def _git_command_return_val(cmd):
if '--diff-filter=D' in cmd:
return 'external/wpt/ghi.html'
if '--diff-filter=R' in cmd:
return 'C external/wpt/van.html external/wpt/wagon.html'
return ''
updater.git_cl = MockGitCL(
host, {
Build('MOCK Android Weblayer - Pie', 123):
TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Android Pie', 101):
TryJobStatus('COMPLETED', 'FAILURE')
})
updater.git.run = _git_command_return_val
updater._relative_to_web_test_dir = lambda test_path: test_path
# Run command
updater.run()
# Check expectations for weblayer
content = host.filesystem.read_text_file(
PRODUCTS_TO_EXPECTATION_FILE_PATHS[ANDROID_WEBLAYER])
self.assertEqual(content, (
'# results: [ Failure Crash Timeout]\n'
'\n'
'# Add untriaged failures in this block\n'
'crbug.com/1050754 external/wpt/abc.html [ Crash Failure Timeout ]\n'
'crbug.com/1050754 external/wpt/cat.html [ Failure ]\n'
'crbug.com/1050754 external/wpt/def.html [ Crash ]\n'
'external/wpt/dog.html [ Crash Timeout ]\n'
'crbug.com/1111111 external/wpt/jkl.html [ Failure ]\n'
'crbug.com/1050754 external/wpt/unexpected_pass.html [ Failure Pass ]\n'
'crbug.com/6789043 external/wpt/wagon.html [ Failure ]\n'
'crbug.com/1050754 external/wpt/weblayer_only.html [ Failure Crash ]\n'
'external/wpt/www.html [ Crash Failure ]\n'
'\n'
'# This comment will not be deleted\n'
'crbug.com/111111 external/wpt/hello_world.html [ Crash ]\n'))
# Check expectations for webview
content = host.filesystem.read_text_file(
PRODUCTS_TO_EXPECTATION_FILE_PATHS[ANDROID_WEBVIEW])
self.assertEqual(content, (
'# results: [ Failure Crash Timeout]\n'
'\n'
'# Add untriaged failures in this block\n'
'external/wpt/abc.html [ Failure ]\n'
'crbug.com/1050754 external/wpt/cat.html [ Crash Failure Timeout ]\n'
'crbug.com/1050754 external/wpt/def.html [ Crash ]\n'
'external/wpt/dog.html [ Crash Timeout ]\n'
'crbug.com/1111111 external/wpt/jkl.html [ Failure ]\n'
'crbug.com/1050754 external/wpt/unexpected_pass.html [ Failure Pass ]\n'
'crbug.com/6789043 external/wpt/wagon.html [ Failure ]\n'
'crbug.com/1050754 external/wpt/webview_only.html [ Timeout ]\n'
'external/wpt/www.html [ Crash Failure ]\n'
'\n'
'# This comment will not be deleted\n'
'crbug.com/111111 external/wpt/hello_world.html [ Crash ]\n'))
# Check expectations chrome
content = host.filesystem.read_text_file(
PRODUCTS_TO_EXPECTATION_FILE_PATHS[CHROME_ANDROID])
self.assertEqual(content, (
'# results: [ Failure Crash Timeout]\n'
'\n'
'# Add untriaged failures in this block\n'
'external/wpt/abc.html [ Failure ]\n'
'crbug.com/1050754 external/wpt/cat.html [ Failure ]\n'
'crbug.com/1050754 external/wpt/chrome_only.html [ Crash Timeout ]\n'
'crbug.com/1050754 external/wpt/def.html [ Crash ]\n'
'external/wpt/dog.html [ Crash Timeout ]\n'
'crbug.com/1111111 crbug.com/1050754'
' external/wpt/jkl.html [ Failure ]\n'
'crbug.com/1050754 external/wpt/unexpected_pass.html [ Failure Pass ]\n'
'crbug.com/6789043 external/wpt/wagon.html [ Failure ]\n'
'external/wpt/www.html [ Crash Failure ]\n'
'\n'
'# This comment will not be deleted\n'
'crbug.com/111111 external/wpt/hello_world.html [ Crash ]\n'))
# Check disabled test file
neverfix_content = host.filesystem.read_text_file(
ANDROID_DISABLED_TESTS)
self.assertEqual(neverfix_content, (
'# tags: [ android-weblayer android-webview chrome-android ]\n'
'# results: [ Skip ]\n'
'\n'
'# Add untriaged disabled tests in this block\n'
'crbug.com/1050754 [ android-webview ] external/wpt/disabled.html [ Skip ]\n'
'crbug.com/1050754 [ chrome-android ] external/wpt/disabled.html [ Skip ]\n'
'crbug.com/1050754 [ android-weblayer ] external/wpt/disabled_weblayer_only.html [ Skip ]\n'
))
def test_run_single_platform_failure(self):
"""Tests the main run method in a case where one test fails on one platform."""
host = self.mock_host()
# Fill in an initial value for TestExpectations
expectations_path = \
host.port_factory.get().path_to_generic_test_expectations_file()
host.filesystem.write_text_file(
expectations_path, WPTExpectationsUpdater.MARKER_COMMENT + '\n')
# Set up fake try job results.
updater = WPTExpectationsUpdater(host)
updater.git_cl = MockGitCL(
updater.host, {
Build('MOCK Try Mac10.10', 333):
TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Try Mac10.11', 111):
TryJobStatus('COMPLETED', 'SUCCESS'),
Build('MOCK Try Trusty', 222):
TryJobStatus('COMPLETED', 'SUCCESS'),
Build('MOCK Try Precise', 333):
TryJobStatus('COMPLETED', 'SUCCESS'),
Build('MOCK Try Win10', 444):
TryJobStatus('COMPLETED', 'SUCCESS'),
Build('MOCK Try Win7', 555):
TryJobStatus('COMPLETED', 'SUCCESS'),
})
# Set up failing results for one try bot. It shouldn't matter what
# results are for the other builders since we shouldn't need to even
# fetch results, since the try job status already tells us that all
# of the tests passed.
host.results_fetcher.set_results(
Build('MOCK Try Mac10.10', 333),
WebTestResults({
'tests': {
'external': {
'wpt': {
'test': {
'path.html': {
'expected': 'PASS',
'actual': 'TIMEOUT',
'is_unexpected': True,
}
}
}
}
}
}))
self.assertEqual(0, updater.run())
# Results are only fetched for failing builds.
self.assertEqual(host.results_fetcher.fetched_builds, [
BuilderStep(Build('MOCK Try Mac10.10', 333),
'blink_web_tests (with patch)')
])
self.assertEqual(
host.filesystem.read_text_file(expectations_path),
'# ====== New tests from wpt-importer added here ======\n'
'crbug.com/626703 [ Mac10.10 ] external/wpt/test/path.html [ Timeout ]\n'
)
def testCleanupAndUpdateTestExpectationsForAll(self):
# Full integration test for expectations cleanup and update
# using builder results.
raw_android_expectations = (
'# results: [ Failure Crash Timeout]\n'
'\n'
'crbug.com/1000754 external/wpt/foo1.html [ Failure ]\n'
'crbug.com/1000754 external/wpt/foo2.html [ Failure ]\n'
'crbug.com/1000754 external/wpt/bar.html [ Failure ]\n'
'\n'
'# Add untriaged failures in this block\n'
'\n'
'# This comment will not be deleted\n')
host = self._setup_host(raw_android_expectations)
# Add results for Weblayer
host.results_fetcher.set_results(
Build('MOCK Android Weblayer - Pie', 123),
WebTestResults({
'tests': {
'disabled_weblayer_only.html': {
'expected': 'SKIP',
'actual': 'SKIP',
'is_unexpected': True,
},
'bar.html': {
'expected': 'FAIL',
'actual': 'CRASH',
'is_unexpected': True
},
},
}, step_name=WEBLAYER_WPT_STEP + ' (with patch)'),
step_name=WEBLAYER_WPT_STEP + ' (with patch)')
updater = AndroidWPTExpectationsUpdater(
host, ['-vvv',
'--clean-up-test-expectations',
'--clean-up-affected-tests-only',
'--include-unexpected-pass',
'--android-product', ANDROID_WEBLAYER])
def _git_command_return_val(cmd):
if '--diff-filter=D' in cmd:
return 'external/wpt/foo2.html'
if '--diff-filter=R' in cmd:
return 'C\texternal/wpt/foo1.html\texternal/wpt/foo3.html'
if '--diff-filter=M' in cmd:
return 'external/wpt/bar.html'
return ''
updater.git_cl = MockGitCL(host, {
Build('MOCK Android Weblayer - Pie', 123):
TryJobStatus('COMPLETED', 'FAILURE')})
updater.git.run = _git_command_return_val
updater._relative_to_web_test_dir = lambda test_path: test_path
# Run command
updater.run()
# Check expectations for weblayer
content = host.filesystem.read_text_file(
PRODUCTS_TO_EXPECTATION_FILE_PATHS[ANDROID_WEBLAYER])
_new_expectations = (
'# results: [ Failure Crash Timeout]\n'
'\n'
'crbug.com/1000754 external/wpt/foo3.html [ Failure ]\n'
'\n'
'# Add untriaged failures in this block\n'
'crbug.com/1050754 external/wpt/bar.html [ Crash ]\n'
'\n'
'# This comment will not be deleted\n')
self.assertEqual(content, _new_expectations)
# Check disabled test file
neverfix_content = host.filesystem.read_text_file(ANDROID_DISABLED_TESTS)
self.assertEqual(
neverfix_content,
('# tags: [ android-weblayer android-webview chrome-android ]\n'
'# results: [ Skip ]\n'
'\n'
'# Add untriaged disabled tests in this block\n'
'crbug.com/1050754 [ android-webview ] external/wpt/disabled.html [ Skip ]\n'
'crbug.com/1050754 [ android-weblayer ] external/wpt/disabled_weblayer_only.html [ Skip ]\n'))
def test_get_failing_results_dict_webdriver_failing_results_(self):
host = self.mock_host()
host.results_fetcher.set_results(
Build('MOCK Try Trusty', 123),
WebTestResults({
'tests': {
'external': {
'wpt': {
'x': {
'failing-test.html': {
'expected': 'PASS',
'actual': 'IMAGE',
'is_unexpected': True,
},
},
},
},
},
}))
host.results_fetcher.set_webdriver_test_results(
Build('MOCK Try Trusty', 123), "tryserver.blink",
WebTestResults({
'tests': {
'external': {
'wpt': {
'y': {
'webdriver-fail.html': {
'expected': 'PASS',
'actual': 'FAIL',
'is_unexpected': True,
},
},
},
},
},
}))
updater = WPTExpectationsUpdater(host)
results_dict = updater.get_failing_results_dict(
Build('MOCK Try Trusty', 123))
self.assertEqual(len(results_dict.keys()), 2)
self.assertEqual(
results_dict, {
'external/wpt/x/failing-test.html': {
'test-linux-trusty':
SimpleTestResult(
actual='IMAGE',
expected='PASS',
bug='crbug.com/626703',
),
},
'external/wpt/y/webdriver-fail.html': {
'test-linux-trusty':
SimpleTestResult(
actual='FAIL',
expected='PASS',
bug='crbug.com/626703',
),
},
})
def testUpdateTestExpectationsForWeblayer(self):
raw_android_expectations = (
'# results: [ Failure Crash Timeout]\n'
'\n'
'crbug.com/1000754 external/wpt/foo.html [ Failure ]\n'
'\n'
'# Add untriaged failures in this block\n'
'crbug.com/1050754 external/wpt/bar.html [ Failure ]\n'
'\n'
'# This comment will not be deleted\n')
host = self._setup_host(raw_android_expectations)
host.results_fetcher.set_results(
Build('MOCK Android Weblayer - Pie', 123),
WebTestResults({
'tests': {
# A test result covered by default expectation
'new1.html': {
'expected': 'PASS',
'actual': 'FAIL',
'is_unexpected': True,
},
# A test result covered by baseline
'new2.html': {
'expected': 'PASS',
'actual': 'FAIL',
'is_unexpected': True,
},
# A new test case
'new3.html': {
'expected': 'PASS',
'actual': 'CRASH CRASH FAIL',
'is_unexpected': True,
},
},
}, step_name=WEBLAYER_WPT_STEP + ' (with patch)'),
step_name=WEBLAYER_WPT_STEP + ' (with patch)')
updater = AndroidWPTExpectationsUpdater(
host, ['-vvv', '--android-product', ANDROID_WEBLAYER,
'--include-unexpected-pass'])
updater.git_cl = MockGitCL(host, {
Build('MOCK Android Weblayer - Pie', 123):
TryJobStatus('COMPLETED', 'FAILURE')})
# Run command
updater.run()
# Get new expectations
content = host.filesystem.read_text_file(
PRODUCTS_TO_EXPECTATION_FILE_PATHS[ANDROID_WEBLAYER])
_new_expectations = (
'# results: [ Failure Crash Timeout]\n'
'\n'
'crbug.com/1000754 external/wpt/foo.html [ Failure ]\n'
'\n'
'# Add untriaged failures in this block\n'
'crbug.com/1050754 external/wpt/bar.html [ Failure ]\n'
'crbug.com/1050754 external/wpt/new3.html [ Crash Failure ]\n'
'\n'
'# This comment will not be deleted\n')
self.assertEqual(content, _new_expectations)
# Check that ANDROID_DISABLED_TESTS expectation files were not changed
self.assertEqual(
self._raw_android_never_fix_tests,
host.filesystem.read_text_file(ANDROID_DISABLED_TESTS))
def testUpdateTestExpectationsForWebview(self):
host = self._setup_host()
host.results_fetcher.set_results(
Build('MOCK Android Pie', 123),
WebTestResults(
{
'tests': {
'abc.html': {
'expected': 'PASS',
'actual': 'CRASH TIMEOUT',
'is_unexpected': True,
},
'jkl.html': {
'expected': 'PASS',
'actual': 'FAIL',
'is_unexpected': True,
},
'cat.html': {
'expected': 'PASS',
'actual': 'CRASH CRASH TIMEOUT',
'is_unexpected': True,
},
'unexpected_pass.html': {
'expected': 'FAIL',
'actual': 'PASS',
'is_unexpected': True
},
'dog.html': {
'expected': 'SKIP',
'actual': 'SKIP',
'is_unexpected': True,
},
},
},
step_name=WEBVIEW_WPT_STEP + ' (with patch)'),
step_name=WEBVIEW_WPT_STEP + ' (with patch)')
updater = AndroidWPTExpectationsUpdater(host, [
'-vvv', '--android-product', ANDROID_WEBVIEW,
'--clean-up-test-expectations', '--clean-up-affected-tests-only',
'--include-unexpected-pass'
])
updater.git_cl = MockGitCL(host, {
Build('MOCK Android Pie', 123):
TryJobStatus('COMPLETED', 'FAILURE')
})
# Run command
updater.run()
# Get new expectations
content = host.filesystem.read_text_file(
PRODUCTS_TO_EXPECTATION_FILE_PATHS[ANDROID_WEBVIEW])
self.assertEqual(content, (
'# results: [ Failure Crash Timeout]\n'
'\n'
'# Add untriaged failures in this block\n'
'crbug.com/1050754 external/wpt/abc.html [ Crash Failure Timeout ]\n'
'crbug.com/1050754 external/wpt/cat.html [ Crash Failure Timeout ]\n'
'crbug.com/1050754 external/wpt/def.html [ Crash ]\n'
'external/wpt/dog.html [ Crash Timeout ]\n'
'crbug.com/1050754 external/wpt/ghi.html [ Timeout ]\n'
'crbug.com/1111111 crbug.com/1050754'
' external/wpt/jkl.html [ Failure ]\n'
'crbug.com/1050754 external/wpt/unexpected_pass.html [ Failure Pass ]\n'
'crbug.com/6789043 external/wpt/van.html [ Failure ]\n'
'external/wpt/www.html [ Crash Failure ]\n'
'\n'
'# This comment will not be deleted\n'
'crbug.com/111111 external/wpt/hello_world.html [ Crash ]\n'))
neverfix_content = host.filesystem.read_text_file(
ANDROID_DISABLED_TESTS)
self.assertEqual(neverfix_content, (
'# tags: [ android-weblayer android-webview chrome-android ]\n'
'# results: [ Skip ]\n'
'\n'
'# Add untriaged disabled tests in this block\n'
'crbug.com/1050754 [ android-webview ] external/wpt/disabled.html [ Skip ]\n'
'crbug.com/1050754 [ android-webview ] external/wpt/dog.html [ Skip ]\n'
))
# check that chrome android's expectation file was not modified
# since the same bot is used to update chrome android & webview
# expectations
self.assertEqual(
host.filesystem.read_text_file(
PRODUCTS_TO_EXPECTATION_FILE_PATHS[CHROME_ANDROID]),
self._raw_android_expectations)
# Check logs
logs = ''.join(self.logMessages()).lower()
self.assertNotIn(WEBLAYER_WPT_STEP, logs)
self.assertNotIn(CHROME_ANDROID_WPT_STEP, logs)
# Check that weblayer and chrome expectation files were not changed
self.assertEqual(
self._raw_android_expectations,
host.filesystem.read_text_file(
PRODUCTS_TO_EXPECTATION_FILE_PATHS[CHROME_ANDROID]))
self.assertEqual(
self._raw_android_expectations,
host.filesystem.read_text_file(
PRODUCTS_TO_EXPECTATION_FILE_PATHS[ANDROID_WEBLAYER]))
def test_has_failing_try_results_with_failing_results(self):
self.assertTrue(
GitCL.some_failed({
Build('some-builder', 1):
TryJobStatus('COMPLETED', 'FAILURE'),
}))
