def test_get_failing_results_dict_webdriver_failing_results_(self):
host = self.mock_host()
host.results_fetcher.set_results(
Build('MOCK Try Trusty', 123),
WebTestResults({
'tests': {
'external': {
'wpt': {
'x': {
'failing-test.html': {
'expected': 'PASS',
'actual': 'IMAGE',
'is_unexpected': True,
},
},
},
},
},
}))
host.results_fetcher.set_webdriver_test_results(
Build('MOCK Try Trusty', 123), "tryserver.blink",
WebTestResults({
'tests': {
'external': {
'wpt': {
'y': {
'webdriver-fail.html': {
'expected': 'PASS',
'actual': 'FAIL',
'is_unexpected': True,
},
},
},
},
},
}))
updater = WPTExpectationsUpdater(host)
results = updater.get_failing_results_dicts(
Build('MOCK Try Trusty', 123))
self.assertEqual(len(results), 2)
self.assertEqual(results, [{
'external/wpt/x/failing-test.html': {
DesktopConfig('test-linux-trusty'):
SimpleTestResult(
actual='IMAGE',
expected='PASS',
bug='crbug.com/626703',
),
}
}, {
'external/wpt/y/webdriver-fail.html': {
DesktopConfig('test-linux-trusty'):
SimpleTestResult(
actual='FAIL',
expected='PASS',
bug='crbug.com/626703',
),
},
}])
def test_merging_platforms_if_possible(self):
host = self.mock_host()
updater = WPTExpectationsUpdater(host)
results = {
'external/wpt/x.html': {
(
DesktopConfig('test-linux-precise'),
DesktopConfig('test-linux-trusty'),
DesktopConfig('test-mac-mac10.10'),
DesktopConfig('test-mac-mac10.11'),
DesktopConfig('test-win-win7'),
):
SimpleTestResult(expected='PASS',
actual='TEXT',
bug='crbug.com/test')
}
}
self.assertEqual(
updater.create_line_dict(results), {
'external/wpt/x.html': [
'crbug.com/test [ Linux ] external/wpt/x.html [ Failure ]',
'crbug.com/test [ Mac ] external/wpt/x.html [ Failure ]',
'crbug.com/test [ Win7 ] external/wpt/x.html [ Failure ]',
]
})
def test_one_platform_has_no_results(self):
# In this example, there is a failure that has been observed on
# Linux and one Mac port, but the other Mac port has no results at all.
# The specifiers are "filled in" and the failure is assumed to apply
# to all Mac platforms.
host = self.mock_host()
updater = WPTExpectationsUpdater(host)
results = {
'external/wpt/x.html': {
(
DesktopConfig(port_name='test-linux-precise'),
DesktopConfig(port_name='test-linux-trusty'),
DesktopConfig(port_name='test-mac-mac10.11'),
):
SimpleTestResult(expected='PASS',
actual='TEXT',
bug='crbug.com/test')
}
}
updater.configs_with_no_results = [
DesktopConfig(port_name='test-mac-mac10.10')
]
self.assertEqual(
updater.create_line_dict(results), {
'external/wpt/x.html': [
'crbug.com/test [ Linux ] external/wpt/x.html [ Failure ]',
'crbug.com/test [ Mac ] external/wpt/x.html [ Failure ]',
]
})
def test_normalized_specifiers(self):
updater = WPTExpectationsUpdater(self.mock_host())
self.assertEqual(
updater.normalized_specifiers(
'x/y.html', [DesktopConfig(port_name='test-mac-mac10.10')]),
['Mac10.10'])
self.assertEqual(updater.normalized_specifiers('x/y.html', []), [''])
self.assertEqual(
updater.normalized_specifiers('x/y.html', [
DesktopConfig(port_name='test-mac-mac10.10'),
DesktopConfig(port_name='test-win-win7')
]), ['Mac10.10', 'Win7'])
def test_write_to_test_expectations_without_newline(self):
host = self.mock_host()
expectations_path = \
host.port_factory.get().path_to_generic_test_expectations_file()
skip_path = host.port_factory.get().path_to_never_fix_tests_file()
test_expectations = {
'external/wpt/fake/file/path-manual.html': {
tuple([DesktopConfig(port_name='test-linux-trusty')]):
SimpleTestResult(actual='TIMEOUT', expected={}, bug='')
}
}
host.filesystem.write_text_file(
expectations_path, WPTExpectationsUpdater.MARKER_COMMENT + '\n')
host.filesystem.write_text_file(
skip_path,
'[ Trusty ] external/wpt/fake/file/path-manual.html [ Skip ]')
updater = WPTExpectationsUpdater(host)
updater.write_to_test_expectations(test_expectations)
expectations_value = host.filesystem.read_text_file(expectations_path)
skip_value = host.filesystem.read_text_file(skip_path)
self.assertMultiLineEqual(expectations_value,
WPTExpectationsUpdater.MARKER_COMMENT + '\n')
self.assertMultiLineEqual(
skip_value,
'[ Trusty ] external/wpt/fake/file/path-manual.html [ Skip ]\n'
'[ Trusty ] external/wpt/fake/file/path-manual.html [ Skip ]\n')
def test_write_to_test_expectations_with_no_marker_comment(self):
host = self.mock_host()
expectations_path = \
host.port_factory.get().path_to_generic_test_expectations_file()
host.filesystem.write_text_file(
expectations_path,
'crbug.com/111 [ Trusty ] foo/bar.html [ Failure ]\n')
updater = WPTExpectationsUpdater(host)
test_expectations = {
'external/wpt/fake/file/path.html': {
tuple([DesktopConfig(port_name='test-linux-trusty')]):
SimpleTestResult(actual='PASS',
expected='',
bug='crbug.com/123')
}
}
skip_path = host.port_factory.get().path_to_never_fix_tests_file()
skip_value_origin = host.filesystem.read_text_file(skip_path)
updater.write_to_test_expectations(test_expectations)
value = host.filesystem.read_text_file(expectations_path)
self.assertMultiLineEqual(value, (
'crbug.com/111 [ Trusty ] foo/bar.html [ Failure ]\n'
'\n' + WPTExpectationsUpdater.MARKER_COMMENT + '\n'
'crbug.com/123 [ Trusty ] external/wpt/fake/file/path.html [ Pass ]'
))
skip_value = host.filesystem.read_text_file(skip_path)
self.assertMultiLineEqual(skip_value, skip_value_origin)
def test_generate_failing_results_dict(self):
updater = WPTExpectationsUpdater(self.mock_host())
web_test_list = WebTestResults(
{
"tests": {
"external/wpt/test/name.html": {
"expected": "bar",
"actual": "foo",
"is_unexpected": True,
"has_stderr": True
}
}
},
step_name='blink_web_tests (with patch)')
updater.port_name = lambda b: b.builder_name
self.assertEqual(
updater.generate_failing_results_dict(
Build(builder_name='test-mac-mac10.10', build_number=1),
web_test_list), {
'external/wpt/test/name.html': {
DesktopConfig(port_name='test-mac-mac10.10'):
SimpleTestResult(
expected='bar',
actual='foo',
bug='crbug.com/626703',
)
}
})
def test_create_line_dict_old_tests(self):
# In this example, there are two failures that are not in wpt.
updater = WPTExpectationsUpdater(self.mock_host())
results = {
'fake/test/path.html': {
tuple([DesktopConfig(port_name='one')]):
SimpleTestResult(expected='FAIL',
actual='PASS',
bug='crbug.com/test'),
tuple([DesktopConfig(port_name='two')]):
SimpleTestResult(expected='FAIL',
actual='PASS',
bug='crbug.com/test'),
}
}
self.assertEqual(updater.create_line_dict(results), {})
def test_get_failing_results_dict_some_failing_results(self):
host = self.mock_host()
host.results_fetcher.set_results(
Build('MOCK Try Mac10.10', 123),
WebTestResults({
'tests': {
'external': {
'wpt': {
'x': {
'failing-test.html': {
'expected': 'PASS',
'actual': 'IMAGE',
'is_unexpected': True,
},
},
},
},
},
}))
updater = WPTExpectationsUpdater(host)
results = updater.get_failing_results_dicts(
Build('MOCK Try Mac10.10', 123))
self.assertEqual(results, [{
'external/wpt/x/failing-test.html': {
DesktopConfig(port_name='test-mac-mac10.10'):
SimpleTestResult(
actual='IMAGE',
expected='PASS',
bug='crbug.com/626703',
),
},
}])
def test_normalized_specifiers_with_skipped_test(self):
host = self.mock_host()
expectations_path = '/test.checkout/wtests/NeverFixTests'
host.filesystem.write_text_file(
expectations_path,
('# tags: [ Linux Mac10.11 ]\n'
'# results: [ Skip ]\n'
'crbug.com/111 [ Linux ] external/wpt/test.html [ Skip ]\n'
'crbug.com/111 [ Mac10.11 ] external/wpt/test.html [ Skip ]\n'))
host.filesystem.write_text_file(
'/test.checkout/wtests/external/wpt/test.html', '')
updater = WPTExpectationsUpdater(host)
self.assertEqual(
updater.normalized_specifiers('external/wpt/test.html', [
DesktopConfig(port_name='test-mac-mac10.10'),
DesktopConfig(port_name='test-win-win7'),
DesktopConfig(port_name='test-win-win10')
]), [''])
self.assertEqual(
updater.normalized_specifiers('external/wpt/test.html', [
DesktopConfig(port_name='test-win-win7'),
DesktopConfig(port_name='test-win-win10')
]), ['Win'])
self.assertEqual(
updater.normalized_specifiers('external/wpt/another.html', [
DesktopConfig('test-win-win7'),
DesktopConfig('test-win-win10')
]), ['Win'])
def test_new_manual_tests_get_skip_expectation(self):
host = self.mock_host()
updater = WPTExpectationsUpdater(host)
results = {
'external/wpt/x-manual.html': {
(
DesktopConfig('test-linux-precise'),
DesktopConfig('test-linux-trusty'),
DesktopConfig('test-mac-mac10.10'),
DesktopConfig('test-mac-mac10.11'),
DesktopConfig('test-win-win7'),
DesktopConfig('test-win-win10'),
):
SimpleTestResult(expected='PASS',
actual='MISSING',
bug='crbug.com/test')
}
}
tests_to_rebaseline, _ = updater.get_tests_to_rebaseline(results)
self.assertEqual(tests_to_rebaseline, [])
self.assertEqual(
updater.create_line_dict(results), {
'external/wpt/x-manual.html':
['crbug.com/test external/wpt/x-manual.html [ Skip ]']
})
def test_create_line_dict_with_manual_tests(self):
# In this example, there are two manual tests that should be skipped.
updater = WPTExpectationsUpdater(self.mock_host())
results = {
'virtual/foo/external/wpt/test/aa-manual.html': {
tuple([DesktopConfig(port_name='test-linux-trusty')]):
SimpleTestResult(expected='PASS',
actual='TIMEOUT',
bug='crbug.com/test'),
tuple([DesktopConfig(port_name='test-mac-mac10.11')]):
SimpleTestResult(expected='FAIL',
actual='TIMEOUT',
bug='crbug.com/test'),
},
}
self.assertEqual(
updater.create_line_dict(results), {
'virtual/foo/external/wpt/test/aa-manual.html': [
'crbug.com/test [ Trusty ] virtual/foo/external/wpt/test/aa-manual.html [ Skip ]',
'crbug.com/test [ Mac10.11 ] virtual/foo/external/wpt/test/aa-manual.html [ Skip ]',
],
})
def test_create_line_dict_new_tests(self):
# In this example, there are three unexpected results for wpt tests.
# The new test expectation lines are sorted by test, and then specifier.
updater = WPTExpectationsUpdater(self.mock_host())
results = {
'external/wpt/test/zzzz.html': {
tuple([DesktopConfig(port_name='test-mac-mac10.10')]):
SimpleTestResult(expected='PASS',
actual='TEXT',
bug='crbug.com/test'),
},
'virtual/foo/external/wpt/test/zzzz.html': {
tuple([DesktopConfig(port_name='test-linux-trusty')]):
SimpleTestResult(expected='FAIL',
actual='PASS',
bug='crbug.com/test'),
tuple([DesktopConfig(port_name='test-mac-mac10.11')]):
SimpleTestResult(expected='FAIL',
actual='TIMEOUT',
bug='crbug.com/test'),
},
'unrelated/test.html': {
tuple([DesktopConfig(port_name='test-linux-trusty')]):
SimpleTestResult(expected='FAIL',
actual='PASS',
bug='crbug.com/test'),
},
}
self.assertEqual(
updater.create_line_dict(results), {
'external/wpt/test/zzzz.html': [
'crbug.com/test [ Mac10.10 ] external/wpt/test/zzzz.html [ Failure ]'
],
'virtual/foo/external/wpt/test/zzzz.html': [
'crbug.com/test [ Trusty ] virtual/foo/external/wpt/test/zzzz.html [ Pass ]',
'crbug.com/test [ Mac10.11 ] virtual/foo/external/wpt/test/zzzz.html [ Timeout ]',
],
})
def test_create_line_dict_with_asterisks(self):
# Literal asterisks in test names need to be escaped in expectations.
updater = WPTExpectationsUpdater(self.mock_host())
results = {
'external/wpt/html/dom/interfaces.https.html?exclude=(Document.*|HTML.*)':
{
tuple([DesktopConfig(port_name='test-linux-trusty')]):
SimpleTestResult(expected='PASS',
actual='FAIL',
bug='crbug.com/test'),
},
}
self.assertEqual(
updater.create_line_dict(results), {
'external/wpt/html/dom/interfaces.https.html?exclude=(Document.*|HTML.*)':
[
'crbug.com/test [ Trusty ] external/wpt/html/dom/interfaces.https.html?exclude=(Document.\*|HTML.\*) [ Failure ]',
],
})
