def test_merging_platforms_if_possible(self):
host = self.mock_host()
updater = WPTExpectationsUpdater(host)
results = {
'external/wpt/x.html': {
(
'test-linux-precise',
'test-linux-trusty',
'test-mac-mac10.10',
'test-mac-mac10.11',
'test-win-win7',
):
SimpleTestResult(expected='PASS',
actual='TEXT',
bug='crbug.com/test')
}
}
self.assertEqual(
updater.create_line_dict(results), {
'external/wpt/x.html': [
'crbug.com/test [ Linux ] external/wpt/x.html [ Failure ]',
'crbug.com/test [ Mac ] external/wpt/x.html [ Failure ]',
'crbug.com/test [ Win7 ] external/wpt/x.html [ Failure ]',
]
})
def test_one_platform_has_no_results(self):
# In this example, there is a failure that has been observed on
# Linux and one Mac port, but the other Mac port has no results at all.
# The specifiers are "filled in" and the failure is assumed to apply
# to all Mac platforms.
host = self.mock_host()
updater = WPTExpectationsUpdater(host)
results = {
'external/wpt/x.html': {
(
'test-linux-precise',
'test-linux-trusty',
'test-mac-mac10.11',
):
SimpleTestResult(expected='PASS',
actual='TEXT',
bug='crbug.com/test')
}
}
updater.ports_with_no_results = {'test-mac-mac10.10'}
self.assertEqual(
updater.create_line_dict(results), {
'external/wpt/x.html': [
'crbug.com/test [ Linux ] external/wpt/x.html [ Failure ]',
'crbug.com/test [ Mac ] external/wpt/x.html [ Failure ]',
]
})
def test_new_manual_tests_get_skip_expectation(self):
host = self.mock_host()
updater = WPTExpectationsUpdater(host)
results = {
'external/wpt/x-manual.html': {
(
'test-linux-precise',
'test-linux-trusty',
'test-mac-mac10.10',
'test-mac-mac10.11',
'test-win-win7',
'test-win-win10',
):
SimpleTestResult(expected='PASS',
actual='MISSING',
bug='crbug.com/test')
}
}
tests_to_rebaseline, _ = updater.get_tests_to_rebaseline(results)
self.assertEqual(tests_to_rebaseline, [])
self.assertEqual(
updater.create_line_dict(results), {
'external/wpt/x-manual.html':
['crbug.com/test external/wpt/x-manual.html [ Skip ]']
})
def test_create_line_dict_new_tests(self):
# In this example, there are three unexpected results for wpt tests.
# The new test expectation lines are sorted by test, and then specifier.
updater = WPTExpectationsUpdater(self.mock_host())
results = {
'external/wpt/test/zzzz.html': {
'test-mac-mac10.10': SimpleTestResult(expected='PASS', actual='TEXT', bug='crbug.com/test'),
},
'virtual/foo/external/wpt/test/zzzz.html': {
'test-linux-trusty': SimpleTestResult(expected='FAIL', actual='PASS', bug='crbug.com/test'),
'test-mac-mac10.11': SimpleTestResult(expected='FAIL', actual='TIMEOUT', bug='crbug.com/test'),
},
'unrelated/test.html': {
'test-linux-trusty': SimpleTestResult(expected='FAIL', actual='PASS', bug='crbug.com/test'),
},
}
self.assertEqual(
updater.create_line_dict(results),
{
'external/wpt/test/zzzz.html': ['crbug.com/test [ Mac10.10 ] external/wpt/test/zzzz.html [ Failure ]'],
'virtual/foo/external/wpt/test/zzzz.html': [
'crbug.com/test [ Trusty ] virtual/foo/external/wpt/test/zzzz.html [ Pass ]',
'crbug.com/test [ Mac10.11 ] virtual/foo/external/wpt/test/zzzz.html [ Timeout ]',
],
})
def test_create_line_dict_old_tests(self):
# In this example, there are two failures that are not in wpt.
updater = WPTExpectationsUpdater(self.mock_host())
results = {
'fake/test/path.html': {
'one': SimpleTestResult(expected='FAIL', actual='PASS', bug='crbug.com/test'),
'two': SimpleTestResult(expected='FAIL', actual='PASS', bug='crbug.com/test'),
}
}
self.assertEqual(updater.create_line_dict(results), {})
def test_create_line_dict_with_manual_tests(self):
# In this example, there are three unexpected results for wpt tests.
# The new test expectation lines are sorted by test, and then specifier.
updater = WPTExpectationsUpdater(self.mock_host())
results = {
'virtual/foo/external/wpt/test/aa-manual.html': {
'test-linux-trusty': SimpleTestResult(expected='PASS', actual='TIMEOUT', bug='crbug.com/test'),
'test-mac-mac10.11': SimpleTestResult(expected='FAIL', actual='TIMEOUT', bug='crbug.com/test'),
},
}
self.assertEqual(
updater.create_line_dict(results),
{
'virtual/foo/external/wpt/test/aa-manual.html': [
'crbug.com/test [ Trusty ] virtual/foo/external/wpt/test/aa-manual.html [ Skip ]',
'crbug.com/test [ Mac10.11 ] virtual/foo/external/wpt/test/aa-manual.html [ Skip ]',
],
})
def test_create_line_dict_with_asterisks(self):
# Literal asterisks in test names need to be escaped in expectations.
updater = WPTExpectationsUpdater(self.mock_host())
results = {
'external/wpt/html/dom/interfaces.https.html?exclude=(Document.*|HTML.*)':
{
'test-linux-trusty':
SimpleTestResult(expected='PASS',
actual='FAIL',
bug='crbug.com/test'),
},
}
self.assertEqual(
updater.create_line_dict(results), {
'external/wpt/html/dom/interfaces.https.html?exclude=(Document.*|HTML.*)':
[
'crbug.com/test [ Trusty ] external/wpt/html/dom/interfaces.https.html?exclude=(Document.\*|HTML.\*) [ Failure ]',
],
})
def test_create_line_dict_with_manual_tests(self):
# In this example, there are two manual tests that should be skipped.
updater = WPTExpectationsUpdater(self.mock_host())
results = {
'virtual/foo/external/wpt/test/aa-manual.html': {
'test-linux-trusty':
SimpleTestResult(expected='PASS',
actual='TIMEOUT',
bug='crbug.com/test'),
'test-mac-mac10.11':
SimpleTestResult(expected='FAIL',
actual='TIMEOUT',
bug='crbug.com/test'),
},
}
self.assertEqual(
updater.create_line_dict(results), {
'virtual/foo/external/wpt/test/aa-manual.html': [
'crbug.com/test [ Trusty ] virtual/foo/external/wpt/test/aa-manual.html [ Skip ]',
'crbug.com/test [ Mac10.11 ] virtual/foo/external/wpt/test/aa-manual.html [ Skip ]',
],
})
