def test_create_line_dict_old_tests(self):
# In this example, there are two failures that are not in wpt.
updater = WPTExpectationsUpdater(self.mock_host())
results = {
'fake/test/path.html': {
'one':
SimpleTestResult(expected='FAIL',
actual='PASS',
bug='crbug.com/test'),
'two':
SimpleTestResult(expected='FAIL',
actual='PASS',
bug='crbug.com/test'),
}
}
self.assertEqual(updater.create_line_dict(results), {})
def test_specifiers_can_extend_to_all_platforms(self):
host = self.mock_host()
expectations_path = '/test.checkout/LayoutTests/NeverFixTests'
host.filesystem.write_text_file(
expectations_path,
'crbug.com/111 [ Linux ] external/wpt/test.html [ WontFix ]\n')
host.filesystem.write_text_file(
'/test.checkout/LayoutTests/external/wpt/test.html', '')
updater = WPTExpectationsUpdater(host)
self.assertTrue(
updater.specifiers_can_extend_to_all_platforms(
['Mac10.10', 'Mac10.11', 'Win7', 'Win10'],
'external/wpt/test.html'))
self.assertFalse(
updater.specifiers_can_extend_to_all_platforms(
['Mac10.10', 'Win7', 'Win10'], 'external/wpt/test.html'))
def test_get_failing_results_dict_non_wpt_test(self):
host = self.mock_host()
host.results_fetcher.set_results(Build('MOCK Try Mac10.10', 123), WebTestResults({
'tests': {
'x': {
'failing-test.html': {
'expected': 'PASS',
'actual': 'IMAGE',
'is_unexpected': True,
},
},
},
}))
updater = WPTExpectationsUpdater(host)
results_dict = updater.get_failing_results_dict(Build('MOCK Try Mac10.10', 123))
self.assertEqual(results_dict, {})
def test_write_to_test_expectations_with_marker_comment(self):
host = self.mock_host()
expectations_path = host.port_factory.get(
).path_to_generic_test_expectations_file()
host.filesystem.write_text_file(expectations_path,
MARKER_COMMENT + '\n')
updater = WPTExpectationsUpdater(host)
line_dict = {
'fake/file/path.html':
['crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]']
}
updater.write_to_test_expectations(line_dict)
value = host.filesystem.read_text_file(expectations_path)
self.assertMultiLineEqual(
value, (MARKER_COMMENT + '\n'
'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]\n'))
def test_merge_same_valued_keys_all_match(self):
updater = WPTExpectationsUpdater(self.mock_host())
self.assertEqual(
updater.merge_same_valued_keys({
'one': {
'expected': 'FAIL',
'actual': 'PASS'
},
'two': {
'expected': 'FAIL',
'actual': 'PASS'
},
}), {('two', 'one'): {
'expected': 'FAIL',
'actual': 'PASS'
}})
def get_updater(host=None, args=None):
host = host or Host()
args = args or []
if ('--update-android-expectations-only' in args or
any(arg.startswith('--android-product') for arg in args)):
return AndroidWPTExpectationsUpdater(host, args)
else:
return WPTExpectationsUpdater(host, args)
def test_get_test_to_rebaseline_returns_only_tests_with_failures(self):
host = self.mock_host()
updater = WPTExpectationsUpdater(host)
two = {
'external/wpt/test/path.html': {
'one':
SimpleTestResult(expected='FAIL', actual='PASS', bug='bug'),
'two':
SimpleTestResult(expected='FAIL', actual='TIMEOUT', bug='bug'),
'three':
SimpleTestResult(expected='FAIL', actual='PASS', bug='bug'),
}
}
tests_to_rebaseline, _ = updater.get_tests_to_rebaseline(two)
# external/wpt/test/zzzz.html is another possible candidate, but it
# is not listed in the results dict, so it shall not be rebaselined.
self.assertEqual(tests_to_rebaseline, ['external/wpt/test/path.html'])
def test_merge_dicts_with_conflict_raise_exception(self):
updater = WPTExpectationsUpdater(self.mock_host())
# Both dicts here have the key "one", and the value is not equal.
with self.assertRaises(ValueError):
updater.merge_dicts(
{
'external/wpt/test/path.html': {
'one': {'expected': 'FAIL', 'actual': 'PASS'},
'two': {'expected': 'FAIL', 'actual': 'TIMEOUT'},
'three': {'expected': 'FAIL', 'actual': 'PASS'},
},
},
{
'external/wpt/test/path.html': {
'one': {'expected': 'FAIL', 'actual': 'TIMEOUT'},
}
})
def test_simplify_specifiers_uses_specifiers_in_builder_list(self):
# Even if there are extra specifiers in the macro dictionary, we can simplify specifier
# lists if they contain all of the specifiers the are represented in the builder list.
# This way specifier simplification can still be done while a new platform is being added.
host = self.mock_host()
updater = WPTExpectationsUpdater(host)
macros = {
'mac': ['Mac10.10', 'mac10.11', 'mac10.14'],
'win': ['Win7', 'win10'],
'Linux': ['Trusty'],
}
self.assertEqual(
updater.simplify_specifiers(['mac10.10', 'mac10.11'], macros),
['Mac'])
self.assertEqual(
updater.simplify_specifiers(
['Mac10.10', 'Mac10.11', 'Trusty', 'Win7', 'Win10'], macros),
[])
def test_create_line_dict_with_manual_tests(self):
# In this example, there are three unexpected results for wpt tests.
# The new test expectation lines are sorted by test, and then specifier.
updater = WPTExpectationsUpdater(self.mock_host())
results = {
'virtual/foo/external/wpt/test/aa-manual.html': {
'test-linux-trusty': SimpleTestResult(expected='PASS', actual='TIMEOUT', bug='crbug.com/test'),
'test-mac-mac10.11': SimpleTestResult(expected='FAIL', actual='TIMEOUT', bug='crbug.com/test'),
},
}
self.assertEqual(
updater.create_line_dict(results),
{
'virtual/foo/external/wpt/test/aa-manual.html': [
'crbug.com/test [ Trusty ] virtual/foo/external/wpt/test/aa-manual.html [ Skip ]',
'crbug.com/test [ Mac10.11 ] virtual/foo/external/wpt/test/aa-manual.html [ Skip ]',
],
})
def test_merge_dicts_merges_second_dict_into_first(self):
updater = WPTExpectationsUpdater(self.mock_host())
one = {
'fake/test/path.html': {
'one': {
'expected': 'FAIL',
'actual': 'PASS'
},
'two': {
'expected': 'FAIL',
'actual': 'PASS'
},
}
}
two = {
'external/wpt/test/path.html': {
'one': {
'expected': 'FAIL',
'actual': 'PASS'
},
'two': {
'expected': 'FAIL',
'actual': 'TIMEOUT'
},
'three': {
'expected': 'FAIL',
'actual': 'PASS'
},
}
}
three = {
'external/wpt/test/path.html': {
'four': {
'expected': 'FAIL',
'actual': 'PASS'
},
}
}
output = updater.merge_dicts(one, three)
self.assertEqual(output, one)
output = updater.merge_dicts(two, three)
self.assertEqual(output, two)
def test_write_to_test_expectations_with_marker_and_no_lines(self):
host = self.mock_host()
expectations_path = host.port_factory.get(
).path_to_generic_test_expectations_file()
host.filesystem.write_text_file(
expectations_path, MARKER_COMMENT + '\n' +
'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]\n')
skip_path = host.port_factory.get().path_to_never_fix_tests_file()
skip_value_origin = host.filesystem.read_text_file(skip_path)
updater = WPTExpectationsUpdater(host)
updater.write_to_test_expectations({})
value = host.filesystem.read_text_file(expectations_path)
self.assertMultiLineEqual(
value, MARKER_COMMENT + '\n' +
'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]\n')
skip_value = host.filesystem.read_text_file(skip_path)
self.assertMultiLineEqual(skip_value, skip_value_origin)
def test_get_tests_to_rebaseline_handles_retries(self):
host = self.mock_host()
results = {
'external/wpt/test/foo.html': {
'bot': SimpleTestResult(expected='PASS', actual='TEXT TEXT', bug='bug'),
},
'external/wpt/test/bar.html': {
'bot': SimpleTestResult(expected='PASS', actual='TIMEOUT TIMEOUT', bug='bug'),
},
}
updater = WPTExpectationsUpdater(host)
tests_to_rebaseline, modified_test_results = updater.get_tests_to_rebaseline(results)
self.assertEqual(tests_to_rebaseline, ['external/wpt/test/foo.html'])
self.assertEqual(modified_test_results, {
'external/wpt/test/foo.html': {},
'external/wpt/test/bar.html': {
'bot': SimpleTestResult(expected='PASS', actual='TIMEOUT TIMEOUT', bug='bug'),
},
})
def test_create_line_dict_with_asterisks(self):
# Literal asterisks in test names need to be escaped in expectations.
updater = WPTExpectationsUpdater(self.mock_host())
results = {
'external/wpt/html/dom/interfaces.https.html?exclude=(Document.*|HTML.*)':
{
'test-linux-trusty':
SimpleTestResult(expected='PASS',
actual='FAIL',
bug='crbug.com/test'),
},
}
self.assertEqual(
updater.create_line_dict(results), {
'external/wpt/html/dom/interfaces.https.html?exclude=(Document.*|HTML.*)':
[
'crbug.com/test [ Trusty ] external/wpt/html/dom/interfaces.https.html?exclude=(Document.\*|HTML.\*) [ Failure ]',
],
})
def test_get_failing_results_dict_only_passing_results(self):
host = self.mock_host()
host.results_fetcher.set_results(Build('MOCK Try Mac10.10', 123), WebTestResults({
'tests': {
'external': {
'wpt': {
'x': {
'passing-test.html': {
'expected': 'PASS',
'actual': 'PASS',
},
},
},
},
},
}))
updater = WPTExpectationsUpdater(host)
self.assertEqual(
updater.get_failing_results_dict(Build('MOCK Try Mac10.10', 123)), {})
def test_get_failing_results_dict_unexpected_pass(self):
host = self.mock_host()
host.buildbot.set_results(Build('MOCK Try Mac10.10', 123), WebTestResults({
'tests': {
'external': {
'wpt': {
'x': {
'passing-test.html': {
'expected': 'FAIL TIMEOUT',
'actual': 'PASS',
'is_unexpected': True,
},
},
},
},
},
}))
updater = WPTExpectationsUpdater(host)
self.assertEqual(
updater.get_failing_results_dict(Build('MOCK Try Mac10.10', 123)), {})
def test_generate_results_dict(self):
updater = WPTExpectationsUpdater(self.mock_host())
web_test_list = [
WebTestResult(
'external/wpt/test/name.html', {
'expected': 'bar',
'actual': 'foo',
'is_unexpected': True,
'has_stderr': True,
})
]
self.assertEqual(updater.generate_results_dict('test-mac-mac10.10', web_test_list), {
'external/wpt/test/name.html': {
'test-mac-mac10.10': SimpleTestResult(
expected='bar',
actual='foo',
bug='crbug.com/626703',
)
}
})
def test_write_to_test_expectations_with_webdriver_lines(self):
host = self.mock_host()
webdriver_expectations_path = \
host.port_factory.get().path_to_webdriver_expectations_file()
host.filesystem.write_text_file(
webdriver_expectations_path,
WPTExpectationsUpdater.MARKER_COMMENT + '\n')
updater = WPTExpectationsUpdater(host)
test_expectations = {
'external/wpt/webdriver/fake/file/path.html': {
tuple([DesktopConfig(port_name='test-linux-trusty')]):
SimpleTestResult(actual='PASS',
expected='',
bug='crbug.com/123')
}
}
expectations_path = \
host.port_factory.get().path_to_generic_test_expectations_file()
expectations_value_origin = host.filesystem.read_text_file(
expectations_path)
skip_path = host.port_factory.get().path_to_never_fix_tests_file()
skip_value_origin = host.filesystem.read_text_file(skip_path)
updater.write_to_test_expectations(test_expectations)
value = host.filesystem.read_text_file(webdriver_expectations_path)
self.assertMultiLineEqual(value, (
WPTExpectationsUpdater.MARKER_COMMENT + '\n'
'crbug.com/123 [ Trusty ] external/wpt/webdriver/fake/file/path.html [ Pass ]\n'
))
skip_value = host.filesystem.read_text_file(skip_path)
self.assertMultiLineEqual(skip_value, skip_value_origin)
expectations_value = host.filesystem.read_text_file(expectations_path)
self.assertMultiLineEqual(expectations_value,
expectations_value_origin)
def test_new_manual_tests_get_skip_expectation(self):
host = self.mock_host()
updater = WPTExpectationsUpdater(host)
results = {
'external/wpt/x-manual.html': {
(
'test-linux-precise',
'test-linux-trusty',
'test-mac-mac10.10',
'test-mac-mac10.11',
'test-win-win7',
'test-win-win10',
): SimpleTestResult(expected='PASS', actual='MISSING', bug='crbug.com/test')
}
}
tests_to_rebaseline, _ = updater.get_tests_to_rebaseline(results)
self.assertEqual(tests_to_rebaseline, [])
self.assertEqual(
updater.create_line_dict(results),
{'external/wpt/x-manual.html': ['crbug.com/test external/wpt/x-manual.html [ Skip ]']}
)
def test_get_tests_to_rebaseline_also_returns_slow_tests(self):
host = self.mock_host()
results = {
'external/wpt/test/path.html': {
'one': SimpleTestResult(expected='SLOW', actual='TEXT', bug='bug'),
'two': SimpleTestResult(expected='SLOW', actual='TIMEOUT', bug='bug'),
},
}
results_copy = copy.deepcopy(results)
updater = WPTExpectationsUpdater(host)
tests_to_rebaseline, modified_test_results = updater.get_tests_to_rebaseline(results)
self.assertEqual(tests_to_rebaseline, ['external/wpt/test/path.html'])
# The record for the builder with a timeout is kept, but not with a text mismatch,
# since that should be covered by downloading a new baseline.
self.assertEqual(modified_test_results, {
'external/wpt/test/path.html': {
'two': SimpleTestResult(expected='SLOW', actual='TIMEOUT', bug='bug'),
},
})
# The original dict isn't modified.
self.assertEqual(results, results_copy)
def test_get_test_to_rebaseline_does_not_return_ref_tests(self):
host = self.mock_host()
updater = WPTExpectationsUpdater(host)
two = {
'external/wpt/reftest.html': {
'one':
SimpleTestResult(expected='FAIL',
actual='PASS',
bug='crbug.com/test'),
'two':
SimpleTestResult(expected='FAIL',
actual='TIMEOUT',
bug='crbug.com/test'),
'three':
SimpleTestResult(expected='FAIL',
actual='PASS',
bug='crbug.com/test'),
}
}
tests_to_rebaseline, _ = updater.get_tests_to_rebaseline(two)
self.assertEqual(tests_to_rebaseline, [])
def test_one_platform_has_no_results(self):
# In this example, there is a failure that has been observed on
# Linux and one Mac port, but the other Mac port has no results at all.
# The specifiers are "filled in" and the failure is assumed to apply
# to all Mac platforms.
host = self.mock_host()
updater = WPTExpectationsUpdater(host)
results = {
'external/wpt/x.html': {
(
'test-linux-precise',
'test-linux-trusty',
'test-mac-mac10.11',
): SimpleTestResult(expected='PASS', actual='TEXT', bug='crbug.com/test')
}
}
updater.ports_with_no_results = {'test-mac-mac10.10'}
self.assertEqual(
updater.create_line_dict(results),
{'external/wpt/x.html': ['crbug.com/test [ Linux Mac ] external/wpt/x.html [ Failure ]']}
)
def test_specifier_part_with_skipped_test(self):
host = self.mock_host()
expectations_path = '/test.checkout/wtests/NeverFixTests'
host.filesystem.write_text_file(
expectations_path,
'crbug.com/111 [ Linux Mac10.11 ] external/wpt/test.html [ WontFix ]\n'
)
host.filesystem.write_text_file(
'/test.checkout/wtests/external/wpt/test.html', '')
updater = WPTExpectationsUpdater(host)
self.assertEqual(
updater.specifier_part(
'external/wpt/test.html',
['test-mac-mac10.10', 'test-win-win7', 'test-win-win10']), '')
self.assertEqual(
updater.specifier_part('external/wpt/test.html',
['test-win-win7', 'test-win-win10']),
'[ Win ]')
self.assertEqual(
updater.specifier_part('external/wpt/another.html',
['test-win-win7', 'test-win-win10']),
'[ Win ]')
def test_create_line_dict_with_manual_tests(self):
# In this example, there are two manual tests that should be skipped.
updater = WPTExpectationsUpdater(self.mock_host())
results = {
'virtual/foo/external/wpt/test/aa-manual.html': {
'test-linux-trusty':
SimpleTestResult(expected='PASS',
actual='TIMEOUT',
bug='crbug.com/test'),
'test-mac-mac10.11':
SimpleTestResult(expected='FAIL',
actual='TIMEOUT',
bug='crbug.com/test'),
},
}
self.assertEqual(
updater.create_line_dict(results), {
'virtual/foo/external/wpt/test/aa-manual.html': [
'crbug.com/test [ Trusty ] virtual/foo/external/wpt/test/aa-manual.html [ Skip ]',
'crbug.com/test [ Mac10.11 ] virtual/foo/external/wpt/test/aa-manual.html [ Skip ]',
],
})
def test_create_line_dict_new_tests(self):
# In this example, there are three unexpected results for wpt tests.
# The new test expectation lines are sorted by test, and then specifier.
updater = WPTExpectationsUpdater(self.mock_host())
results = {
'external/wpt/test/zzzz.html': {
'test-mac-mac10.10':
SimpleTestResult(expected='PASS',
actual='TEXT',
bug='crbug.com/test'),
},
'virtual/foo/external/wpt/test/zzzz.html': {
'test-linux-trusty':
SimpleTestResult(expected='FAIL',
actual='PASS',
bug='crbug.com/test'),
'test-mac-mac10.11':
SimpleTestResult(expected='FAIL',
actual='TIMEOUT',
bug='crbug.com/test'),
},
'unrelated/test.html': {
'test-linux-trusty':
SimpleTestResult(expected='FAIL',
actual='PASS',
bug='crbug.com/test'),
},
}
self.assertEqual(
updater.create_line_dict(results), {
'external/wpt/test/zzzz.html': [
'crbug.com/test [ Mac10.10 ] external/wpt/test/zzzz.html [ Failure ]'
],
'virtual/foo/external/wpt/test/zzzz.html': [
'crbug.com/test [ Trusty ] virtual/foo/external/wpt/test/zzzz.html [ Pass ]',
'crbug.com/test [ Mac10.11 ] virtual/foo/external/wpt/test/zzzz.html [ Timeout ]',
],
})
def test_write_to_test_expectations_with_no_marker_comment(self):
host = self.mock_host()
expectations_path = host.port_factory.get().path_to_generic_test_expectations_file()
host.filesystem.write_text_file(
expectations_path,
'crbug.com/111 [ Trusty ] foo/bar.html [ Failure ]\n')
updater = WPTExpectationsUpdater(host)
line_dict = {'fake/file/path.html': ['crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]']}
skip_path = host.port_factory.get().path_to_never_fix_tests_file()
skip_value_origin = host.filesystem.read_text_file(skip_path)
updater.write_to_test_expectations(line_dict)
value = host.filesystem.read_text_file(expectations_path)
self.assertMultiLineEqual(
value,
('crbug.com/111 [ Trusty ] foo/bar.html [ Failure ]\n'
'\n' + MARKER_COMMENT + '\n'
'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]'))
skip_value = host.filesystem.read_text_file(skip_path)
self.assertMultiLineEqual(
skip_value,
skip_value_origin)
def test_write_to_test_expectations_without_newline(self):
host = self.mock_host()
expectations_path = host.port_factory.get(
).path_to_generic_test_expectations_file()
skip_path = host.port_factory.get().path_to_never_fix_tests_file()
line_dict = {
'fake/file/path-manual.html':
['[ Trusty ] fake/file/path-manual.html [ Skip ]']
}
host.filesystem.write_text_file(expectations_path,
MARKER_COMMENT + '\n')
host.filesystem.write_text_file(
skip_path, '[ Trusty ] fake/file/path-manual.html [ Skip ]')
updater = WPTExpectationsUpdater(host)
updater.write_to_test_expectations(line_dict)
expectations_value = host.filesystem.read_text_file(expectations_path)
skip_value = host.filesystem.read_text_file(skip_path)
self.assertMultiLineEqual(expectations_value, MARKER_COMMENT + '\n')
self.assertMultiLineEqual(
skip_value, '[ Trusty ] fake/file/path-manual.html [ Skip ]\n'
'[ Trusty ] fake/file/path-manual.html [ Skip ]\n')
def test_merging_platforms_if_possible(self):
host = self.mock_host()
updater = WPTExpectationsUpdater(host)
results = {
'external/wpt/x.html': {
(
'test-linux-precise',
'test-linux-trusty',
'test-mac-mac10.10',
'test-mac-mac10.11',
'test-win-win7',
): SimpleTestResult(expected='PASS', actual='TEXT', bug='crbug.com/test')
}
}
self.assertEqual(
updater.create_line_dict(results),
{'external/wpt/x.html': [
'crbug.com/test [ Linux ] external/wpt/x.html [ Failure ]',
'crbug.com/test [ Mac ] external/wpt/x.html [ Failure ]',
'crbug.com/test [ Win7 ] external/wpt/x.html [ Failure ]',
]
}
)
def __init__(self, host, wpt_github=None, wpt_manifests=None):
self.host = host
self.wpt_github = wpt_github
self.executive = host.executive
self.fs = host.filesystem
self.finder = PathFinder(self.fs)
self.chromium_git = self.host.git(self.finder.chromium_base())
self.dest_path = self.finder.path_from_web_tests('external', 'wpt')
# A common.net.git_cl.GitCL instance.
self.git_cl = None
# Another Git instance with local WPT as CWD, which can only be
# instantiated after the working directory is created.
self.wpt_git = None
# The WPT revision we are importing and the one imported last time.
self.wpt_revision = None
self.last_wpt_revision = None
# A set of rebaselined tests and a dictionary of new test expectations
# mapping failing tests to platforms to
# wpt_expectations_updater.SimpleTestResult.
self.rebaselined_tests = set()
self.new_test_expectations = {}
# New override expectations for Android targets. A dictionary mapping
# products to dictionary that maps test names to test expectation lines
self.new_override_expectations = {}
self.verbose = False
args = [
'--clean-up-affected-tests-only', '--clean-up-test-expectations'
]
self._expectations_updater = WPTExpectationsUpdater(
self.host, args, wpt_manifests)
args = ['--android-product', 'android_weblayer']
self._android_expectations_updater = AndroidWPTExpectationsUpdater(
self.host, args, wpt_manifests)
def test_write_to_test_expectations_with_webdriver_lines(self):
host = self.mock_host()
webdriver_expectations_path = host.port_factory.get(
).path_to_webdriver_expectations_file()
host.filesystem.write_text_file(webdriver_expectations_path,
MARKER_COMMENT + '\n')
updater = WPTExpectationsUpdater(host)
line_dict = {
'external/wpt/webdriver/fake/file/path.html': [
'crbug.com/123 [ Trusty ] external/wpt/webdriver/fake/file/path.html [ Pass ]'
]
}
expectations_path = host.port_factory.get(
).path_to_generic_test_expectations_file()
expectations_value_origin = host.filesystem.read_text_file(
expectations_path)
skip_path = host.port_factory.get().path_to_never_fix_tests_file()
skip_value_origin = host.filesystem.read_text_file(skip_path)
updater.write_to_test_expectations(line_dict)
value = host.filesystem.read_text_file(webdriver_expectations_path)
self.assertMultiLineEqual(value, (
MARKER_COMMENT + '\n'
'crbug.com/123 [ Trusty ] external/wpt/webdriver/fake/file/path.html [ Pass ]\n'
))
skip_value = host.filesystem.read_text_file(skip_path)
self.assertMultiLineEqual(skip_value, skip_value_origin)
expectations_value = host.filesystem.read_text_file(expectations_path)
self.assertMultiLineEqual(expectations_value,
expectations_value_origin)
