def test_merge_dicts_merges_second_dict_into_first(self):
updater = WPTExpectationsUpdater(self.mock_host())
one = {
'fake/test/path.html': {
'one': {'expected': 'FAIL', 'actual': 'PASS'},
'two': {'expected': 'FAIL', 'actual': 'PASS'},
}
}
two = {
'external/wpt/test/path.html': {
'one': {'expected': 'FAIL', 'actual': 'PASS'},
'two': {'expected': 'FAIL', 'actual': 'TIMEOUT'},
'three': {'expected': 'FAIL', 'actual': 'PASS'},
}
}
three = {
'external/wpt/test/path.html': {
'four': {'expected': 'FAIL', 'actual': 'PASS'},
}
}
output = updater.merge_dicts(one, three)
self.assertEqual(output, one)
output = updater.merge_dicts(two, three)
self.assertEqual(output, two)
def test_run_single_platform_failure(self):
"""Tests the main run method in a case where one test fails on one platform."""
host = self.mock_host()
# Fill in an initial value for TestExpectations
expectations_path = host.port_factory.get(
).path_to_generic_test_expectations_file()
host.filesystem.write_text_file(expectations_path,
MARKER_COMMENT + '\n')
# Set up fake try job results.
updater = WPTExpectationsUpdater(host)
updater.git_cl = MockGitCL(
updater.host, {
Build('MOCK Try Mac10.10', 333):
TryJobStatus('COMPLETED', 'FAILURE'),
Build('MOCK Try Mac10.11', 111):
TryJobStatus('COMPLETED', 'SUCCESS'),
Build('MOCK Try Trusty', 222):
TryJobStatus('COMPLETED', 'SUCCESS'),
Build('MOCK Try Precise', 333):
TryJobStatus('COMPLETED', 'SUCCESS'),
Build('MOCK Try Win10', 444):
TryJobStatus('COMPLETED', 'SUCCESS'),
Build('MOCK Try Win7', 555):
TryJobStatus('COMPLETED', 'SUCCESS'),
})
# Set up failing results for one try bot. It shouldn't matter what
# results are for the other builders since we shouldn't need to even
# fetch results, since the try job status already tells us that all
# of the tests passed.
host.buildbot.set_results(
Build('MOCK Try Mac10.10', 333),
LayoutTestResults({
'tests': {
'external': {
'wpt': {
'test': {
'path.html': {
'expected': 'PASS',
'actual': 'TIMEOUT',
'is_unexpected': True,
}
}
}
}
}
}))
self.assertEqual(0, updater.run(args=[]))
# Results are only fetched for failing builds.
self.assertEqual(host.buildbot.fetched_builds,
[Build('MOCK Try Mac10.10', 333)])
self.assertEqual(
host.filesystem.read_text_file(expectations_path),
'# ====== New tests from wpt-importer added here ======\n'
'crbug.com/626703 [ Mac10.10 ] external/wpt/test/path.html [ Timeout ]\n'
)
def test_get_test_to_rebaseline_returns_only_js_tests(self):
host = self.mock_host()
host.filesystem.files[
'/mock-checkout/third_party/WebKit/LayoutTests/external/fake/test/path.html'] = (
'this file does not look like a testharness JS test.')
updater = WPTExpectationsUpdater(host)
two = {
'external/fake/test/path.html': {
'one': {
'expected': 'FAIL',
'actual': 'PASS',
'bug': 'crbug.com/test'
},
'two': {
'expected': 'FAIL',
'actual': 'TIMEOUT',
'bug': 'crbug.com/test'
},
'three': {
'expected': 'FAIL',
'actual': 'PASS',
'bug': 'crbug.com/test'
},
}
}
tests_to_rebaseline, _ = updater.get_tests_to_rebaseline(two)
self.assertEqual(tests_to_rebaseline, [])
def test_get_tests_to_rebaseline_also_returns_slow_tests(self):
host = self.mock_host()
test_results_dict = {
'external/fake/test/path.html': {
'one': {
'expected': 'SLOW',
'actual': 'TEXT'
},
'two': {
'expected': 'SLOW',
'actual': 'TIMEOUT'
},
},
}
test_results_dict_copy = copy.deepcopy(test_results_dict)
host.filesystem.files[
'/mock-checkout/third_party/WebKit/LayoutTests/external/fake/test/path.html'] = (
'<script src="/resources/testharness.js"></script>')
updater = WPTExpectationsUpdater(host)
tests_to_rebaseline, modified_test_results = updater.get_tests_to_rebaseline(
test_results_dict)
self.assertEqual(tests_to_rebaseline, ['external/fake/test/path.html'])
# The record for the builder with a timeout is kept, but not with a text mismatch,
# since that should be covered by downloading a new baseline.
self.assertEqual(
modified_test_results, {
'external/fake/test/path.html': {
'two': {
'expected': 'SLOW',
'actual': 'TIMEOUT'
},
},
})
# The original dict isn't modified.
self.assertEqual(test_results_dict, test_results_dict_copy)
def test_get_test_to_rebaseline_returns_only_tests_with_failures(self):
host = self.mock_host()
host.filesystem.files[
'/mock-checkout/third_party/WebKit/LayoutTests/external/fake/test/path.html'] = (
'<script src="/resources/testharness.js"></script>')
host.filesystem.files[
'/mock-checkout/third_party/WebKit/LayoutTests/external/other/test/path.html'] = (
'<script src="/resources/testharness.js"></script>')
updater = WPTExpectationsUpdater(host)
two = {
'external/fake/test/path.html': {
'one': {
'expected': 'FAIL',
'actual': 'PASS'
},
'two': {
'expected': 'FAIL',
'actual': 'TIMEOUT'
},
'three': {
'expected': 'FAIL',
'actual': 'PASS'
},
}
}
tests_to_rebaseline, _ = updater.get_tests_to_rebaseline(two)
# The other test doesn't have an entry in the test results dict, so it is not listed as a test to rebaseline.
self.assertEqual(tests_to_rebaseline, ['external/fake/test/path.html'])
def test_is_js_test_false(self):
host = self.mock_host()
host.filesystem.files[
'/mock-checkout/third_party/WebKit/LayoutTests/foo/bar.html'] = (
'<script src="ref-test.html"></script>')
updater = WPTExpectationsUpdater(host)
self.assertFalse(updater.is_js_test('foo/bar.html'))
def test_is_js_test_true(self):
host = self.mock_host()
host.filesystem.files[
'/mock-checkout/third_party/WebKit/LayoutTests/foo/bar.html'] = (
'<script src="/resources/testharness.js"></script>')
updater = WPTExpectationsUpdater(host)
self.assertTrue(updater.is_js_test('foo/bar.html'))
def test_get_failing_results_dict_no_results(self):
host = self.mock_host()
host.buildbot = MockBuildBot()
host.buildbot.set_results(Build('MOCK Try Mac10.10', 123), None)
updater = WPTExpectationsUpdater(host)
self.assertEqual(
updater.get_failing_results_dict(Build('MOCK Try Mac10.10', 123)), {})
def test_one_platform_has_no_results(self):
# In this example, there is a failure that has been observed on
# Linux and one Mac port, but the other Mac port has no results at all.
# The specifiers are "filled in" and the failure is assumed to apply
# to all Mac platforms.
host = self.mock_host()
updater = WPTExpectationsUpdater(host)
results = {
'external/wpt/x.html': {
(
'test-linux-precise',
'test-linux-trusty',
'test-mac-mac10.11',
):
SimpleTestResult(expected='PASS',
actual='TEXT',
bug='crbug.com/test')
}
}
updater.ports_with_no_results = {'test-mac-mac10.10'}
self.assertEqual(
updater.create_line_dict(results), {
'external/wpt/x.html': [
'crbug.com/test [ Linux Mac ] external/wpt/x.html [ Failure ]'
]
})
def test_merge_same_valued_keys_one_mismatch(self):
updater = WPTExpectationsUpdater(self.mock_host())
self.assertEqual(
updater.merge_same_valued_keys({
'one': {
'expected': 'FAIL',
'actual': 'PASS'
},
'two': {
'expected': 'FAIL',
'actual': 'TIMEOUT'
},
'three': {
'expected': 'FAIL',
'actual': 'PASS'
},
}), {
('three', 'one'): {
'expected': 'FAIL',
'actual': 'PASS'
},
'two': {
'expected': 'FAIL',
'actual': 'TIMEOUT'
},
})
def test_run_no_issue_number(self):
# TODO(qyearsley): For testing: Consider making a MockGitCL class
# and use that class to set fake return values when using git cl.
updater = WPTExpectationsUpdater(self.mock_host())
updater.get_issue_number = lambda: 'None'
self.assertEqual(1, updater.run(args=[]))
self.assertLog(['ERROR: No issue on current branch.\n'])
def test_get_failing_results_dict_some_failing_results(self):
host = self.mock_host()
host.buildbot.set_results(
Build('MOCK Try Mac10.10', 123),
LayoutTestResults({
'tests': {
'external': {
'wpt': {
'x': {
'failing-test.html': {
'expected': 'PASS',
'actual': 'IMAGE',
'is_unexpected': True,
},
},
},
},
},
}))
updater = WPTExpectationsUpdater(host)
results_dict = updater.get_failing_results_dict(
Build('MOCK Try Mac10.10', 123))
self.assertEqual(
results_dict, {
'external/wpt/x/failing-test.html': {
'test-mac-mac10.10':
SimpleTestResult(
actual='IMAGE',
expected='PASS',
bug='crbug.com/626703',
),
},
})
def test_get_tests_to_rebaseline_also_returns_slow_tests(self):
host = self.mock_host()
results = {
'external/wpt/test/path.html': {
'one':
SimpleTestResult(expected='SLOW', actual='TEXT', bug='bug'),
'two':
SimpleTestResult(expected='SLOW', actual='TIMEOUT', bug='bug'),
},
}
results_copy = copy.deepcopy(results)
updater = WPTExpectationsUpdater(host)
tests_to_rebaseline, modified_test_results = updater.get_tests_to_rebaseline(
results)
self.assertEqual(tests_to_rebaseline, ['external/wpt/test/path.html'])
# The record for the builder with a timeout is kept, but not with a text mismatch,
# since that should be covered by downloading a new baseline.
self.assertEqual(
modified_test_results, {
'external/wpt/test/path.html': {
'two':
SimpleTestResult(
expected='SLOW', actual='TIMEOUT', bug='bug'),
},
})
# The original dict isn't modified.
self.assertEqual(results, results_copy)
def test_create_line_list_new_tests(self):
# In this example, there are three unexpected results. The new
# test expectation lines are sorted by test, and then specifier.
updater = WPTExpectationsUpdater(self.mock_host())
results = {
'external/fake/test/zzzz.html': {
'test-mac-mac10.10': {
'expected': 'PASS',
'actual': 'FAIL',
'bug': 'crbug.com/test'
},
},
'external/fake/test/path.html': {
'test-linux-trusty': {
'expected': 'FAIL',
'actual': 'PASS',
'bug': 'crbug.com/test'
},
'test-mac-mac10.11': {
'expected': 'FAIL',
'actual': 'TIMEOUT',
'bug': 'crbug.com/test'
},
},
}
self.assertEqual(updater.create_line_list(results), [
'crbug.com/test [ Trusty ] external/fake/test/path.html [ Pass ]',
'crbug.com/test [ Mac10.11 ] external/fake/test/path.html [ Timeout ]',
'crbug.com/test [ Mac10.10 ] external/fake/test/zzzz.html [ Failure ]',
])
def test_run_no_try_results(self):
updater = WPTExpectationsUpdater(self.mock_host())
updater.git_cl = MockGitCL(updater.host, {})
with self.assertRaises(ScriptError) as e:
updater.run(args=[])
self.assertEqual(e.exception.message,
'No try job information was collected.')
def test_new_manual_tests_get_skip_expectation(self):
host = self.mock_host()
updater = WPTExpectationsUpdater(host)
results = {
'external/wpt/x-manual.html': {
(
'test-linux-precise',
'test-linux-trusty',
'test-mac-mac10.10',
'test-mac-mac10.11',
'test-win-win7',
'test-win-win10',
):
SimpleTestResult(expected='PASS',
actual='MISSING',
bug='crbug.com/test')
}
}
tests_to_rebaseline, _ = updater.get_tests_to_rebaseline(results)
self.assertEqual(tests_to_rebaseline, [])
self.assertEqual(
updater.create_line_dict(results), {
'external/wpt/x-manual.html':
['crbug.com/test external/wpt/x-manual.html [ Skip ]']
})
def test_merge_dicts_with_conflict_raise_exception(self):
updater = WPTExpectationsUpdater(self.mock_host())
# Both dicts here have the key "one", and the value is not equal.
with self.assertRaises(ValueError):
updater.merge_dicts(
{
'external/wpt/test/path.html': {
'one': {
'expected': 'FAIL',
'actual': 'PASS'
},
'two': {
'expected': 'FAIL',
'actual': 'TIMEOUT'
},
'three': {
'expected': 'FAIL',
'actual': 'PASS'
},
},
}, {
'external/wpt/test/path.html': {
'one': {
'expected': 'FAIL',
'actual': 'TIMEOUT'
},
}
})
def fetch_new_expectations_and_baselines(self):
"""Adds new expectations and downloads baselines based on try job results, then commits and uploads the change."""
_log.info('Adding test expectations lines to LayoutTests/TestExpectations.')
expectation_updater = WPTExpectationsUpdater(self.host)
expectation_updater.run(args=[])
message = 'Update test expectations and baselines.'
self.check_run(['git', 'commit', '-a', '-m', message])
self.git_cl.run(['upload', '-t', message, '--gerrit'])
def test_skipped_specifiers_when_test_is_wontfix(self):
host = self.mock_host()
expectations_path = '/test.checkout/LayoutTests/NeverFixTests'
host.filesystem.write_text_file(
expectations_path,
'crbug.com/111 [ Linux ] external/wpt/test.html [ WontFix ]\n')
host.filesystem.write_text_file('/test.checkout/LayoutTests/external/wpt/test.html', '')
updater = WPTExpectationsUpdater(host)
self.assertEqual(updater.skipped_specifiers('external/wpt/test.html'), ['Precise', 'Trusty'])
def test_skipped_specifiers_when_test_is_wontfix(self):
host = self.mock_host()
expectations_path = '/test.checkout/LayoutTests/NeverFixTests'
host.filesystem.files[
expectations_path] = 'crbug.com/111 [ Trusty ] external/wpt/test.html [ WontFix ]\n'
host.filesystem.files[
'/test.checkout/LayoutTests/external/wpt/test.html'] = ''
updater = WPTExpectationsUpdater(host)
self.assertEqual(updater.skipped_specifiers('external/wpt/test.html'),
['Trusty'])
def test_create_line_list_old_tests(self):
# In this example, there are two failures that are not in wpt.
updater = WPTExpectationsUpdater(self.mock_host())
results = {
'fake/test/path.html': {
'one': {'expected': 'FAIL', 'actual': 'PASS', 'bug': 'crbug.com/test'},
'two': {'expected': 'FAIL', 'actual': 'PASS', 'bug': 'crbug.com/test'},
}
}
self.assertEqual(updater.create_line_list(results), [])
def fetch_new_expectations_and_baselines(self):
"""Modifies expectation lines and baselines based on try job results.
Assuming that there are some try job results available, this
adds new expectation lines to TestExpectations and downloads new
baselines based on the try job results.
This is the same as invoking the `wpt-update-expectations` script.
"""
_log.info('Adding test expectations lines to LayoutTests/TestExpectations.')
expectation_updater = WPTExpectationsUpdater(self.host)
self.rebaselined_tests, self.new_test_expectations = expectation_updater.update_expectations()
def test_get_test_to_rebaseline_does_not_return_ref_tests(self):
host = self.mock_host()
updater = WPTExpectationsUpdater(host)
two = {
'external/wpt/reftest.html': {
'one': {'expected': 'FAIL', 'actual': 'PASS', 'bug': 'crbug.com/test'},
'two': {'expected': 'FAIL', 'actual': 'TIMEOUT', 'bug': 'crbug.com/test'},
'three': {'expected': 'FAIL', 'actual': 'PASS', 'bug': 'crbug.com/test'},
}
}
tests_to_rebaseline, _ = updater.get_tests_to_rebaseline(two)
self.assertEqual(tests_to_rebaseline, [])
def test_specifiers_can_extend_to_all_platforms(self):
host = self.mock_host()
expectations_path = '/test.checkout/LayoutTests/NeverFixTests'
host.filesystem.write_text_file(
expectations_path,
'crbug.com/111 [ Linux ] external/wpt/test.html [ WontFix ]\n')
host.filesystem.write_text_file('/test.checkout/LayoutTests/external/wpt/test.html', '')
updater = WPTExpectationsUpdater(host)
self.assertTrue(updater.specifiers_can_extend_to_all_platforms(
['Mac10.10', 'Mac10.11', 'Win7', 'Win10'], 'external/wpt/test.html'))
self.assertFalse(updater.specifiers_can_extend_to_all_platforms(
['Mac10.10', 'Win7', 'Win10'], 'external/wpt/test.html'))
def test_simplify_specifiers(self):
macros = {
'mac': ['Mac10.10', 'mac10.11'],
'win': ['Win7', 'win10'],
'Linux': ['Trusty'],
}
self.assertEqual(WPTExpectationsUpdater.simplify_specifiers(['mac10.10', 'mac10.11'], macros), ['Mac'])
self.assertEqual(WPTExpectationsUpdater.simplify_specifiers(['Mac10.10', 'Mac10.11', 'Trusty'], macros), ['Linux', 'Mac'])
self.assertEqual(
WPTExpectationsUpdater.simplify_specifiers(['Mac10.10', 'Mac10.11', 'Trusty', 'Win7', 'Win10'], macros), [])
self.assertEqual(WPTExpectationsUpdater.simplify_specifiers(['a', 'b', 'c'], {}), ['A', 'B', 'C'])
self.assertEqual(WPTExpectationsUpdater.simplify_specifiers(['Mac', 'Win', 'Linux'], macros), [])
def test_write_to_test_expectations_with_marker_and_no_lines(self):
host = self.mock_host()
expectations_path = host.port_factory.get().path_to_generic_test_expectations_file()
host.filesystem.write_text_file(
expectations_path,
MARKER_COMMENT + '\n' + 'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]\n')
updater = WPTExpectationsUpdater(host)
updater.write_to_test_expectations([])
value = host.filesystem.read_text_file(expectations_path)
self.assertMultiLineEqual(
value,
MARKER_COMMENT + '\n' + 'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]\n')
def test_simplify_specifiers_uses_specifiers_in_builder_list(self):
# Even if there are extra specifiers in the macro dictionary, we can simplify specifier
# lists if they contain all of the specifiers the are represented in the builder list.
# This way specifier simplification can still be done while a new platform is being added.
host = self.mock_host()
updater = WPTExpectationsUpdater(host)
macros = {
'mac': ['Mac10.10', 'mac10.11', 'mac10.14'],
'win': ['Win7', 'win10'],
'Linux': ['Trusty'],
}
self.assertEqual(updater.simplify_specifiers(['mac10.10', 'mac10.11'], macros), ['Mac'])
self.assertEqual(updater.simplify_specifiers(['Mac10.10', 'Mac10.11', 'Trusty', 'Win7', 'Win10'], macros), [])
def test_write_to_test_expectations_with_marker_comment(self):
host = self.mock_host()
expectations_path = host.port_factory.get(
).path_to_generic_test_expectations_file()
host.filesystem.files[expectations_path] = MARKER_COMMENT + '\n'
updater = WPTExpectationsUpdater(host)
line_list = ['crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]']
updater.write_to_test_expectations(line_list)
value = updater.host.filesystem.read_text_file(expectations_path)
self.assertMultiLineEqual(
value, (MARKER_COMMENT + '\n'
'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]\n'))
def test_specifier_part_with_skipped_test(self):
host = self.mock_host()
expectations_path = '/test.checkout/LayoutTests/NeverFixTests'
host.filesystem.write_text_file(
expectations_path,
'crbug.com/111 [ Linux Mac10.11 ] external/wpt/test.html [ WontFix ]\n')
host.filesystem.write_text_file('/test.checkout/LayoutTests/external/wpt/test.html', '')
updater = WPTExpectationsUpdater(host)
self.assertEqual(
updater.specifier_part(['test-mac-mac10.10', 'test-win-win7', 'test-win-win10'], 'external/wpt/test.html'), '')
self.assertEqual(
updater.specifier_part(['test-win-win7', 'test-win-win10'], 'external/wpt/test.html'), '[ Win ]')
self.assertEqual(
updater.specifier_part(['test-win-win7', 'test-win-win10'], 'external/wpt/another.html'), '[ Win ]')
def test_get_test_to_rebaseline_returns_only_tests_with_failures(self):
host = self.mock_host()
updater = WPTExpectationsUpdater(host)
two = {
'external/wpt/test/path.html': {
'one': {'expected': 'FAIL', 'actual': 'PASS'},
'two': {'expected': 'FAIL', 'actual': 'TIMEOUT'},
'three': {'expected': 'FAIL', 'actual': 'PASS'},
}
}
tests_to_rebaseline, _ = updater.get_tests_to_rebaseline(two)
# external/wpt/test/zzzz.html is another possible candidate, but it
# is not listed in the results dict, so it shall not be rebaselined.
self.assertEqual(tests_to_rebaseline, ['external/wpt/test/path.html'])