def assertMergeResults(self,
before,
after,
mock_filesystem_contents,
inputargs,
filesystem_contains,
json_data_merger=None):
mock_filesystem = MockFileSystem(mock_filesystem_contents,
dirs=['/output'])
file_merger = merge_results.MergeFilesJSONP(mock_filesystem,
json_data_merger)
file_merger(*inputargs)
files = mock_filesystem.files_under('/output')
self.assertTrue(len(files) == 1)
expected_mock_filesystem = MockFileSystem(filesystem_contains)
expected_files = expected_mock_filesystem.files_under('/output')
actual_output = mock_filesystem.read_text_file(files[0])
expected_output = expected_mock_filesystem.read_text_file(
expected_files[0])
self.assertTrue(self.check_before_after(actual_output, before, after))
self.assertTrue(self.check_before_after(expected_output, before,
after))
actual_json_str = self.remove_before_after(actual_output, before,
after)
expected_json_str = self.remove_before_after(expected_output, before,
after)
self.assertEqual(json.loads(actual_json_str),
json.loads(expected_json_str))
def test_find_log_darwin(self):
if not SystemHost().platform.is_mac():
return
older_mock_crash_report = make_mock_crash_report_darwin(
'DumpRenderTree', 28528)
mock_crash_report = make_mock_crash_report_darwin(
'DumpRenderTree', 28530)
newer_mock_crash_report = make_mock_crash_report_darwin(
'DumpRenderTree', 28529)
other_process_mock_crash_report = make_mock_crash_report_darwin(
'FooProcess', 28527)
misformatted_mock_crash_report = 'Junk that should not appear in a crash report' + \
make_mock_crash_report_darwin('DumpRenderTree', 28526)[200:]
files = {
'/Users/mock/Library/Logs/DiagnosticReports/DumpRenderTree_2011-06-13-150718_quadzen.crash':
older_mock_crash_report,
'/Users/mock/Library/Logs/DiagnosticReports/DumpRenderTree_2011-06-13-150719_quadzen.crash':
mock_crash_report,
'/Users/mock/Library/Logs/DiagnosticReports/DumpRenderTree_2011-06-13-150720_quadzen.crash':
newer_mock_crash_report,
'/Users/mock/Library/Logs/DiagnosticReports/DumpRenderTree_2011-06-13-150721_quadzen.crash':
None,
'/Users/mock/Library/Logs/DiagnosticReports/DumpRenderTree_2011-06-13-150722_quadzen.crash':
other_process_mock_crash_report,
'/Users/mock/Library/Logs/DiagnosticReports/DumpRenderTree_2011-06-13-150723_quadzen.crash':
misformatted_mock_crash_report,
}
filesystem = MockFileSystem(files)
crash_logs = CrashLogs(MockSystemHost(filesystem=filesystem))
log = crash_logs.find_newest_log('DumpRenderTree')
self.assertMultiLineEqual(log, newer_mock_crash_report)
log = crash_logs.find_newest_log('DumpRenderTree', 28529)
self.assertMultiLineEqual(log, newer_mock_crash_report)
log = crash_logs.find_newest_log('DumpRenderTree', 28530)
self.assertMultiLineEqual(log, mock_crash_report)
log = crash_logs.find_newest_log('DumpRenderTree', 28531)
self.assertIsNone(log)
log = crash_logs.find_newest_log('DumpRenderTree', newer_than=1.0)
self.assertIsNone(log)
def bad_read(_):
raise IOError('IOError: No such file or directory')
def bad_mtime(_):
raise OSError('OSError: No such file or directory')
filesystem.read_text_file = bad_read
log = crash_logs.find_newest_log('DumpRenderTree',
28531,
include_errors=True)
self.assertIn('IOError: No such file or directory', log)
filesystem = MockFileSystem(files)
crash_logs = CrashLogs(MockSystemHost(filesystem=filesystem))
filesystem.mtime = bad_mtime
log = crash_logs.find_newest_log('DumpRenderTree',
newer_than=1.0,
include_errors=True)
self.assertIn('OSError: No such file or directory', log)
class BaselineOptimizerTest(unittest.TestCase):
def setUp(self):
self.host = MockHost()
self.fs = MockFileSystem()
self.host.filesystem = self.fs
# TODO(robertma): Even though we have mocked the builder list (and hence
# all_port_names), we are still relying on the knowledge of currently
# configured ports and their fallback order. Ideally, we should improve
# MockPortFactory and use it.
self.host.builders = BuilderList({
'Fake Test Win10.20h2': {
'port_name': 'win-win10.20h2',
'specifiers': ['Win10.20h2', 'Release']
},
'Fake Test Linux': {
'port_name': 'linux-trusty',
'specifiers': ['Trusty', 'Release']
},
'Fake Test Mac11.0': {
'port_name': 'mac-mac11',
'specifiers': ['Mac11', 'Release']
},
'Fake Test Mac10.15': {
'port_name': 'mac-mac10.15',
'specifiers': ['Mac10.15', 'Release']
},
'Fake Test Mac10.14': {
'port_name': 'mac-mac10.14',
'specifiers': ['Mac10.14', 'Release']
},
'Fake Test Mac10.13': {
'port_name': 'mac-mac10.13',
'specifiers': ['Mac10.13', 'Release']
},
'Fake Test Mac10.12': {
'port_name': 'mac-mac10.12',
'specifiers': ['Mac10.12', 'Release']
},
})
# Note: this is a pre-assumption of the tests in this file. If this
# assertion fails, port configurations are likely changed, and the
# tests need to be adjusted accordingly.
self.assertEqual(sorted(self.host.port_factory.all_port_names()), [
'linux-trusty', 'mac-mac10.12', 'mac-mac10.13', 'mac-mac10.14',
'mac-mac10.15', 'mac-mac11', 'win-win10.20h2'
])
def _assert_optimization(self,
results_by_directory,
directory_to_new_results,
baseline_dirname='',
suffix='txt'):
web_tests_dir = PathFinder(self.fs).web_tests_dir()
test_name = 'mock-test.html'
baseline_name = 'mock-test-expected.' + suffix
self.fs.write_text_file(
self.fs.join(web_tests_dir, 'VirtualTestSuites'),
'[{"prefix": "gpu", "bases": ["fast/canvas"], "args": ["--foo"]}]')
for dirname, contents in results_by_directory.items():
self.fs.write_text_file(
self.fs.join(web_tests_dir, dirname, baseline_name), contents)
baseline_optimizer = BaselineOptimizer(
self.host, self.host.port_factory.get(),
self.host.port_factory.all_port_names())
self.assertTrue(
baseline_optimizer.optimize(
self.fs.join(baseline_dirname, test_name), suffix))
for dirname, contents in directory_to_new_results.items():
path = self.fs.join(web_tests_dir, dirname, baseline_name)
if contents is None:
# Check files that are explicitly marked as absent.
self.assertFalse(
self.fs.exists(path),
'%s should not exist after optimization' % path)
else:
self.assertEqual(self.fs.read_text_file(path), contents,
'Content of %s != "%s"' % (path, contents))
for dirname in results_by_directory:
path = self.fs.join(web_tests_dir, dirname, baseline_name)
if (dirname not in directory_to_new_results
or directory_to_new_results[dirname] is None):
self.assertFalse(
self.fs.exists(path),
'%s should not exist after optimization' % path)
def _assert_reftest_optimization(self,
results_by_directory,
directory_to_new_results,
test_path='',
baseline_dirname=''):
web_tests_dir = PathFinder(self.fs).web_tests_dir()
self.fs.write_text_file(
self.fs.join(web_tests_dir, test_path, 'mock-test-expected.html'),
'ref')
self._assert_optimization(
results_by_directory,
directory_to_new_results,
baseline_dirname,
suffix='png')
def test_linux_redundant_with_win(self):
self._assert_optimization({
'platform/win': '1',
'platform/linux': '1',
}, {
'platform/win': '1',
})
def test_covers_mac_win_linux(self):
self._assert_optimization({
'platform/mac': '1',
'platform/win': '1',
'platform/linux': '1',
}, {
'': '1',
})
def test_overwrites_root(self):
self._assert_optimization({
'platform/mac': '1',
'platform/win': '1',
'platform/linux': '1',
'': '2',
}, {
'': '1',
})
def test_no_new_common_directory(self):
self._assert_optimization({
'platform/mac': '1',
'platform/linux': '1',
'': '2',
}, {
'platform/mac': '1',
'platform/linux': '1',
'': '2',
})
def test_local_optimization(self):
self._assert_optimization({
'platform/mac': '1',
'platform/linux': '1',
'platform/mac-mac10.14': '1',
}, {
'platform/mac': '1',
'platform/linux': '1',
})
def test_local_optimization_skipping_a_port_in_the_middle(self):
# mac-mac10.13 -> mac-mac10.14 -> mac
self._assert_optimization({
'platform/mac': '1',
'platform/linux': '1',
'platform/mac-mac10.13': '1',
}, {
'platform/mac': '1',
'platform/linux': '1',
})
def test_baseline_redundant_with_root(self):
self._assert_optimization({
'platform/mac': '1',
'platform/win': '2',
'': '2',
}, {
'platform/mac': '1',
'': '2',
})
def test_root_baseline_unused(self):
self._assert_optimization({
'platform/mac': '1',
'platform/win': '2',
'': '3',
}, {
'platform/mac': '1',
'platform/win': '2',
})
def test_root_baseline_unused_and_non_existant(self):
self._assert_optimization({
'platform/mac': '1',
'platform/win': '2',
}, {
'platform/mac': '1',
'platform/win': '2',
})
def test_virtual_baseline_redundant_with_non_virtual(self):
self._assert_optimization({
'platform/win/virtual/gpu/fast/canvas': '2',
'platform/win/fast/canvas': '2',
}, {
'platform/win/fast/canvas': '2',
},
baseline_dirname='virtual/gpu/fast/canvas')
def test_virtual_baseline_redundant_with_non_virtual_fallback(self):
# virtual linux -> virtual win -> virtual root -> linux -> win
self._assert_optimization(
{
'platform/linux/virtual/gpu/fast/canvas': '2',
'platform/win/fast/canvas': '2',
}, {
'platform/win/virtual/gpu/fast/canvas': None,
'platform/win/fast/canvas': '2',
},
baseline_dirname='virtual/gpu/fast/canvas')
def test_virtual_baseline_redundant_with_actual_root(self):
self._assert_optimization({
'platform/win/virtual/gpu/fast/canvas': '2',
'fast/canvas': '2',
}, {
'fast/canvas': '2',
},
baseline_dirname='virtual/gpu/fast/canvas')
def test_virtual_root_redundant_with_actual_root(self):
self._assert_optimization({
'virtual/gpu/fast/canvas': '2',
'fast/canvas': '2',
}, {
'fast/canvas': '2',
},
baseline_dirname='virtual/gpu/fast/canvas')
def test_virtual_root_redundant_with_ancestors(self):
self._assert_optimization({
'virtual/gpu/fast/canvas': '2',
'platform/mac/fast/canvas': '2',
'platform/win/fast/canvas': '2',
}, {
'fast/canvas': '2',
},
baseline_dirname='virtual/gpu/fast/canvas')
def test_virtual_root_not_redundant_with_ancestors(self):
self._assert_optimization({
'virtual/gpu/fast/canvas': '2',
'platform/mac/fast/canvas': '1',
}, {
'virtual/gpu/fast/canvas': '2',
'platform/mac/fast/canvas': '1',
},
baseline_dirname='virtual/gpu/fast/canvas')
def test_virtual_covers_mac_win_linux(self):
self._assert_optimization(
{
'platform/mac/virtual/gpu/fast/canvas': '1',
'platform/win/virtual/gpu/fast/canvas': '1',
'platform/linux/virtual/gpu/fast/canvas': '1',
}, {
'virtual/gpu/fast/canvas': '1',
},
baseline_dirname='virtual/gpu/fast/canvas')
def test_all_pass_testharness_at_root(self):
self._assert_optimization({
'': ALL_PASS_TESTHARNESS_RESULT
}, {'': None})
def test_all_pass_testharness_at_linux(self):
self._assert_optimization({
'platform/linux': ALL_PASS_TESTHARNESS_RESULT
}, {'platform/linux': None})
def test_all_pass_testharness_at_linux_and_win(self):
# https://crbug.com/805008
self._assert_optimization(
{
'platform/linux': ALL_PASS_TESTHARNESS_RESULT,
'platform/win': ALL_PASS_TESTHARNESS_RESULT
}, {
'platform/linux': None,
'platform/win': None
})
def test_all_pass_testharness_at_virtual_root(self):
self._assert_optimization(
{
'virtual/gpu/fast/canvas': ALL_PASS_TESTHARNESS_RESULT
}, {'virtual/gpu/fast/canvas': None},
baseline_dirname='virtual/gpu/fast/canvas')
def test_all_pass_testharness_at_virtual_linux(self):
self._assert_optimization(
{
'platform/linux/virtual/gpu/fast/canvas':
ALL_PASS_TESTHARNESS_RESULT
}, {'platform/linux/virtual/gpu/fast/canvas': None},
baseline_dirname='virtual/gpu/fast/canvas')
def test_all_pass_testharness_can_be_updated(self):
# https://crbug.com/866802
self._assert_optimization(
{
'fast/canvas':
'failure',
'virtual/gpu/fast/canvas':
ALL_PASS_TESTHARNESS_RESULT,
'platform/win/virtual/gpu/fast/canvas':
ALL_PASS_TESTHARNESS_RESULT2,
'platform/mac/virtual/gpu/fast/canvas':
ALL_PASS_TESTHARNESS_RESULT2,
}, {
'fast/canvas': 'failure',
'virtual/gpu/fast/canvas': ALL_PASS_TESTHARNESS_RESULT2,
'platform/win/virtual/gpu/fast/canvas': None,
'platform/mac/virtual/gpu/fast/canvas': None,
},
baseline_dirname='virtual/gpu/fast/canvas')
def test_all_pass_testharness_falls_back_to_non_pass(self):
# The all-PASS baseline needs to be preserved in this case.
self._assert_optimization(
{
'platform/linux': ALL_PASS_TESTHARNESS_RESULT,
'': '1'
}, {
'platform/linux': ALL_PASS_TESTHARNESS_RESULT,
'': '1'
})
def test_virtual_all_pass_testharness_falls_back_to_base(self):
# The all-PASS baseline needs to be preserved in this case.
self._assert_optimization(
{
'virtual/gpu/fast/canvas': ALL_PASS_TESTHARNESS_RESULT,
'platform/linux/fast/canvas': '1',
}, {
'virtual/gpu/fast/canvas': ALL_PASS_TESTHARNESS_RESULT,
'platform/linux/fast/canvas': '1',
},
baseline_dirname='virtual/gpu/fast/canvas')
def test_empty_at_root(self):
self._assert_optimization({'': ''}, {'': None})
def test_empty_at_linux(self):
self._assert_optimization({
'platform/linux': ''
}, {'platform/linux': None})
def test_empty_at_linux_and_win(self):
# https://crbug.com/805008
self._assert_optimization({
'platform/linux': '',
'platform/win': '',
}, {
'platform/linux': None,
'platform/win': None,
})
def test_empty_at_virtual_root(self):
self._assert_optimization({
'virtual/gpu/fast/canvas': ''
}, {'virtual/gpu/fast/canvas': None},
baseline_dirname='virtual/gpu/fast/canvas')
def test_empty_at_virtual_linux(self):
self._assert_optimization(
{
'platform/linux/virtual/gpu/fast/canvas': ''
}, {'platform/linux/virtual/gpu/fast/canvas': None},
baseline_dirname='virtual/gpu/fast/canvas')
def test_empty_falls_back_to_non_empty(self):
# The empty baseline needs to be preserved in this case.
self._assert_optimization({
'platform/linux': '',
'': '1',
}, {
'platform/linux': '',
'': '1',
})
def test_virtual_empty_falls_back_to_non_empty(self):
# The empty baseline needs to be preserved in this case.
self._assert_optimization({
'virtual/gpu/fast/canvas': '',
'platform/linux/fast/canvas': '1',
}, {
'virtual/gpu/fast/canvas': '',
'platform/linux/fast/canvas': '1',
},
baseline_dirname='virtual/gpu/fast/canvas')
def test_extra_png_for_reftest_at_root(self):
self._assert_reftest_optimization({'': 'extra'}, {'': None})
def test_extra_png_for_reftest_at_linux(self):
self._assert_reftest_optimization({
'platform/linux': 'extra'
}, {'platform/linux': None})
def test_extra_png_for_reftest_at_linux_and_win(self):
# https://crbug.com/805008
self._assert_reftest_optimization({
'platform/linux': 'extra1',
'platform/win': 'extra2',
}, {
'platform/linux': None,
'platform/win': None,
})
def test_extra_png_for_reftest_at_virtual_root(self):
self._assert_reftest_optimization(
{
'virtual/gpu/fast/canvas': 'extra'
}, {'virtual/gpu/fast/canvas': None},
test_path='fast/canvas',
baseline_dirname='virtual/gpu/fast/canvas')
def test_extra_png_for_reftest_at_virtual_linux(self):
self._assert_reftest_optimization(
{
'platform/linux/virtual/gpu/fast/canvas': 'extra'
}, {'platform/linux/virtual/gpu/fast/canvas': None},
test_path='fast/canvas',
baseline_dirname='virtual/gpu/fast/canvas')
def test_extra_png_for_reftest_falls_back_to_base(self):
# The extra png for reftest should be removed even if it's different
# from the fallback.
self._assert_reftest_optimization({
'platform/linux': 'extra1',
'': 'extra2',
}, {
'platform/linux': None,
'': None,
})
def test_virtual_extra_png_for_reftest_falls_back_to_base(self):
# The extra png for reftest should be removed even if it's different
# from the fallback.
self._assert_reftest_optimization(
{
'virtual/gpu/fast/canvas': 'extra',
'platform/linux/fast/canvas': 'extra2',
}, {
'virtual/gpu/fast/canvas': None,
'platform/linux/fast/canvas': None,
},
test_path='fast/canvas',
baseline_dirname='virtual/gpu/fast/canvas')
# Tests for protected methods - pylint: disable=protected-access
def test_move_baselines(self):
self.fs.write_text_file(MOCK_WEB_TESTS + 'VirtualTestSuites', '[]')
self.fs.write_binary_file(
MOCK_WEB_TESTS + 'platform/win/another/test-expected.txt',
'result A')
self.fs.write_binary_file(
MOCK_WEB_TESTS + 'platform/mac/another/test-expected.txt',
'result A')
self.fs.write_binary_file(MOCK_WEB_TESTS + 'another/test-expected.txt',
'result B')
baseline_optimizer = BaselineOptimizer(
self.host, self.host.port_factory.get(),
self.host.port_factory.all_port_names())
baseline_optimizer._move_baselines(
'another/test-expected.txt', {
MOCK_WEB_TESTS + 'platform/win': 'aaa',
MOCK_WEB_TESTS + 'platform/mac': 'aaa',
MOCK_WEB_TESTS[:-1]: 'bbb',
}, {
MOCK_WEB_TESTS[:-1]: 'aaa',
})
self.assertEqual(
self.fs.read_binary_file(MOCK_WEB_TESTS +
'another/test-expected.txt'), 'result A')
def test_move_baselines_skip_git_commands(self):
self.fs.write_text_file(MOCK_WEB_TESTS + 'VirtualTestSuites', '[]')
self.fs.write_binary_file(
MOCK_WEB_TESTS + 'platform/win/another/test-expected.txt',
'result A')
self.fs.write_binary_file(
MOCK_WEB_TESTS + 'platform/mac/another/test-expected.txt',
'result A')
self.fs.write_binary_file(MOCK_WEB_TESTS + 'another/test-expected.txt',
'result B')
baseline_optimizer = BaselineOptimizer(
self.host, self.host.port_factory.get(),
self.host.port_factory.all_port_names())
baseline_optimizer._move_baselines(
'another/test-expected.txt', {
MOCK_WEB_TESTS + 'platform/win': 'aaa',
MOCK_WEB_TESTS + 'platform/mac': 'aaa',
MOCK_WEB_TESTS[:-1]: 'bbb',
}, {
MOCK_WEB_TESTS + 'platform/linux': 'bbb',
MOCK_WEB_TESTS[:-1]: 'aaa',
})
self.assertEqual(
self.fs.read_binary_file(MOCK_WEB_TESTS +
'another/test-expected.txt'), 'result A')
