def assertMergeResults(self, before, after, mock_filesystem_contents, inputargs, filesystem_contains, json_data_merger=None): mock_filesystem = MockFileSystem(mock_filesystem_contents, dirs=['/output']) file_merger = merge_results.MergeFilesJSONP(mock_filesystem, json_data_merger) file_merger(*inputargs) files = mock_filesystem.files_under('/output') self.assertTrue(len(files) == 1) expected_mock_filesystem = MockFileSystem(filesystem_contains) expected_files = expected_mock_filesystem.files_under('/output') actual_output = mock_filesystem.read_text_file(files[0]) expected_output = expected_mock_filesystem.read_text_file( expected_files[0]) self.assertTrue(self.check_before_after(actual_output, before, after)) self.assertTrue(self.check_before_after(expected_output, before, after)) actual_json_str = self.remove_before_after(actual_output, before, after) expected_json_str = self.remove_before_after(expected_output, before, after) self.assertEqual(json.loads(actual_json_str), json.loads(expected_json_str))
def test_find_log_darwin(self): if not SystemHost().platform.is_mac(): return older_mock_crash_report = make_mock_crash_report_darwin( 'DumpRenderTree', 28528) mock_crash_report = make_mock_crash_report_darwin( 'DumpRenderTree', 28530) newer_mock_crash_report = make_mock_crash_report_darwin( 'DumpRenderTree', 28529) other_process_mock_crash_report = make_mock_crash_report_darwin( 'FooProcess', 28527) misformatted_mock_crash_report = 'Junk that should not appear in a crash report' + \ make_mock_crash_report_darwin('DumpRenderTree', 28526)[200:] files = { '/Users/mock/Library/Logs/DiagnosticReports/DumpRenderTree_2011-06-13-150718_quadzen.crash': older_mock_crash_report, '/Users/mock/Library/Logs/DiagnosticReports/DumpRenderTree_2011-06-13-150719_quadzen.crash': mock_crash_report, '/Users/mock/Library/Logs/DiagnosticReports/DumpRenderTree_2011-06-13-150720_quadzen.crash': newer_mock_crash_report, '/Users/mock/Library/Logs/DiagnosticReports/DumpRenderTree_2011-06-13-150721_quadzen.crash': None, '/Users/mock/Library/Logs/DiagnosticReports/DumpRenderTree_2011-06-13-150722_quadzen.crash': other_process_mock_crash_report, '/Users/mock/Library/Logs/DiagnosticReports/DumpRenderTree_2011-06-13-150723_quadzen.crash': misformatted_mock_crash_report, } filesystem = MockFileSystem(files) crash_logs = CrashLogs(MockSystemHost(filesystem=filesystem)) log = crash_logs.find_newest_log('DumpRenderTree') self.assertMultiLineEqual(log, newer_mock_crash_report) log = crash_logs.find_newest_log('DumpRenderTree', 28529) self.assertMultiLineEqual(log, newer_mock_crash_report) log = crash_logs.find_newest_log('DumpRenderTree', 28530) self.assertMultiLineEqual(log, mock_crash_report) log = crash_logs.find_newest_log('DumpRenderTree', 28531) self.assertIsNone(log) log = crash_logs.find_newest_log('DumpRenderTree', newer_than=1.0) self.assertIsNone(log) def bad_read(_): raise IOError('IOError: No such file or directory') def bad_mtime(_): raise OSError('OSError: No such file or directory') filesystem.read_text_file = bad_read log = crash_logs.find_newest_log('DumpRenderTree', 28531, include_errors=True) self.assertIn('IOError: No such file or directory', log) filesystem = MockFileSystem(files) crash_logs = CrashLogs(MockSystemHost(filesystem=filesystem)) filesystem.mtime = bad_mtime log = crash_logs.find_newest_log('DumpRenderTree', newer_than=1.0, include_errors=True) self.assertIn('OSError: No such file or directory', log)
class BaselineOptimizerTest(unittest.TestCase): def setUp(self): self.host = MockHost() self.fs = MockFileSystem() self.host.filesystem = self.fs # TODO(robertma): Even though we have mocked the builder list (and hence # all_port_names), we are still relying on the knowledge of currently # configured ports and their fallback order. Ideally, we should improve # MockPortFactory and use it. self.host.builders = BuilderList({ 'Fake Test Win10.20h2': { 'port_name': 'win-win10.20h2', 'specifiers': ['Win10.20h2', 'Release'] }, 'Fake Test Linux': { 'port_name': 'linux-trusty', 'specifiers': ['Trusty', 'Release'] }, 'Fake Test Mac11.0': { 'port_name': 'mac-mac11', 'specifiers': ['Mac11', 'Release'] }, 'Fake Test Mac10.15': { 'port_name': 'mac-mac10.15', 'specifiers': ['Mac10.15', 'Release'] }, 'Fake Test Mac10.14': { 'port_name': 'mac-mac10.14', 'specifiers': ['Mac10.14', 'Release'] }, 'Fake Test Mac10.13': { 'port_name': 'mac-mac10.13', 'specifiers': ['Mac10.13', 'Release'] }, 'Fake Test Mac10.12': { 'port_name': 'mac-mac10.12', 'specifiers': ['Mac10.12', 'Release'] }, }) # Note: this is a pre-assumption of the tests in this file. If this # assertion fails, port configurations are likely changed, and the # tests need to be adjusted accordingly. self.assertEqual(sorted(self.host.port_factory.all_port_names()), [ 'linux-trusty', 'mac-mac10.12', 'mac-mac10.13', 'mac-mac10.14', 'mac-mac10.15', 'mac-mac11', 'win-win10.20h2' ]) def _assert_optimization(self, results_by_directory, directory_to_new_results, baseline_dirname='', suffix='txt'): web_tests_dir = PathFinder(self.fs).web_tests_dir() test_name = 'mock-test.html' baseline_name = 'mock-test-expected.' + suffix self.fs.write_text_file( self.fs.join(web_tests_dir, 'VirtualTestSuites'), '[{"prefix": "gpu", "bases": ["fast/canvas"], "args": ["--foo"]}]') for dirname, contents in results_by_directory.items(): self.fs.write_text_file( self.fs.join(web_tests_dir, dirname, baseline_name), contents) baseline_optimizer = BaselineOptimizer( self.host, self.host.port_factory.get(), self.host.port_factory.all_port_names()) self.assertTrue( baseline_optimizer.optimize( self.fs.join(baseline_dirname, test_name), suffix)) for dirname, contents in directory_to_new_results.items(): path = self.fs.join(web_tests_dir, dirname, baseline_name) if contents is None: # Check files that are explicitly marked as absent. self.assertFalse( self.fs.exists(path), '%s should not exist after optimization' % path) else: self.assertEqual(self.fs.read_text_file(path), contents, 'Content of %s != "%s"' % (path, contents)) for dirname in results_by_directory: path = self.fs.join(web_tests_dir, dirname, baseline_name) if (dirname not in directory_to_new_results or directory_to_new_results[dirname] is None): self.assertFalse( self.fs.exists(path), '%s should not exist after optimization' % path) def _assert_reftest_optimization(self, results_by_directory, directory_to_new_results, test_path='', baseline_dirname=''): web_tests_dir = PathFinder(self.fs).web_tests_dir() self.fs.write_text_file( self.fs.join(web_tests_dir, test_path, 'mock-test-expected.html'), 'ref') self._assert_optimization( results_by_directory, directory_to_new_results, baseline_dirname, suffix='png') def test_linux_redundant_with_win(self): self._assert_optimization({ 'platform/win': '1', 'platform/linux': '1', }, { 'platform/win': '1', }) def test_covers_mac_win_linux(self): self._assert_optimization({ 'platform/mac': '1', 'platform/win': '1', 'platform/linux': '1', }, { '': '1', }) def test_overwrites_root(self): self._assert_optimization({ 'platform/mac': '1', 'platform/win': '1', 'platform/linux': '1', '': '2', }, { '': '1', }) def test_no_new_common_directory(self): self._assert_optimization({ 'platform/mac': '1', 'platform/linux': '1', '': '2', }, { 'platform/mac': '1', 'platform/linux': '1', '': '2', }) def test_local_optimization(self): self._assert_optimization({ 'platform/mac': '1', 'platform/linux': '1', 'platform/mac-mac10.14': '1', }, { 'platform/mac': '1', 'platform/linux': '1', }) def test_local_optimization_skipping_a_port_in_the_middle(self): # mac-mac10.13 -> mac-mac10.14 -> mac self._assert_optimization({ 'platform/mac': '1', 'platform/linux': '1', 'platform/mac-mac10.13': '1', }, { 'platform/mac': '1', 'platform/linux': '1', }) def test_baseline_redundant_with_root(self): self._assert_optimization({ 'platform/mac': '1', 'platform/win': '2', '': '2', }, { 'platform/mac': '1', '': '2', }) def test_root_baseline_unused(self): self._assert_optimization({ 'platform/mac': '1', 'platform/win': '2', '': '3', }, { 'platform/mac': '1', 'platform/win': '2', }) def test_root_baseline_unused_and_non_existant(self): self._assert_optimization({ 'platform/mac': '1', 'platform/win': '2', }, { 'platform/mac': '1', 'platform/win': '2', }) def test_virtual_baseline_redundant_with_non_virtual(self): self._assert_optimization({ 'platform/win/virtual/gpu/fast/canvas': '2', 'platform/win/fast/canvas': '2', }, { 'platform/win/fast/canvas': '2', }, baseline_dirname='virtual/gpu/fast/canvas') def test_virtual_baseline_redundant_with_non_virtual_fallback(self): # virtual linux -> virtual win -> virtual root -> linux -> win self._assert_optimization( { 'platform/linux/virtual/gpu/fast/canvas': '2', 'platform/win/fast/canvas': '2', }, { 'platform/win/virtual/gpu/fast/canvas': None, 'platform/win/fast/canvas': '2', }, baseline_dirname='virtual/gpu/fast/canvas') def test_virtual_baseline_redundant_with_actual_root(self): self._assert_optimization({ 'platform/win/virtual/gpu/fast/canvas': '2', 'fast/canvas': '2', }, { 'fast/canvas': '2', }, baseline_dirname='virtual/gpu/fast/canvas') def test_virtual_root_redundant_with_actual_root(self): self._assert_optimization({ 'virtual/gpu/fast/canvas': '2', 'fast/canvas': '2', }, { 'fast/canvas': '2', }, baseline_dirname='virtual/gpu/fast/canvas') def test_virtual_root_redundant_with_ancestors(self): self._assert_optimization({ 'virtual/gpu/fast/canvas': '2', 'platform/mac/fast/canvas': '2', 'platform/win/fast/canvas': '2', }, { 'fast/canvas': '2', }, baseline_dirname='virtual/gpu/fast/canvas') def test_virtual_root_not_redundant_with_ancestors(self): self._assert_optimization({ 'virtual/gpu/fast/canvas': '2', 'platform/mac/fast/canvas': '1', }, { 'virtual/gpu/fast/canvas': '2', 'platform/mac/fast/canvas': '1', }, baseline_dirname='virtual/gpu/fast/canvas') def test_virtual_covers_mac_win_linux(self): self._assert_optimization( { 'platform/mac/virtual/gpu/fast/canvas': '1', 'platform/win/virtual/gpu/fast/canvas': '1', 'platform/linux/virtual/gpu/fast/canvas': '1', }, { 'virtual/gpu/fast/canvas': '1', }, baseline_dirname='virtual/gpu/fast/canvas') def test_all_pass_testharness_at_root(self): self._assert_optimization({ '': ALL_PASS_TESTHARNESS_RESULT }, {'': None}) def test_all_pass_testharness_at_linux(self): self._assert_optimization({ 'platform/linux': ALL_PASS_TESTHARNESS_RESULT }, {'platform/linux': None}) def test_all_pass_testharness_at_linux_and_win(self): # https://crbug.com/805008 self._assert_optimization( { 'platform/linux': ALL_PASS_TESTHARNESS_RESULT, 'platform/win': ALL_PASS_TESTHARNESS_RESULT }, { 'platform/linux': None, 'platform/win': None }) def test_all_pass_testharness_at_virtual_root(self): self._assert_optimization( { 'virtual/gpu/fast/canvas': ALL_PASS_TESTHARNESS_RESULT }, {'virtual/gpu/fast/canvas': None}, baseline_dirname='virtual/gpu/fast/canvas') def test_all_pass_testharness_at_virtual_linux(self): self._assert_optimization( { 'platform/linux/virtual/gpu/fast/canvas': ALL_PASS_TESTHARNESS_RESULT }, {'platform/linux/virtual/gpu/fast/canvas': None}, baseline_dirname='virtual/gpu/fast/canvas') def test_all_pass_testharness_can_be_updated(self): # https://crbug.com/866802 self._assert_optimization( { 'fast/canvas': 'failure', 'virtual/gpu/fast/canvas': ALL_PASS_TESTHARNESS_RESULT, 'platform/win/virtual/gpu/fast/canvas': ALL_PASS_TESTHARNESS_RESULT2, 'platform/mac/virtual/gpu/fast/canvas': ALL_PASS_TESTHARNESS_RESULT2, }, { 'fast/canvas': 'failure', 'virtual/gpu/fast/canvas': ALL_PASS_TESTHARNESS_RESULT2, 'platform/win/virtual/gpu/fast/canvas': None, 'platform/mac/virtual/gpu/fast/canvas': None, }, baseline_dirname='virtual/gpu/fast/canvas') def test_all_pass_testharness_falls_back_to_non_pass(self): # The all-PASS baseline needs to be preserved in this case. self._assert_optimization( { 'platform/linux': ALL_PASS_TESTHARNESS_RESULT, '': '1' }, { 'platform/linux': ALL_PASS_TESTHARNESS_RESULT, '': '1' }) def test_virtual_all_pass_testharness_falls_back_to_base(self): # The all-PASS baseline needs to be preserved in this case. self._assert_optimization( { 'virtual/gpu/fast/canvas': ALL_PASS_TESTHARNESS_RESULT, 'platform/linux/fast/canvas': '1', }, { 'virtual/gpu/fast/canvas': ALL_PASS_TESTHARNESS_RESULT, 'platform/linux/fast/canvas': '1', }, baseline_dirname='virtual/gpu/fast/canvas') def test_empty_at_root(self): self._assert_optimization({'': ''}, {'': None}) def test_empty_at_linux(self): self._assert_optimization({ 'platform/linux': '' }, {'platform/linux': None}) def test_empty_at_linux_and_win(self): # https://crbug.com/805008 self._assert_optimization({ 'platform/linux': '', 'platform/win': '', }, { 'platform/linux': None, 'platform/win': None, }) def test_empty_at_virtual_root(self): self._assert_optimization({ 'virtual/gpu/fast/canvas': '' }, {'virtual/gpu/fast/canvas': None}, baseline_dirname='virtual/gpu/fast/canvas') def test_empty_at_virtual_linux(self): self._assert_optimization( { 'platform/linux/virtual/gpu/fast/canvas': '' }, {'platform/linux/virtual/gpu/fast/canvas': None}, baseline_dirname='virtual/gpu/fast/canvas') def test_empty_falls_back_to_non_empty(self): # The empty baseline needs to be preserved in this case. self._assert_optimization({ 'platform/linux': '', '': '1', }, { 'platform/linux': '', '': '1', }) def test_virtual_empty_falls_back_to_non_empty(self): # The empty baseline needs to be preserved in this case. self._assert_optimization({ 'virtual/gpu/fast/canvas': '', 'platform/linux/fast/canvas': '1', }, { 'virtual/gpu/fast/canvas': '', 'platform/linux/fast/canvas': '1', }, baseline_dirname='virtual/gpu/fast/canvas') def test_extra_png_for_reftest_at_root(self): self._assert_reftest_optimization({'': 'extra'}, {'': None}) def test_extra_png_for_reftest_at_linux(self): self._assert_reftest_optimization({ 'platform/linux': 'extra' }, {'platform/linux': None}) def test_extra_png_for_reftest_at_linux_and_win(self): # https://crbug.com/805008 self._assert_reftest_optimization({ 'platform/linux': 'extra1', 'platform/win': 'extra2', }, { 'platform/linux': None, 'platform/win': None, }) def test_extra_png_for_reftest_at_virtual_root(self): self._assert_reftest_optimization( { 'virtual/gpu/fast/canvas': 'extra' }, {'virtual/gpu/fast/canvas': None}, test_path='fast/canvas', baseline_dirname='virtual/gpu/fast/canvas') def test_extra_png_for_reftest_at_virtual_linux(self): self._assert_reftest_optimization( { 'platform/linux/virtual/gpu/fast/canvas': 'extra' }, {'platform/linux/virtual/gpu/fast/canvas': None}, test_path='fast/canvas', baseline_dirname='virtual/gpu/fast/canvas') def test_extra_png_for_reftest_falls_back_to_base(self): # The extra png for reftest should be removed even if it's different # from the fallback. self._assert_reftest_optimization({ 'platform/linux': 'extra1', '': 'extra2', }, { 'platform/linux': None, '': None, }) def test_virtual_extra_png_for_reftest_falls_back_to_base(self): # The extra png for reftest should be removed even if it's different # from the fallback. self._assert_reftest_optimization( { 'virtual/gpu/fast/canvas': 'extra', 'platform/linux/fast/canvas': 'extra2', }, { 'virtual/gpu/fast/canvas': None, 'platform/linux/fast/canvas': None, }, test_path='fast/canvas', baseline_dirname='virtual/gpu/fast/canvas') # Tests for protected methods - pylint: disable=protected-access def test_move_baselines(self): self.fs.write_text_file(MOCK_WEB_TESTS + 'VirtualTestSuites', '[]') self.fs.write_binary_file( MOCK_WEB_TESTS + 'platform/win/another/test-expected.txt', 'result A') self.fs.write_binary_file( MOCK_WEB_TESTS + 'platform/mac/another/test-expected.txt', 'result A') self.fs.write_binary_file(MOCK_WEB_TESTS + 'another/test-expected.txt', 'result B') baseline_optimizer = BaselineOptimizer( self.host, self.host.port_factory.get(), self.host.port_factory.all_port_names()) baseline_optimizer._move_baselines( 'another/test-expected.txt', { MOCK_WEB_TESTS + 'platform/win': 'aaa', MOCK_WEB_TESTS + 'platform/mac': 'aaa', MOCK_WEB_TESTS[:-1]: 'bbb', }, { MOCK_WEB_TESTS[:-1]: 'aaa', }) self.assertEqual( self.fs.read_binary_file(MOCK_WEB_TESTS + 'another/test-expected.txt'), 'result A') def test_move_baselines_skip_git_commands(self): self.fs.write_text_file(MOCK_WEB_TESTS + 'VirtualTestSuites', '[]') self.fs.write_binary_file( MOCK_WEB_TESTS + 'platform/win/another/test-expected.txt', 'result A') self.fs.write_binary_file( MOCK_WEB_TESTS + 'platform/mac/another/test-expected.txt', 'result A') self.fs.write_binary_file(MOCK_WEB_TESTS + 'another/test-expected.txt', 'result B') baseline_optimizer = BaselineOptimizer( self.host, self.host.port_factory.get(), self.host.port_factory.all_port_names()) baseline_optimizer._move_baselines( 'another/test-expected.txt', { MOCK_WEB_TESTS + 'platform/win': 'aaa', MOCK_WEB_TESTS + 'platform/mac': 'aaa', MOCK_WEB_TESTS[:-1]: 'bbb', }, { MOCK_WEB_TESTS + 'platform/linux': 'bbb', MOCK_WEB_TESTS[:-1]: 'aaa', }) self.assertEqual( self.fs.read_binary_file(MOCK_WEB_TESTS + 'another/test-expected.txt'), 'result A')