def test_additional_platform_directory(self): filesystem = MockFileSystem() options, args = optparse.OptionParser().parse_args([]) port = base.Port(port_name='foo', filesystem=filesystem, options=options) port.baseline_search_path = lambda: [] layout_test_dir = port.layout_tests_dir() test_file = filesystem.join(layout_test_dir, 'fast', 'test.html') # No additional platform directory self.assertEqual(port.expected_baselines(test_file, '.txt'), [(None, 'fast/test-expected.txt')]) # Simple additional platform directory options.additional_platform_directory = ['/tmp/local-baselines'] filesystem.files = { '/tmp/local-baselines/fast/test-expected.txt': 'foo', } self.assertEqual(port.expected_baselines(test_file, '.txt'), [('/tmp/local-baselines', 'fast/test-expected.txt')]) # Multiple additional platform directories options.additional_platform_directory = [ '/foo', '/tmp/local-baselines' ] self.assertEqual(port.expected_baselines(test_file, '.txt'), [('/tmp/local-baselines', 'fast/test-expected.txt')])
def test_additional_platform_directory(self): filesystem = MockFileSystem() options, args = optparse.OptionParser().parse_args([]) port = base.Port(port_name='foo', filesystem=filesystem, options=options) port.baseline_search_path = lambda: ['LayoutTests/platform/foo'] layout_test_dir = port.layout_tests_dir() test_file = filesystem.join(layout_test_dir, 'fast', 'test.html') # No additional platform directory self.assertEqual( port.expected_baselines(test_file, '.txt'), [(None, 'fast/test-expected.txt')]) self.assertEqual(port.baseline_path(), 'LayoutTests/platform/foo') # Simple additional platform directory options.additional_platform_directory = ['/tmp/local-baselines'] filesystem.files = { '/tmp/local-baselines/fast/test-expected.txt': 'foo', } self.assertEqual( port.expected_baselines(test_file, '.txt'), [('/tmp/local-baselines', 'fast/test-expected.txt')]) self.assertEqual(port.baseline_path(), '/tmp/local-baselines') # Multiple additional platform directories options.additional_platform_directory = ['/foo', '/tmp/local-baselines'] self.assertEqual( port.expected_baselines(test_file, '.txt'), [('/tmp/local-baselines', 'fast/test-expected.txt')]) self.assertEqual(port.baseline_path(), '/foo')
def test_collect_tests(self): runner = self.create_runner() webkit_base = '/test.checkout' filesystem = MockFileSystem() filename = filesystem.join(webkit_base, 'PerformanceTests', 'a_file.html') filesystem.files[filename] = 'a content' tests = runner._collect_tests(webkit_base, filesystem) self.assertEqual(len(tests), 1)
def test_collect_tests(self): runner = self.create_runner() runner._base_path = '/test.checkout/PerformanceTests' filesystem = MockFileSystem() filename = filesystem.join(runner._base_path, 'inspector', 'a_file.html') filesystem.maybe_make_directory(runner._base_path, 'inspector') filesystem.files[filename] = 'a content' runner._host.filesystem = filesystem tests = runner._collect_tests() self.assertEqual(len(tests), 1)
class HttpLockTest(unittest.TestCase): def setUp(self): self.filesystem = MockFileSystem() self.http_lock = HttpLock(None, "WebKitTestHttpd.lock.", "WebKitTest.lock", filesystem=self.filesystem, executive=MockExecutive()) # FIXME: Shouldn't we be able to get these values from the http_lock object directly? self.lock_file_path_prefix = self.filesystem.join( self.http_lock._lock_path, self.http_lock._lock_file_prefix) self.lock_file_name = self.lock_file_path_prefix + "0" def test_current_lock_pid(self): # FIXME: Once Executive wraps getpid, we can mock this and not use a real pid. current_pid = os.getpid() self.http_lock._filesystem.write_text_file(self.lock_file_name, str(current_pid)) self.assertEqual(self.http_lock._current_lock_pid(), current_pid) def test_extract_lock_number(self): lock_file_list = ( self.lock_file_path_prefix + "00", self.lock_file_path_prefix + "9", self.lock_file_path_prefix + "001", self.lock_file_path_prefix + "021", ) expected_number_list = (0, 9, 1, 21) for lock_file, expected in zip(lock_file_list, expected_number_list): self.assertEqual(self.http_lock._extract_lock_number(lock_file), expected) def test_lock_file_list(self): self.http_lock._filesystem = MockFileSystem({ self.lock_file_path_prefix + "6": "", self.lock_file_path_prefix + "1": "", self.lock_file_path_prefix + "4": "", self.lock_file_path_prefix + "3": "", }) expected_file_list = [ self.lock_file_path_prefix + "1", self.lock_file_path_prefix + "3", self.lock_file_path_prefix + "4", self.lock_file_path_prefix + "6", ] self.assertEqual(self.http_lock._lock_file_list(), expected_file_list)
def get_test_config(test_files=[], result_files=[]): # We could grab this from port.layout_tests_dir(), but instantiating a fully mocked port is a pain. layout_tests_directory = "/mock-checkout/LayoutTests" results_directory = '/WebKitBuild/Debug/layout-test-results' mock_filesystem = MockFileSystem() for file in test_files: file_path = mock_filesystem.join(layout_tests_directory, file) mock_filesystem.files[file_path] = '' for file in result_files: file_path = mock_filesystem.join(results_directory, file) mock_filesystem.files[file_path] = '' class TestMacPort(WebKitPort): port_name = "mac" def __init__(self): WebKitPort.__init__(self, filesystem=mock_filesystem, host=MockHost()) return TestConfig( TestMacPort(), layout_tests_directory, results_directory, ('mac', 'mac-leopard', 'win', 'linux'), mock_filesystem, MockSCM())
class HttpLockTest(unittest.TestCase): def setUp(self): self.filesystem = MockFileSystem() self.http_lock = HttpLock( None, "WebKitTestHttpd.lock.", "WebKitTest.lock", filesystem=self.filesystem, executive=MockExecutive() ) # FIXME: Shouldn't we be able to get these values from the http_lock object directly? self.lock_file_path_prefix = self.filesystem.join(self.http_lock._lock_path, self.http_lock._lock_file_prefix) self.lock_file_name = self.lock_file_path_prefix + "0" def test_current_lock_pid(self): # FIXME: Once Executive wraps getpid, we can mock this and not use a real pid. current_pid = os.getpid() self.http_lock._filesystem.write_text_file(self.lock_file_name, str(current_pid)) self.assertEqual(self.http_lock._current_lock_pid(), current_pid) def test_extract_lock_number(self): lock_file_list = ( self.lock_file_path_prefix + "00", self.lock_file_path_prefix + "9", self.lock_file_path_prefix + "001", self.lock_file_path_prefix + "021", ) expected_number_list = (0, 9, 1, 21) for lock_file, expected in zip(lock_file_list, expected_number_list): self.assertEqual(self.http_lock._extract_lock_number(lock_file), expected) def test_lock_file_list(self): self.http_lock._filesystem = MockFileSystem( { self.lock_file_path_prefix + "6": "", self.lock_file_path_prefix + "1": "", self.lock_file_path_prefix + "4": "", self.lock_file_path_prefix + "3": "", } ) expected_file_list = [ self.lock_file_path_prefix + "1", self.lock_file_path_prefix + "3", self.lock_file_path_prefix + "4", self.lock_file_path_prefix + "6", ] self.assertEqual(self.http_lock._lock_file_list(), expected_file_list)
def test_additional_platform_directory(self): filesystem = MockFileSystem() options, args = optparse.OptionParser().parse_args([]) port = base.Port(port_name="foo", filesystem=filesystem, options=options) port.baseline_search_path = lambda: [] layout_test_dir = port.layout_tests_dir() test_file = filesystem.join(layout_test_dir, "fast", "test.html") # No additional platform directory self.assertEqual(port.expected_baselines(test_file, ".txt"), [(None, "fast/test-expected.txt")]) # Simple additional platform directory options.additional_platform_directory = ["/tmp/local-baselines"] filesystem.files = {"/tmp/local-baselines/fast/test-expected.txt": "foo"} self.assertEqual( port.expected_baselines(test_file, ".txt"), [("/tmp/local-baselines", "fast/test-expected.txt")] ) # Multiple additional platform directories options.additional_platform_directory = ["/foo", "/tmp/local-baselines"] self.assertEqual( port.expected_baselines(test_file, ".txt"), [("/tmp/local-baselines", "fast/test-expected.txt")] )
class BaselineOptimizerTest(unittest.TestCase): def setUp(self): self.host = MockHost() self.fs = MockFileSystem() self.host.filesystem = self.fs # TODO(robertma): Even though we have mocked the builder list (and hence # all_port_names), we are still relying on the knowledge of currently # configured ports and their fallback order. Ideally, we should improve # MockPortFactory and use it. self.host.builders = BuilderList({ 'Fake Test Win10': { 'port_name': 'win-win10', 'specifiers': ['Win10', 'Release'] }, 'Fake Test Linux': { 'port_name': 'linux-trusty', 'specifiers': ['Trusty', 'Release'] }, 'Fake Test Mac10.13': { 'port_name': 'mac-mac10.13', 'specifiers': ['Mac10.13', 'Release'] }, 'Fake Test Mac10.12': { 'port_name': 'mac-mac10.12', 'specifiers': ['Mac10.12', 'Release'] }, 'Fake Test Mac10.11': { 'port_name': 'mac-mac10.11', 'specifiers': ['Mac10.11', 'Release'] }, 'Fake Test Mac10.10': { 'port_name': 'mac-mac10.10', 'specifiers': ['Mac10.10', 'Release'] }, }) # Note: this is a pre-assumption of the tests in this file. If this # assertion fails, port configurations are likely changed, and the # tests need to be adjusted accordingly. self.assertEqual(sorted(self.host.port_factory.all_port_names()), [ 'linux-trusty', 'mac-mac10.10', 'mac-mac10.11', 'mac-mac10.12', 'mac-mac10.13', 'win-win10' ]) def _assert_optimization(self, results_by_directory, directory_to_new_results, baseline_dirname=''): layout_tests_dir = PathFinder(self.fs).layout_tests_dir() test_name = 'mock-test.html' baseline_name = 'mock-test-expected.txt' self.fs.write_text_file( self.fs.join(layout_tests_dir, 'VirtualTestSuites'), '[{"prefix": "gpu", "base": "fast/canvas", "args": ["--foo"]}]') for dirname, contents in results_by_directory.items(): self.fs.write_binary_file( self.fs.join(layout_tests_dir, dirname, baseline_name), contents) baseline_optimizer = BaselineOptimizer( self.host, self.host.port_factory.get(), self.host.port_factory.all_port_names()) self.assertTrue( baseline_optimizer.optimize( self.fs.join(baseline_dirname, test_name), 'txt')) for dirname, contents in directory_to_new_results.items(): path = self.fs.join(layout_tests_dir, dirname, baseline_name) if contents is None: # Check files that are explicitly marked as absent. self.assertFalse( self.fs.exists(path), '%s should not exist after optimization' % path) else: self.assertEqual(self.fs.read_binary_file(path), contents, 'Content of %s != "%s"' % (path, contents)) for dirname in results_by_directory: path = self.fs.join(layout_tests_dir, dirname, baseline_name) if dirname not in directory_to_new_results or directory_to_new_results[ dirname] is None: self.assertFalse( self.fs.exists(path), '%s should not exist after optimization' % path) def test_linux_redundant_with_win(self): self._assert_optimization( { 'platform/win': '1', 'platform/linux': '1', }, { 'platform/win': '1', }) def test_covers_mac_win_linux(self): self._assert_optimization( { 'platform/mac': '1', 'platform/win': '1', 'platform/linux': '1', }, { '': '1', }) def test_overwrites_root(self): self._assert_optimization( { 'platform/mac': '1', 'platform/win': '1', 'platform/linux': '1', '': '2', }, { '': '1', }) def test_no_new_common_directory(self): self._assert_optimization( { 'platform/mac': '1', 'platform/linux': '1', '': '2', }, { 'platform/mac': '1', 'platform/linux': '1', '': '2', }) def test_local_optimization(self): self._assert_optimization( { 'platform/mac': '1', 'platform/linux': '1', 'platform/mac-mac10.11': '1', }, { 'platform/mac': '1', 'platform/linux': '1', }) def test_local_optimization_skipping_a_port_in_the_middle(self): # mac-mac10.10 -> mac-mac10.11 -> mac self._assert_optimization( { 'platform/mac': '1', 'platform/linux': '1', 'platform/mac-mac10.10': '1', }, { 'platform/mac': '1', 'platform/linux': '1', }) def test_baseline_redundant_with_root(self): self._assert_optimization( { 'platform/mac': '1', 'platform/win': '2', '': '2', }, { 'platform/mac': '1', '': '2', }) def test_root_baseline_unused(self): self._assert_optimization( { 'platform/mac': '1', 'platform/win': '2', '': '3', }, { 'platform/mac': '1', 'platform/win': '2', }) def test_root_baseline_unused_and_non_existant(self): self._assert_optimization({ 'platform/mac': '1', 'platform/win': '2', }, { 'platform/mac': '1', 'platform/win': '2', }) def test_virtual_baseline_redundant_with_non_virtual(self): self._assert_optimization( { 'platform/win/virtual/gpu/fast/canvas': '2', 'platform/win/fast/canvas': '2', }, { 'platform/win/fast/canvas': '2', }, baseline_dirname='virtual/gpu/fast/canvas') def test_virtual_baseline_redundant_with_non_virtual_fallback(self): # virtual linux -> virtual win -> virtual root -> linux -> win self._assert_optimization( { 'platform/linux/virtual/gpu/fast/canvas': '2', 'platform/win/fast/canvas': '2', }, { 'platform/win/virtual/gpu/fast/canvas': None, 'platform/win/fast/canvas': '2', }, baseline_dirname='virtual/gpu/fast/canvas') def test_virtual_baseline_redundant_with_actual_root(self): self._assert_optimization( { 'platform/win/virtual/gpu/fast/canvas': '2', 'fast/canvas': '2', }, { 'fast/canvas': '2', }, baseline_dirname='virtual/gpu/fast/canvas') def test_virtual_root_redundant_with_actual_root(self): self._assert_optimization( { 'virtual/gpu/fast/canvas': '2', 'fast/canvas': '2', }, { 'fast/canvas': '2', }, baseline_dirname='virtual/gpu/fast/canvas') def test_virtual_root_redundant_with_ancestors(self): self._assert_optimization( { 'virtual/gpu/fast/canvas': '2', 'platform/mac/fast/canvas': '2', 'platform/win/fast/canvas': '2', }, { 'fast/canvas': '2', }, baseline_dirname='virtual/gpu/fast/canvas') def test_virtual_root_not_redundant_with_ancestors(self): self._assert_optimization( { 'virtual/gpu/fast/canvas': '2', 'platform/mac/fast/canvas': '1', }, { 'virtual/gpu/fast/canvas': '2', 'platform/mac/fast/canvas': '1', }, baseline_dirname='virtual/gpu/fast/canvas') def test_virtual_covers_mac_win_linux(self): self._assert_optimization( { 'platform/mac/virtual/gpu/fast/canvas': '1', 'platform/win/virtual/gpu/fast/canvas': '1', 'platform/linux/virtual/gpu/fast/canvas': '1', }, { 'virtual/gpu/fast/canvas': '1', }, baseline_dirname='virtual/gpu/fast/canvas') def test_all_pass_testharness_at_root(self): self._assert_optimization({'': ALL_PASS_TESTHARNESS_RESULT}, {'': None}) def test_all_pass_testharness_at_linux(self): self._assert_optimization( {'platform/linux': ALL_PASS_TESTHARNESS_RESULT}, {'platform/linux': None}) def test_all_pass_testharness_at_linux_and_win(self): # https://crbug.com/805008 self._assert_optimization( { 'platform/linux': ALL_PASS_TESTHARNESS_RESULT, 'platform/win': ALL_PASS_TESTHARNESS_RESULT }, { 'platform/linux': None, 'platform/win': None }) def test_all_pass_testharness_at_virtual_root(self): self._assert_optimization( {'virtual/gpu/fast/canvas': ALL_PASS_TESTHARNESS_RESULT}, {'virtual/gpu/fast/canvas': None}, baseline_dirname='virtual/gpu/fast/canvas') def test_all_pass_testharness_at_virtual_linux(self): self._assert_optimization( { 'platform/linux/virtual/gpu/fast/canvas': ALL_PASS_TESTHARNESS_RESULT }, {'platform/linux/virtual/gpu/fast/canvas': None}, baseline_dirname='virtual/gpu/fast/canvas') def test_all_pass_testharness_falls_back_to_non_pass(self): # The all-PASS baseline needs to be preserved in this case. self._assert_optimization( { 'platform/linux': ALL_PASS_TESTHARNESS_RESULT, '': '1' }, { 'platform/linux': ALL_PASS_TESTHARNESS_RESULT, '': '1' }) def test_virtual_all_pass_testharness_falls_back_to_base(self): # The all-PASS baseline needs to be preserved in this case. self._assert_optimization( { 'virtual/gpu/fast/canvas': ALL_PASS_TESTHARNESS_RESULT, 'platform/linux/fast/canvas': '1', }, { 'virtual/gpu/fast/canvas': ALL_PASS_TESTHARNESS_RESULT, 'platform/linux/fast/canvas': '1', }, baseline_dirname='virtual/gpu/fast/canvas') # Tests for protected methods - pylint: disable=protected-access def test_move_baselines(self): self.fs.write_text_file( '/mock-checkout/third_party/WebKit/LayoutTests/VirtualTestSuites', '[]') self.fs.write_binary_file( '/mock-checkout/third_party/WebKit/LayoutTests/platform/win/another/test-expected.txt', 'result A') self.fs.write_binary_file( '/mock-checkout/third_party/WebKit/LayoutTests/platform/mac/another/test-expected.txt', 'result A') self.fs.write_binary_file( '/mock-checkout/third_party/WebKit/LayoutTests/another/test-expected.txt', 'result B') baseline_optimizer = BaselineOptimizer( self.host, self.host.port_factory.get(), self.host.port_factory.all_port_names()) baseline_optimizer._move_baselines( 'another/test-expected.txt', { '/mock-checkout/third_party/WebKit/LayoutTests/platform/win': 'aaa', '/mock-checkout/third_party/WebKit/LayoutTests/platform/mac': 'aaa', '/mock-checkout/third_party/WebKit/LayoutTests': 'bbb', }, { '/mock-checkout/third_party/WebKit/LayoutTests': 'aaa', }) self.assertEqual( self.fs.read_binary_file( '/mock-checkout/third_party/WebKit/LayoutTests/another/test-expected.txt' ), 'result A') def test_move_baselines_skip_git_commands(self): self.fs.write_text_file( '/mock-checkout/third_party/WebKit/LayoutTests/VirtualTestSuites', '[]') self.fs.write_binary_file( '/mock-checkout/third_party/WebKit/LayoutTests/platform/win/another/test-expected.txt', 'result A') self.fs.write_binary_file( '/mock-checkout/third_party/WebKit/LayoutTests/platform/mac/another/test-expected.txt', 'result A') self.fs.write_binary_file( '/mock-checkout/third_party/WebKit/LayoutTests/another/test-expected.txt', 'result B') baseline_optimizer = BaselineOptimizer( self.host, self.host.port_factory.get(), self.host.port_factory.all_port_names()) baseline_optimizer._move_baselines( 'another/test-expected.txt', { '/mock-checkout/third_party/WebKit/LayoutTests/platform/win': 'aaa', '/mock-checkout/third_party/WebKit/LayoutTests/platform/mac': 'aaa', '/mock-checkout/third_party/WebKit/LayoutTests': 'bbb', }, { '/mock-checkout/third_party/WebKit/LayoutTests/platform/linux': 'bbb', '/mock-checkout/third_party/WebKit/LayoutTests': 'aaa', }) self.assertEqual( self.fs.read_binary_file( '/mock-checkout/third_party/WebKit/LayoutTests/another/test-expected.txt' ), 'result A')