def test_additional_platform_directory(self):
filesystem = MockFileSystem()
options, args = optparse.OptionParser().parse_args([])
port = base.Port(port_name='foo',
filesystem=filesystem,
options=options)
port.baseline_search_path = lambda: []
layout_test_dir = port.layout_tests_dir()
test_file = filesystem.join(layout_test_dir, 'fast', 'test.html')
# No additional platform directory
self.assertEqual(port.expected_baselines(test_file, '.txt'),
[(None, 'fast/test-expected.txt')])
# Simple additional platform directory
options.additional_platform_directory = ['/tmp/local-baselines']
filesystem.files = {
'/tmp/local-baselines/fast/test-expected.txt': 'foo',
}
self.assertEqual(port.expected_baselines(test_file, '.txt'),
[('/tmp/local-baselines', 'fast/test-expected.txt')])
# Multiple additional platform directories
options.additional_platform_directory = [
'/foo', '/tmp/local-baselines'
]
self.assertEqual(port.expected_baselines(test_file, '.txt'),
[('/tmp/local-baselines', 'fast/test-expected.txt')])
def test_additional_platform_directory(self):
filesystem = MockFileSystem()
options, args = optparse.OptionParser().parse_args([])
port = base.Port(port_name='foo', filesystem=filesystem, options=options)
port.baseline_search_path = lambda: ['LayoutTests/platform/foo']
layout_test_dir = port.layout_tests_dir()
test_file = filesystem.join(layout_test_dir, 'fast', 'test.html')
# No additional platform directory
self.assertEqual(
port.expected_baselines(test_file, '.txt'),
[(None, 'fast/test-expected.txt')])
self.assertEqual(port.baseline_path(), 'LayoutTests/platform/foo')
# Simple additional platform directory
options.additional_platform_directory = ['/tmp/local-baselines']
filesystem.files = {
'/tmp/local-baselines/fast/test-expected.txt': 'foo',
}
self.assertEqual(
port.expected_baselines(test_file, '.txt'),
[('/tmp/local-baselines', 'fast/test-expected.txt')])
self.assertEqual(port.baseline_path(), '/tmp/local-baselines')
# Multiple additional platform directories
options.additional_platform_directory = ['/foo', '/tmp/local-baselines']
self.assertEqual(
port.expected_baselines(test_file, '.txt'),
[('/tmp/local-baselines', 'fast/test-expected.txt')])
self.assertEqual(port.baseline_path(), '/foo')
def test_collect_tests(self):
runner = self.create_runner()
webkit_base = '/test.checkout'
filesystem = MockFileSystem()
filename = filesystem.join(webkit_base, 'PerformanceTests', 'a_file.html')
filesystem.files[filename] = 'a content'
tests = runner._collect_tests(webkit_base, filesystem)
self.assertEqual(len(tests), 1)
def test_collect_tests(self):
runner = self.create_runner()
runner._base_path = '/test.checkout/PerformanceTests'
filesystem = MockFileSystem()
filename = filesystem.join(runner._base_path, 'inspector', 'a_file.html')
filesystem.maybe_make_directory(runner._base_path, 'inspector')
filesystem.files[filename] = 'a content'
runner._host.filesystem = filesystem
tests = runner._collect_tests()
self.assertEqual(len(tests), 1)
class HttpLockTest(unittest.TestCase):
def setUp(self):
self.filesystem = MockFileSystem()
self.http_lock = HttpLock(None,
"WebKitTestHttpd.lock.",
"WebKitTest.lock",
filesystem=self.filesystem,
executive=MockExecutive())
# FIXME: Shouldn't we be able to get these values from the http_lock object directly?
self.lock_file_path_prefix = self.filesystem.join(
self.http_lock._lock_path, self.http_lock._lock_file_prefix)
self.lock_file_name = self.lock_file_path_prefix + "0"
def test_current_lock_pid(self):
# FIXME: Once Executive wraps getpid, we can mock this and not use a real pid.
current_pid = os.getpid()
self.http_lock._filesystem.write_text_file(self.lock_file_name,
str(current_pid))
self.assertEqual(self.http_lock._current_lock_pid(), current_pid)
def test_extract_lock_number(self):
lock_file_list = (
self.lock_file_path_prefix + "00",
self.lock_file_path_prefix + "9",
self.lock_file_path_prefix + "001",
self.lock_file_path_prefix + "021",
)
expected_number_list = (0, 9, 1, 21)
for lock_file, expected in zip(lock_file_list, expected_number_list):
self.assertEqual(self.http_lock._extract_lock_number(lock_file),
expected)
def test_lock_file_list(self):
self.http_lock._filesystem = MockFileSystem({
self.lock_file_path_prefix + "6":
"",
self.lock_file_path_prefix + "1":
"",
self.lock_file_path_prefix + "4":
"",
self.lock_file_path_prefix + "3":
"",
})
expected_file_list = [
self.lock_file_path_prefix + "1",
self.lock_file_path_prefix + "3",
self.lock_file_path_prefix + "4",
self.lock_file_path_prefix + "6",
]
self.assertEqual(self.http_lock._lock_file_list(), expected_file_list)
def get_test_config(test_files=[], result_files=[]):
# We could grab this from port.layout_tests_dir(), but instantiating a fully mocked port is a pain.
layout_tests_directory = "/mock-checkout/LayoutTests"
results_directory = '/WebKitBuild/Debug/layout-test-results'
mock_filesystem = MockFileSystem()
for file in test_files:
file_path = mock_filesystem.join(layout_tests_directory, file)
mock_filesystem.files[file_path] = ''
for file in result_files:
file_path = mock_filesystem.join(results_directory, file)
mock_filesystem.files[file_path] = ''
class TestMacPort(WebKitPort):
port_name = "mac"
def __init__(self):
WebKitPort.__init__(self, filesystem=mock_filesystem, host=MockHost())
return TestConfig(
TestMacPort(),
layout_tests_directory,
results_directory,
('mac', 'mac-leopard', 'win', 'linux'),
mock_filesystem,
MockSCM())
class HttpLockTest(unittest.TestCase):
def setUp(self):
self.filesystem = MockFileSystem()
self.http_lock = HttpLock(
None, "WebKitTestHttpd.lock.", "WebKitTest.lock", filesystem=self.filesystem, executive=MockExecutive()
)
# FIXME: Shouldn't we be able to get these values from the http_lock object directly?
self.lock_file_path_prefix = self.filesystem.join(self.http_lock._lock_path, self.http_lock._lock_file_prefix)
self.lock_file_name = self.lock_file_path_prefix + "0"
def test_current_lock_pid(self):
# FIXME: Once Executive wraps getpid, we can mock this and not use a real pid.
current_pid = os.getpid()
self.http_lock._filesystem.write_text_file(self.lock_file_name, str(current_pid))
self.assertEqual(self.http_lock._current_lock_pid(), current_pid)
def test_extract_lock_number(self):
lock_file_list = (
self.lock_file_path_prefix + "00",
self.lock_file_path_prefix + "9",
self.lock_file_path_prefix + "001",
self.lock_file_path_prefix + "021",
)
expected_number_list = (0, 9, 1, 21)
for lock_file, expected in zip(lock_file_list, expected_number_list):
self.assertEqual(self.http_lock._extract_lock_number(lock_file), expected)
def test_lock_file_list(self):
self.http_lock._filesystem = MockFileSystem(
{
self.lock_file_path_prefix + "6": "",
self.lock_file_path_prefix + "1": "",
self.lock_file_path_prefix + "4": "",
self.lock_file_path_prefix + "3": "",
}
)
expected_file_list = [
self.lock_file_path_prefix + "1",
self.lock_file_path_prefix + "3",
self.lock_file_path_prefix + "4",
self.lock_file_path_prefix + "6",
]
self.assertEqual(self.http_lock._lock_file_list(), expected_file_list)
def test_additional_platform_directory(self):
filesystem = MockFileSystem()
options, args = optparse.OptionParser().parse_args([])
port = base.Port(port_name="foo", filesystem=filesystem, options=options)
port.baseline_search_path = lambda: []
layout_test_dir = port.layout_tests_dir()
test_file = filesystem.join(layout_test_dir, "fast", "test.html")
# No additional platform directory
self.assertEqual(port.expected_baselines(test_file, ".txt"), [(None, "fast/test-expected.txt")])
# Simple additional platform directory
options.additional_platform_directory = ["/tmp/local-baselines"]
filesystem.files = {"/tmp/local-baselines/fast/test-expected.txt": "foo"}
self.assertEqual(
port.expected_baselines(test_file, ".txt"), [("/tmp/local-baselines", "fast/test-expected.txt")]
)
# Multiple additional platform directories
options.additional_platform_directory = ["/foo", "/tmp/local-baselines"]
self.assertEqual(
port.expected_baselines(test_file, ".txt"), [("/tmp/local-baselines", "fast/test-expected.txt")]
)
class BaselineOptimizerTest(unittest.TestCase):
def setUp(self):
self.host = MockHost()
self.fs = MockFileSystem()
self.host.filesystem = self.fs
# TODO(robertma): Even though we have mocked the builder list (and hence
# all_port_names), we are still relying on the knowledge of currently
# configured ports and their fallback order. Ideally, we should improve
# MockPortFactory and use it.
self.host.builders = BuilderList({
'Fake Test Win10': {
'port_name': 'win-win10',
'specifiers': ['Win10', 'Release']
},
'Fake Test Linux': {
'port_name': 'linux-trusty',
'specifiers': ['Trusty', 'Release']
},
'Fake Test Mac10.13': {
'port_name': 'mac-mac10.13',
'specifiers': ['Mac10.13', 'Release']
},
'Fake Test Mac10.12': {
'port_name': 'mac-mac10.12',
'specifiers': ['Mac10.12', 'Release']
},
'Fake Test Mac10.11': {
'port_name': 'mac-mac10.11',
'specifiers': ['Mac10.11', 'Release']
},
'Fake Test Mac10.10': {
'port_name': 'mac-mac10.10',
'specifiers': ['Mac10.10', 'Release']
},
})
# Note: this is a pre-assumption of the tests in this file. If this
# assertion fails, port configurations are likely changed, and the
# tests need to be adjusted accordingly.
self.assertEqual(sorted(self.host.port_factory.all_port_names()), [
'linux-trusty', 'mac-mac10.10', 'mac-mac10.11', 'mac-mac10.12',
'mac-mac10.13', 'win-win10'
])
def _assert_optimization(self,
results_by_directory,
directory_to_new_results,
baseline_dirname=''):
layout_tests_dir = PathFinder(self.fs).layout_tests_dir()
test_name = 'mock-test.html'
baseline_name = 'mock-test-expected.txt'
self.fs.write_text_file(
self.fs.join(layout_tests_dir, 'VirtualTestSuites'),
'[{"prefix": "gpu", "base": "fast/canvas", "args": ["--foo"]}]')
for dirname, contents in results_by_directory.items():
self.fs.write_binary_file(
self.fs.join(layout_tests_dir, dirname, baseline_name),
contents)
baseline_optimizer = BaselineOptimizer(
self.host, self.host.port_factory.get(),
self.host.port_factory.all_port_names())
self.assertTrue(
baseline_optimizer.optimize(
self.fs.join(baseline_dirname, test_name), 'txt'))
for dirname, contents in directory_to_new_results.items():
path = self.fs.join(layout_tests_dir, dirname, baseline_name)
if contents is None:
# Check files that are explicitly marked as absent.
self.assertFalse(
self.fs.exists(path),
'%s should not exist after optimization' % path)
else:
self.assertEqual(self.fs.read_binary_file(path), contents,
'Content of %s != "%s"' % (path, contents))
for dirname in results_by_directory:
path = self.fs.join(layout_tests_dir, dirname, baseline_name)
if dirname not in directory_to_new_results or directory_to_new_results[
dirname] is None:
self.assertFalse(
self.fs.exists(path),
'%s should not exist after optimization' % path)
def test_linux_redundant_with_win(self):
self._assert_optimization(
{
'platform/win': '1',
'platform/linux': '1',
}, {
'platform/win': '1',
})
def test_covers_mac_win_linux(self):
self._assert_optimization(
{
'platform/mac': '1',
'platform/win': '1',
'platform/linux': '1',
}, {
'': '1',
})
def test_overwrites_root(self):
self._assert_optimization(
{
'platform/mac': '1',
'platform/win': '1',
'platform/linux': '1',
'': '2',
}, {
'': '1',
})
def test_no_new_common_directory(self):
self._assert_optimization(
{
'platform/mac': '1',
'platform/linux': '1',
'': '2',
}, {
'platform/mac': '1',
'platform/linux': '1',
'': '2',
})
def test_local_optimization(self):
self._assert_optimization(
{
'platform/mac': '1',
'platform/linux': '1',
'platform/mac-mac10.11': '1',
}, {
'platform/mac': '1',
'platform/linux': '1',
})
def test_local_optimization_skipping_a_port_in_the_middle(self):
# mac-mac10.10 -> mac-mac10.11 -> mac
self._assert_optimization(
{
'platform/mac': '1',
'platform/linux': '1',
'platform/mac-mac10.10': '1',
}, {
'platform/mac': '1',
'platform/linux': '1',
})
def test_baseline_redundant_with_root(self):
self._assert_optimization(
{
'platform/mac': '1',
'platform/win': '2',
'': '2',
}, {
'platform/mac': '1',
'': '2',
})
def test_root_baseline_unused(self):
self._assert_optimization(
{
'platform/mac': '1',
'platform/win': '2',
'': '3',
}, {
'platform/mac': '1',
'platform/win': '2',
})
def test_root_baseline_unused_and_non_existant(self):
self._assert_optimization({
'platform/mac': '1',
'platform/win': '2',
}, {
'platform/mac': '1',
'platform/win': '2',
})
def test_virtual_baseline_redundant_with_non_virtual(self):
self._assert_optimization(
{
'platform/win/virtual/gpu/fast/canvas': '2',
'platform/win/fast/canvas': '2',
}, {
'platform/win/fast/canvas': '2',
},
baseline_dirname='virtual/gpu/fast/canvas')
def test_virtual_baseline_redundant_with_non_virtual_fallback(self):
# virtual linux -> virtual win -> virtual root -> linux -> win
self._assert_optimization(
{
'platform/linux/virtual/gpu/fast/canvas': '2',
'platform/win/fast/canvas': '2',
}, {
'platform/win/virtual/gpu/fast/canvas': None,
'platform/win/fast/canvas': '2',
},
baseline_dirname='virtual/gpu/fast/canvas')
def test_virtual_baseline_redundant_with_actual_root(self):
self._assert_optimization(
{
'platform/win/virtual/gpu/fast/canvas': '2',
'fast/canvas': '2',
}, {
'fast/canvas': '2',
},
baseline_dirname='virtual/gpu/fast/canvas')
def test_virtual_root_redundant_with_actual_root(self):
self._assert_optimization(
{
'virtual/gpu/fast/canvas': '2',
'fast/canvas': '2',
}, {
'fast/canvas': '2',
},
baseline_dirname='virtual/gpu/fast/canvas')
def test_virtual_root_redundant_with_ancestors(self):
self._assert_optimization(
{
'virtual/gpu/fast/canvas': '2',
'platform/mac/fast/canvas': '2',
'platform/win/fast/canvas': '2',
}, {
'fast/canvas': '2',
},
baseline_dirname='virtual/gpu/fast/canvas')
def test_virtual_root_not_redundant_with_ancestors(self):
self._assert_optimization(
{
'virtual/gpu/fast/canvas': '2',
'platform/mac/fast/canvas': '1',
}, {
'virtual/gpu/fast/canvas': '2',
'platform/mac/fast/canvas': '1',
},
baseline_dirname='virtual/gpu/fast/canvas')
def test_virtual_covers_mac_win_linux(self):
self._assert_optimization(
{
'platform/mac/virtual/gpu/fast/canvas': '1',
'platform/win/virtual/gpu/fast/canvas': '1',
'platform/linux/virtual/gpu/fast/canvas': '1',
}, {
'virtual/gpu/fast/canvas': '1',
},
baseline_dirname='virtual/gpu/fast/canvas')
def test_all_pass_testharness_at_root(self):
self._assert_optimization({'': ALL_PASS_TESTHARNESS_RESULT},
{'': None})
def test_all_pass_testharness_at_linux(self):
self._assert_optimization(
{'platform/linux': ALL_PASS_TESTHARNESS_RESULT},
{'platform/linux': None})
def test_all_pass_testharness_at_linux_and_win(self):
# https://crbug.com/805008
self._assert_optimization(
{
'platform/linux': ALL_PASS_TESTHARNESS_RESULT,
'platform/win': ALL_PASS_TESTHARNESS_RESULT
}, {
'platform/linux': None,
'platform/win': None
})
def test_all_pass_testharness_at_virtual_root(self):
self._assert_optimization(
{'virtual/gpu/fast/canvas': ALL_PASS_TESTHARNESS_RESULT},
{'virtual/gpu/fast/canvas': None},
baseline_dirname='virtual/gpu/fast/canvas')
def test_all_pass_testharness_at_virtual_linux(self):
self._assert_optimization(
{
'platform/linux/virtual/gpu/fast/canvas':
ALL_PASS_TESTHARNESS_RESULT
}, {'platform/linux/virtual/gpu/fast/canvas': None},
baseline_dirname='virtual/gpu/fast/canvas')
def test_all_pass_testharness_falls_back_to_non_pass(self):
# The all-PASS baseline needs to be preserved in this case.
self._assert_optimization(
{
'platform/linux': ALL_PASS_TESTHARNESS_RESULT,
'': '1'
}, {
'platform/linux': ALL_PASS_TESTHARNESS_RESULT,
'': '1'
})
def test_virtual_all_pass_testharness_falls_back_to_base(self):
# The all-PASS baseline needs to be preserved in this case.
self._assert_optimization(
{
'virtual/gpu/fast/canvas': ALL_PASS_TESTHARNESS_RESULT,
'platform/linux/fast/canvas': '1',
}, {
'virtual/gpu/fast/canvas': ALL_PASS_TESTHARNESS_RESULT,
'platform/linux/fast/canvas': '1',
},
baseline_dirname='virtual/gpu/fast/canvas')
# Tests for protected methods - pylint: disable=protected-access
def test_move_baselines(self):
self.fs.write_text_file(
'/mock-checkout/third_party/WebKit/LayoutTests/VirtualTestSuites',
'[]')
self.fs.write_binary_file(
'/mock-checkout/third_party/WebKit/LayoutTests/platform/win/another/test-expected.txt',
'result A')
self.fs.write_binary_file(
'/mock-checkout/third_party/WebKit/LayoutTests/platform/mac/another/test-expected.txt',
'result A')
self.fs.write_binary_file(
'/mock-checkout/third_party/WebKit/LayoutTests/another/test-expected.txt',
'result B')
baseline_optimizer = BaselineOptimizer(
self.host, self.host.port_factory.get(),
self.host.port_factory.all_port_names())
baseline_optimizer._move_baselines(
'another/test-expected.txt', {
'/mock-checkout/third_party/WebKit/LayoutTests/platform/win':
'aaa',
'/mock-checkout/third_party/WebKit/LayoutTests/platform/mac':
'aaa',
'/mock-checkout/third_party/WebKit/LayoutTests': 'bbb',
}, {
'/mock-checkout/third_party/WebKit/LayoutTests': 'aaa',
})
self.assertEqual(
self.fs.read_binary_file(
'/mock-checkout/third_party/WebKit/LayoutTests/another/test-expected.txt'
), 'result A')
def test_move_baselines_skip_git_commands(self):
self.fs.write_text_file(
'/mock-checkout/third_party/WebKit/LayoutTests/VirtualTestSuites',
'[]')
self.fs.write_binary_file(
'/mock-checkout/third_party/WebKit/LayoutTests/platform/win/another/test-expected.txt',
'result A')
self.fs.write_binary_file(
'/mock-checkout/third_party/WebKit/LayoutTests/platform/mac/another/test-expected.txt',
'result A')
self.fs.write_binary_file(
'/mock-checkout/third_party/WebKit/LayoutTests/another/test-expected.txt',
'result B')
baseline_optimizer = BaselineOptimizer(
self.host, self.host.port_factory.get(),
self.host.port_factory.all_port_names())
baseline_optimizer._move_baselines(
'another/test-expected.txt', {
'/mock-checkout/third_party/WebKit/LayoutTests/platform/win':
'aaa',
'/mock-checkout/third_party/WebKit/LayoutTests/platform/mac':
'aaa',
'/mock-checkout/third_party/WebKit/LayoutTests': 'bbb',
}, {
'/mock-checkout/third_party/WebKit/LayoutTests/platform/linux':
'bbb',
'/mock-checkout/third_party/WebKit/LayoutTests': 'aaa',
})
self.assertEqual(
self.fs.read_binary_file(
'/mock-checkout/third_party/WebKit/LayoutTests/another/test-expected.txt'
), 'result A')
