def setUp(self):
self.host = MockHost()
self.fs = MockFileSystem()
self.host.filesystem = self.fs
self.fs.write_text_file('/all-pass/foo-expected.txt', ALL_PASS_TESTHARNESS_RESULT)
self.fs.write_text_file('/all-pass/bar-expected.txt', ALL_PASS_TESTHARNESS_RESULT2)
self.fs.write_text_file('/failures/baz-expected.txt', 'failure')
def test_depot_tools_base_not_found(self):
filesystem = MockFileSystem()
filesystem.path_to_module = lambda _: (
'/mock-checkout/third_party/blink/tools/blinkpy/common/'
'path_finder.py')
finder = PathFinder(filesystem)
self.assertIsNone(finder.depot_tools_base())
def setUp(self):
# We always have an OWNERS file at web_tests/external.
self.filesystem = MockFileSystem(files={
MOCK_WEB_TESTS + 'external/OWNERS':
'*****@*****.**'
})
self.extractor = DirectoryOwnersExtractor(self.filesystem)
def test_layout_tests_dir_with_backslash_sep(self):
filesystem = MockFileSystem()
filesystem.sep = '\\'
filesystem.path_to_module = lambda _: (
'C:\\mock-checkout\\third_party\\blink\\tools\\blinkpy\\foo.py')
finder = PathFinder(filesystem)
self.assertEqual(finder.layout_tests_dir(),
'C:\\mock-checkout\\third_party\\blink\\web_tests')
def setUp(self):
# We always have an OWNERS file at LayoutTests/external.
self.filesystem = MockFileSystem(
files={
'/mock-checkout/third_party/WebKit/LayoutTests/external/OWNERS':
'*****@*****.**'
})
self.extractor = DirectoryOwnersExtractor(self.filesystem)
def test_depot_tools_base_exists(self):
filesystem = MockFileSystem()
filesystem.path_to_module = lambda _: (
'/checkout/third_party/blink/tools/blinkpy/common/'
'path_finder.py')
filesystem.maybe_make_directory('/checkout/third_party/depot_tools')
finder = PathFinder(filesystem)
self.assertEqual(finder.depot_tools_base(),
'/checkout/third_party/depot_tools')
def assertMergeResults(self,
before,
after,
mock_filesystem_contents,
inputargs,
filesystem_contains,
json_data_merger=None):
mock_filesystem = MockFileSystem(mock_filesystem_contents,
dirs=['/output'])
file_merger = merge_results.MergeFilesJSONP(mock_filesystem,
json_data_merger)
file_merger(*inputargs)
files = mock_filesystem.files_under('/output')
self.assertTrue(len(files) == 1)
expected_mock_filesystem = MockFileSystem(filesystem_contains)
expected_files = expected_mock_filesystem.files_under('/output')
actual_output = mock_filesystem.read_text_file(files[0])
expected_output = expected_mock_filesystem.read_text_file(
expected_files[0])
self.assertTrue(self.check_before_after(actual_output, before, after))
self.assertTrue(self.check_before_after(expected_output, before,
after))
actual_json_str = self.remove_before_after(actual_output, before,
after)
expected_json_str = self.remove_before_after(expected_output, before,
after)
self.assertEqual(json.loads(actual_json_str),
json.loads(expected_json_str))
def setUp(self):
self.host = MockHost()
self.fs = MockFileSystem()
self.host.filesystem = self.fs
# TODO(robertma): Even though we have mocked the builder list (and hence
# all_port_names), we are still relying on the knowledge of currently
# configured ports and their fallback order. Ideally, we should improve
# MockPortFactory and use it.
self.host.builders = BuilderList({
'Fake Test Win10': {
'port_name': 'win-win10',
'specifiers': ['Win10', 'Release']
},
'Fake Test Linux': {
'port_name': 'linux-trusty',
'specifiers': ['Trusty', 'Release']
},
'Fake Test Mac10.15': {
'port_name': 'mac-mac10.15',
'specifiers': ['Mac10.15', 'Release']
},
'Fake Test Mac10.14': {
'port_name': 'mac-mac10.14',
'specifiers': ['Mac10.14', 'Release']
},
'Fake Test Mac10.13': {
'port_name': 'mac-mac10.13',
'specifiers': ['Mac10.13', 'Release']
},
'Fake Test Mac10.12': {
'port_name': 'mac-mac10.12',
'specifiers': ['Mac10.12', 'Release']
},
'Fake Test Mac10.11': {
'port_name': 'mac-mac10.11',
'specifiers': ['Mac10.11', 'Release']
},
'Fake Test Mac10.10': {
'port_name': 'mac-mac10.10',
'specifiers': ['Mac10.10', 'Release']
},
})
# Note: this is a pre-assumption of the tests in this file. If this
# assertion fails, port configurations are likely changed, and the
# tests need to be adjusted accordingly.
self.assertEqual(sorted(self.host.port_factory.all_port_names()), [
'linux-trusty', 'mac-mac10.10', 'mac-mac10.11', 'mac-mac10.12',
'mac-mac10.13', 'mac-mac10.14', 'mac-mac10.15', 'win-win10'
])
def setUp(self):
self.host = MockHost()
self.fs = MockFileSystem()
self.host.filesystem = self.fs
self.fs.write_text_file('/all-pass/foo-expected.txt',
ALL_PASS_TESTHARNESS_RESULT)
self.fs.write_text_file('/all-pass/bar-expected.txt',
ALL_PASS_TESTHARNESS_RESULT2)
self.fs.write_text_file('/failures/baz-expected.txt', 'failure')
self.fs.write_binary_file('/others/reftest-expected.png', 'extra')
self.fs.write_binary_file('/others/reftest2-expected.png', 'extra2')
self.fs.write_text_file('/others/empty-expected.txt', '')
self.fs.write_binary_file('/others/something-expected.png',
'Something')
self.fs.write_binary_file('/others/empty-expected.png', '')
def test(self):
mock_filesystem = MockFileSystem({'/s1/file1': 'a', '/s2/file1': 'b', '/s3/file1': 'c'}, dirs=['/output'])
merger = merge_results.MergeFilesKeepFiles(mock_filesystem)
with self.assertFilesAdded(mock_filesystem, {'/output/out_0': 'a', '/output/out_1': 'b', '/output/out_2': 'c'}):
merger('/output/out', ['/s1/file1', '/s2/file1', '/s3/file1'])
def test(self):
mock_filesystem = MockFileSystem(
{
'/s/file1': 'A\nC\n',
'/s/file2': 'B\n',
'/s/file3': 'A\nB\n'
},
dirs=['/output'])
merger = merge_results.MergeFilesLinesSorted(mock_filesystem)
with self.assertFilesAdded(mock_filesystem,
{'/output/out1': 'A\nC\n'}):
merger('/output/out1', ['/s/file1'])
with self.assertFilesAdded(mock_filesystem, {'/output/out2': 'B\n'}):
merger('/output/out2', ['/s/file2'])
with self.assertFilesAdded(mock_filesystem,
{'/output/out3': 'A\nB\nC\n'}):
merger('/output/out3', ['/s/file1', '/s/file2'])
with self.assertFilesAdded(mock_filesystem,
{'/output/out4': 'A\nB\nB\n'}):
merger('/output/out4', ['/s/file2', '/s/file3'])
def test(self):
fs = MockFileSystem(self.web_test_filesystem)
merger = merge_results.WebTestDirMerger(
fs, results_json_value_overrides={'layout_tests_dir': 'src'})
merger.merge('/out', ['/shards/0', '/shards/1'])
for fname, expected_contents in self.web_test_output_filesystem.items(
):
self.assertIn(fname, fs.files)
if fname.endswith(".json"):
actual_json_str = fs.files[fname]
expected_json_str = expected_contents
if "failing_results" in fname:
self.assertTrue(
MergeFilesJSONPTests.check_before_after(
fs.files[fname], b'ADD_RESULTS(', b");"))
self.assertTrue(
MergeFilesJSONPTests.check_before_after(
expected_contents, 'ADD_RESULTS(', ");"))
actual_json_str = MergeFilesJSONPTests.remove_before_after(
fs.files[fname], b'ADD_RESULTS(', b");")
expected_json_str = MergeFilesJSONPTests.remove_before_after(
expected_contents, b'ADD_RESULTS(', b");")
self.assertEqual(json.loads(actual_json_str),
json.loads(expected_json_str))
else:
self.assertMultiLineEqual(expected_contents, fs.files[fname])
def test_same_metadata_file_for_variants(self):
"""Variants of a test all go in the same metadata file."""
test_name1 = "external/wpt/variant.html?foo=bar/abc"
test_name2 = "external/wpt/variant.html?foo=baz"
expectation_dict = OrderedDict()
_append_to_expectation_dict(expectation_dict, "TestExpectations",
test_name1, "FAILURE")
_append_to_expectation_dict(expectation_dict, "TestExpectations",
test_name2, "TIMEOUT")
expectations = _make_expectation_with_dict(self.port, expectation_dict)
metadata_builder = WPTMetadataBuilder(expectations, self.port)
metadata_builder.metadata_output_dir = "out"
metadata_builder.fs = MockFileSystem()
metadata_builder._build_metadata_and_write()
# Both the tests go into the same metadata file, named without any
# variants.
metadata_file = os.path.join("out", "variant.html.ini")
# Inside the metadata file, we have separate entries for each variant
self.assertEqual(
"[variant.html?foo=baz]\n blink_expect_any_subtest_status: True # wpt_metadata_builder.py\n"
" expected: [TIMEOUT]\n"
"[variant.html?foo=bar/abc]\n blink_expect_any_subtest_status: True # wpt_metadata_builder.py\n"
" expected: [FAIL, ERROR]\n",
metadata_builder.fs.read_text_file(metadata_file))
def test_ref_test_with_ref_is_copied(self):
host = MockHost()
host.filesystem = MockFileSystem(
files={
'/blink/w3c/dir1/my-ref-test.html':
'<html><head><link rel="match" href="ref-file.html" />test</head></html>',
'/blink/w3c/dir1/ref-file.html':
'<html><head>test</head></html>',
MOCK_WEB_TESTS + 'W3CImportExpectations':
'',
})
copier = TestCopier(host, FAKE_SOURCE_REPO_DIR)
copier.find_importable_tests()
self.assertEqual(copier.import_list, [{
'copy_list': [{
'src': '/blink/w3c/dir1/ref-file.html',
'dest': 'ref-file.html'
},
{
'src': '/blink/w3c/dir1/my-ref-test.html',
'dest': 'my-ref-test.html'
}],
'dirname':
'/blink/w3c/dir1',
}])
def test_harness_no_expectations(self):
"""Tests behavior when TestExpectations file doesn't exist.
Tests that a warning is outputted if the TestExpectations file
doesn't exist.
"""
# Set up the mock host and port.
host = MockHost()
host.port_factory = FakePortFactory(host)
# Write the test file but not the TestExpectations file.
test_expectation_path = (
host.port_factory.get().path_to_generic_test_expectations_file())
host.filesystem = MockFileSystem()
self._write_tests_into_filesystem(host.filesystem)
# Write out the fake builder bot results.
expectation_factory = FakeBotTestExpectationsFactory()
expectation_factory.all_results_by_builder = {}
self.assertFalse(host.filesystem.isfile(test_expectation_path))
return_code = main(host, expectation_factory, [])
self.assertEqual(return_code, 1)
self.assertLog([
"WARNING: Didn't find generic expectations file at: %s\n" % test_expectation_path
])
self.assertFalse(host.filesystem.isfile(test_expectation_path))
def test_ref_test_with_ref_is_copied(self):
host = MockHost()
host.filesystem = MockFileSystem(
files={
'/blink/w3c/dir1/my-ref-test.html':
b'<html><head><link rel="match" href="ref-file.html" />test</head></html>',
'/blink/w3c/dir1/ref-file.html':
b'<html><head>test</head></html>',
MOCK_WEB_TESTS + 'W3CImportExpectations': b'',
})
copier = TestCopier(host, FAKE_SOURCE_REPO_DIR)
copier.find_importable_tests()
self.assertEqual(len(copier.import_list), 1)
# The order of copy_list depends on the implementation of
# filesystem.walk, so don't check the order
if six.PY3:
self.assertCountEqual(
copier.import_list[0]['copy_list'], [{
'src': '/blink/w3c/dir1/ref-file.html',
'dest': 'ref-file.html'
}, {
'src': '/blink/w3c/dir1/my-ref-test.html',
'dest': 'my-ref-test.html'
}])
else:
self.assertItemsEqual(
copier.import_list[0]['copy_list'], [{
'src': '/blink/w3c/dir1/ref-file.html',
'dest': 'ref-file.html'
}, {
'src': '/blink/w3c/dir1/my-ref-test.html',
'dest': 'my-ref-test.html'
}])
self.assertEqual(copier.import_list[0]['dirname'], '/blink/w3c/dir1')
def test_expectation_overwrites_checked_in_metadata(self):
"""Test an entry in an expectation overwriting checked-in metadata.
When an expectation has no annotation to use checked-in metadata then
the expectation will overwrite any checked-in metadata."""
test_name = "external/wpt/test.html"
expectations = _make_expectation(self.port, test_name, "TIMEOUT")
mb = WPTMetadataBuilder(expectations, self.port)
# Set the metadata builder to use mock filesystem populated with some
# test data
mb.checked_in_metadata_dir = "src"
mb.metadata_output_dir = "out"
mock_checked_in_files = {
"src/external/wpt/test.html": "",
"src/external/wpt/test.html.ini": "checked-in metadata",
}
mb.fs = MockFileSystem(files=mock_checked_in_files)
mb._build_metadata_and_write()
# Ensure that the data written to the metadata file is the translated
# status, not the checked-in contents.
resulting_ini_file = os.path.join("out", "test.html.ini")
self.assertEqual(
"[test.html]\n blink_expect_any_subtest_status: True # wpt_metadata_builder.py\n expected: [TIMEOUT]\n",
mb.fs.read_text_file(resulting_ini_file))
def assertMergeRaises(self, mock_filesystem_contents, inputargs):
mock_filesystem = MockFileSystem(mock_filesystem_contents,
dirs=['/output'])
file_merger = merge_results.MergeFilesJSONP(mock_filesystem)
with self.assertRaises(merge_results.MergeFailure):
file_merger(*inputargs)
def test_success_same_file_but_matching_contents(self):
mock_filesystem = MockFileSystem({
'/shard0/file1': '1',
'/shard1/file1': '1'
})
d = merge_results.DirMerger(mock_filesystem)
with self.assertFilesAdded(mock_filesystem, {'/output/file1': '1'}):
d.merge('/output', ['/shard0', '/shard1'])
def make_info(self,
sys_module=None,
platform_module=None,
filesystem_module=None,
executive=None):
return PlatformInfo(sys_module or fake_sys(), platform_module
or fake_platform(), filesystem_module
or MockFileSystem(), executive or fake_executive())
def test_files_with_shebang_are_made_executable(self):
host = MockHost()
host.filesystem = MockFileSystem(files=FAKE_FILES)
copier = TestCopier(host, FAKE_SOURCE_REPO_DIR)
copier.do_import()
self.assertEqual(
host.filesystem.executable_files,
set([MOCK_WEB_TESTS + 'external/blink/w3c/dir/has_shebang.txt']))
def test_strip_web_tests_path(self):
finder = PathFinder(MockFileSystem())
path_with_web_tests = '/mock-checkout/' + RELATIVE_WEB_TESTS + 'external/wpt'
self.assertEqual(finder.strip_web_tests_path(path_with_web_tests),
'external/wpt')
path_without_web_tests = '/checkout/' + RELATIVE_WEB_TESTS + 'external/wpt'
self.assertEqual(finder.strip_web_tests_path(path_without_web_tests),
path_without_web_tests)
def test_filesystem_walk(self):
mock_dir = 'foo'
mock_files = {'foo/bar/baz': '', 'foo/a': '', 'foo/b': '', 'foo/c': ''}
host = MockHost()
host.filesystem = MockFileSystem(files=mock_files)
self.assertEquals(host.filesystem.walk(mock_dir),
[('foo', ['bar'], ['a', 'b', 'c']),
('foo/bar', [], ['baz'])])
def test_failure_same_file_but_contents_differ(self):
mock_filesystem = MockFileSystem({
'/shard0/file1': '1',
'/shard1/file1': '2'
})
d = merge_results.DirMerger(mock_filesystem)
with self.assertRaises(merge_results.MergeFailure):
d.merge('/output', ['/shard0', '/shard1'])
def test_mark_missing_shards(self):
fs = MockFileSystem(self.web_test_filesystem)
merge_results.mark_missing_shards(
'/swarm/summary.json',
['/0'], #only dir paths
'/out/output.json',
fs)
final_merged_output_json = fs.files['/out/output.json']
self.assertEqual(final_merged_output_json, self.final_output_json)
def test(self):
fs = MockFileSystem(self.layout_test_filesystem)
merger = merge_results.LayoutTestDirMerger(fs, results_json_value_overrides={'layout_tests_dir': 'src'})
merger.merge('/out', ['/shards/0', '/shards/1'])
for fname, contents in self.layout_test_output_filesystem.items():
self.assertIn(fname, fs.files)
self.assertMultiLineEqual(contents, fs.files[fname])
def setUp(self):
self.host = MockHost()
# Mock a virtual test suite at virtual/gpu/external/wpt/foo.
self.host.filesystem = MockFileSystem({
MOCK_WEB_TESTS + 'VirtualTestSuites':
'[{"prefix": "gpu", "bases": ["external/wpt/foo"], "args": ["--foo"]}]'
})
self.git = self.host.git()
self.local_wpt = MockLocalWPT()
self.notifier = ImportNotifier(self.host, self.git, self.local_wpt)
def setUp(self):
self.host = MockHost()
# Mock a virtual test suite at virtual/gpu/external/wpt/foo.
self.host.filesystem = MockFileSystem({
'/mock-checkout/third_party/WebKit/LayoutTests/VirtualTestSuites':
'[{"prefix": "gpu", "base": "external/wpt/foo", "args": ["--foo"]}]'
})
self.git = self.host.git()
self.local_wpt = MockLocalWPT()
self.notifier = ImportNotifier(self.host, self.git, self.local_wpt)
def test_strip_webdriver_tests_path(self):
finder = PathFinder(MockFileSystem())
path_with_webdriver_prefix = 'external/wpt/webdriver/' + 'foo/bar.py>>test'
self.assertEqual(
finder.strip_webdriver_tests_path(path_with_webdriver_prefix),
'foo/bar.py>>test')
path_without_webdriver_prefix = 'external/wpt' + 'bar/foo.py>>test'
self.assertEqual(
finder.strip_webdriver_tests_path(path_without_webdriver_prefix),
path_without_webdriver_prefix)
def test_files_with_shebang_are_made_executable(self):
host = MockHost()
host.filesystem = MockFileSystem(files=FAKE_FILES)
copier = TestCopier(host, FAKE_SOURCE_REPO_DIR)
copier.do_import()
self.assertEqual(
host.filesystem.executable_files,
set([
'/mock-checkout/third_party/WebKit/LayoutTests/external/blink/w3c/dir/has_shebang.txt'
]))
