class RealFileSystemTest(unittest.TestCase, GenericFileSystemTests):
def setUp(self):
self.fs = FileSystem()
self.setup_generic_test_dir()
self._this_dir = os.path.dirname(os.path.abspath(__file__))
self._missing_file = os.path.join(self._this_dir, 'missing_file.py')
self._this_file = os.path.join(self._this_dir,
'filesystem_unittest.py')
def tearDown(self):
self.teardown_generic_test_dir()
self.fs = None
def test_chdir(self):
fs = FileSystem()
cwd = fs.getcwd()
newdir = '/'
if sys.platform == 'win32':
newdir = 'c:\\'
fs.chdir(newdir)
self.assertEqual(fs.getcwd(), newdir)
fs.chdir(cwd)
def test_chdir__notexists(self):
fs = FileSystem()
newdir = '/dirdoesnotexist'
if sys.platform == 'win32':
newdir = 'c:\\dirdoesnotexist'
with self.assertRaises(OSError):
fs.chdir(newdir)
def test_exists__true(self):
fs = FileSystem()
self.assertTrue(fs.exists(self._this_file))
def test_exists__false(self):
fs = FileSystem()
self.assertFalse(fs.exists(self._missing_file))
def test_getcwd(self):
fs = FileSystem()
self.assertTrue(fs.exists(fs.getcwd()))
def test_isdir__true(self):
fs = FileSystem()
self.assertTrue(fs.isdir(self._this_dir))
def test_isdir__false(self):
fs = FileSystem()
self.assertFalse(fs.isdir(self._this_file))
def test_join(self):
fs = FileSystem()
self.assertEqual(fs.join('foo', 'bar'), os.path.join('foo', 'bar'))
def test_listdir(self):
fs = FileSystem()
with fs.mkdtemp(prefix='filesystem_unittest_') as d:
self.assertEqual(fs.listdir(d), [])
new_file = os.path.join(d, 'foo')
fs.write_text_file(new_file, u'foo')
self.assertEqual(fs.listdir(d), ['foo'])
os.remove(new_file)
def test_walk(self):
fs = FileSystem()
with fs.mkdtemp(prefix='filesystem_unittest_') as d:
self.assertEqual(list(fs.walk(d)), [(d, [], [])])
new_file = os.path.join(d, 'foo')
fs.write_text_file(new_file, u'foo')
self.assertEqual(list(fs.walk(d)), [(d, [], ['foo'])])
os.remove(new_file)
def test_maybe_make_directory__success(self):
fs = FileSystem()
with fs.mkdtemp(prefix='filesystem_unittest_') as base_path:
sub_path = os.path.join(base_path, 'newdir')
self.assertFalse(os.path.exists(sub_path))
self.assertFalse(fs.isdir(sub_path))
fs.maybe_make_directory(sub_path)
self.assertTrue(os.path.exists(sub_path))
self.assertTrue(fs.isdir(sub_path))
# Make sure we can re-create it.
fs.maybe_make_directory(sub_path)
self.assertTrue(os.path.exists(sub_path))
self.assertTrue(fs.isdir(sub_path))
# Clean up.
os.rmdir(sub_path)
self.assertFalse(os.path.exists(base_path))
self.assertFalse(fs.isdir(base_path))
def test_maybe_make_directory__failure(self):
# FIXME: os.chmod() doesn't work on Windows to set directories
# as readonly, so we skip this test for now.
if sys.platform == 'win32':
return
fs = FileSystem()
with fs.mkdtemp(prefix='filesystem_unittest_') as d:
# Remove write permissions on the parent directory.
os.chmod(d, stat.S_IRUSR)
# Now try to create a sub directory - should fail.
sub_dir = fs.join(d, 'subdir')
with self.assertRaises(OSError):
fs.maybe_make_directory(sub_dir)
# Clean up in case the test failed and we did create the
# directory.
if os.path.exists(sub_dir):
os.rmdir(sub_dir)
def test_read_and_write_text_file(self):
fs = FileSystem()
text_path = None
unicode_text_string = u'\u016An\u012Dc\u014Dde\u033D'
try:
text_path = tempfile.mktemp(prefix='tree_unittest_')
file = fs.open_text_file_for_writing(text_path)
file.write(unicode_text_string)
file.close()
file = fs.open_text_file_for_reading(text_path)
read_text = file.read()
file.close()
self.assertEqual(read_text, unicode_text_string)
finally:
if text_path and fs.isfile(text_path):
os.remove(text_path)
def test_read_and_write_file(self):
fs = FileSystem()
text_path = None
binary_path = None
unicode_text_string = u'\u016An\u012Dc\u014Dde\u033D'
hex_equivalent = '\xC5\xAA\x6E\xC4\xAD\x63\xC5\x8D\x64\x65\xCC\xBD'
try:
text_path = tempfile.mktemp(prefix='tree_unittest_')
binary_path = tempfile.mktemp(prefix='tree_unittest_')
fs.write_text_file(text_path, unicode_text_string)
contents = fs.read_binary_file(text_path)
self.assertEqual(contents, hex_equivalent)
fs.write_binary_file(binary_path, hex_equivalent)
text_contents = fs.read_text_file(binary_path)
self.assertEqual(text_contents, unicode_text_string)
finally:
if text_path and fs.isfile(text_path):
os.remove(text_path)
if binary_path and fs.isfile(binary_path):
os.remove(binary_path)
def test_read_binary_file__missing(self):
fs = FileSystem()
with self.assertRaises(IOError):
fs.read_binary_file(self._missing_file)
def test_read_text_file__missing(self):
fs = FileSystem()
with self.assertRaises(IOError):
fs.read_text_file(self._missing_file)
def test_remove_file_with_retry(self):
RealFileSystemTest._remove_failures = 2
def remove_with_exception(filename):
RealFileSystemTest._remove_failures -= 1
if RealFileSystemTest._remove_failures >= 0:
try:
raise WindowsError
except NameError:
raise FileSystem._WindowsError
fs = FileSystem()
self.assertTrue(fs.remove('filename', remove_with_exception))
self.assertEqual(-1, RealFileSystemTest._remove_failures)
def test_sep(self):
fs = FileSystem()
self.assertEqual(fs.sep, os.sep)
self.assertEqual(fs.join('foo', 'bar'), os.path.join('foo', 'bar'))
def test_long_paths(self):
# This mostly tests UNC paths on Windows for path names > 260 chars.
# Currently, only makedirs, copyfile, and various open methods are
# verified to support UNC paths.
long_path = self.fs.join(self.generic_test_dir, 'x' * 100, 'y' * 100,
'z' * 100)
self.fs.maybe_make_directory(long_path)
file1 = self.fs.join(long_path, 'foo')
file2 = self.fs.join(long_path, 'bar')
self.fs.write_text_file(file1, 'hello')
self.fs.copyfile(file1, file2)
content = self.fs.read_text_file(file2)
self.fs.remove(file2) # No exception should be raised.
self.assertEqual(content, 'hello')
# (long_path is left in the filesystem and its removal is tested during cleanup.)
# On Windows, rmtree can handle trees containing long paths as long as
# the root is not a long path.
long_path1 = self.fs.join(self.generic_test_dir, 'a' * 100,
'b' * 100 + " 'b")
long_path2 = self.fs.join(long_path1, 'c' * 100)
self.fs.maybe_make_directory(long_path2)
file1 = self.fs.join(long_path2, 'foo')
self.fs.write_text_file(file1, 'hello')
if sys.platform == 'win32':
with self.assertRaises(AssertionError):
self.fs.rmtree(long_path2, ignore_errors=False)
else:
self.fs.rmtree(long_path2, ignore_errors=False)
self.fs.rmtree(long_path1, ignore_errors=False)
class WPTMetadataBuilder(object):
def __init__(self, expectations, port):
"""
Args:
expectations: a blinkpy.web_tests.models.test_expectations.TestExpectations object
port: a blinkpy.web_tests.port.Port object
"""
self.expectations = expectations
self.port = port
# TODO(lpz): Use self.fs everywhere in this class and add tests
self.fs = FileSystem()
self.wpt_manifest = self.port.wpt_manifest("external/wpt")
self.metadata_output_dir = ""
self.checked_in_metadata_dir = ""
self.process_baselines = True
self.handle_annotations = True
def run(self, args=None):
"""Main entry point to parse flags and execute the script."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--metadata-output-dir",
help="The directory to output the metadata files into.")
parser.add_argument(
"--checked-in-metadata-dir",
help="Root directory of any checked-in WPT metadata files to use. "
"If set, these files will take precedence over legacy expectations "
"and baselines when both exist for a test.")
parser.add_argument(
'-v',
'--verbose',
action='store_true',
help='More verbose logging.')
parser.add_argument(
"--process-baselines",
action="store_true",
default=True,
dest="process_baselines",
help="Whether to translate baseline (-expected.txt) files into WPT "
"metadata files. This translation is lossy and results in any "
"subtest being accepted by wptrunner.")
parser.add_argument("--no-process-baselines",
action="store_false",
dest="process_baselines")
parser.add_argument(
"--handle-annotations",
action="store_true",
default=True,
dest="handle_annotations",
help="Whether to handle annotations in expectations files. These "
"are trailing comments that give additional details for how "
"to translate an expectation into WPT metadata.")
parser.add_argument("--no-handle-annotations",
action="store_false",
dest="handle_annotations")
args = parser.parse_args(args)
log_level = logging.DEBUG if args.verbose else logging.INFO
configure_logging(logging_level=log_level, include_time=True)
self.metadata_output_dir = args.metadata_output_dir
self.checked_in_metadata_dir = args.checked_in_metadata_dir
self.process_baselines = args.process_baselines
self.handle_annotations = args.handle_annotations
self._build_metadata_and_write()
return 0
@staticmethod
def status_bitmap_to_string(test_status_bitmap):
statuses = []
result = ""
if test_status_bitmap & SUBTEST_FAIL:
result += " blink_expect_any_subtest_status: True # wpt_metadata_builder.py\n"
if test_status_bitmap & HARNESS_ERROR:
statuses.append("ERROR")
if test_status_bitmap & TEST_PASS:
# We need both PASS and OK. Reftests will PASS while testharness
# tests are OK.
statuses.append("PASS")
statuses.append("OK")
if test_status_bitmap & TEST_FAIL:
# We need both FAIL and ERROR. Reftests will FAIL while testharness
# tests have ERRORs.
statuses.append("FAIL")
statuses.append("ERROR")
if test_status_bitmap & TEST_TIMEOUT:
statuses.append("TIMEOUT")
if test_status_bitmap & TEST_CRASH:
statuses.append("CRASH")
if test_status_bitmap & TEST_PRECONDITION_FAILED:
statuses.append("PRECONDITION_FAILED")
if statuses:
result += " expected: [%s]\n" % ", ".join(statuses)
return result
def _build_metadata_and_write(self):
"""Build the metadata files and write them to disk."""
if os.path.exists(self.metadata_output_dir):
_log.debug("Output dir exists, deleting: %s",
self.metadata_output_dir)
import shutil
shutil.rmtree(self.metadata_output_dir)
tests_for_metadata = self.get_tests_needing_metadata()
_log.info("Found %d tests requiring metadata", len(tests_for_metadata))
for test_name, test_status_bitmap in tests_for_metadata.items():
filename, file_contents = self.get_metadata_filename_and_contents(
test_name, test_status_bitmap)
if not filename or not file_contents:
continue
self._write_to_file(filename, file_contents)
if self.checked_in_metadata_dir and os.path.exists(
self.checked_in_metadata_dir):
_log.info("Copying checked-in WPT metadata on top of translated "
"files.")
self._copy_checked_in_metadata()
else:
_log.warning("Not using checked-in WPT metadata, path is empty or "
"does not exist: %s" % self.checked_in_metadata_dir)
# Finally, output a stamp file with the same name as the output
# directory. The stamp file is empty, it's only used for its mtime.
# This makes the GN build system happy (see crbug.com/995112).
with open(self.metadata_output_dir + ".stamp", "w"):
pass
def _copy_checked_in_metadata(self):
"""Copies checked-in metadata files to the metadata output directory."""
for filename in self.fs.files_under(self.checked_in_metadata_dir):
# We match any .ini files in the path. This will find .ini files
# other than just metadata (such as tox.ini), but that is ok
# since wptrunner will just ignore those.
if not fnmatch.fnmatch(filename, "*.ini"):
continue
# Found a checked-in .ini file. Copy it to the metadata output
# directory in the same sub-path as where it is checked in.
# So /checked/in/a/b/c.ini goes to /metadata/out/a/b/c.ini
output_path = filename.replace(self.checked_in_metadata_dir,
self.metadata_output_dir)
if not self.fs.exists(self.fs.dirname(output_path)):
self.fs.maybe_make_directory(self.fs.dirname(output_path))
_log.debug("Copying %s to %s" % (filename, output_path))
self.fs.copyfile(filename, output_path)
def _write_to_file(self, filename, file_contents):
# Write the contents to the file name
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
# Note that we append to the metadata file in order to allow multiple
# tests to be present in the same .ini file (ie: for multi-global tests)
with open(filename, "a") as metadata_file:
metadata_file.write(file_contents)
def get_tests_needing_metadata(self):
"""Determines which tests need metadata files.
This function loops over the tests to be run and checks whether each test
has an expectation (eg: in TestExpectations) and/or a baseline (ie:
test-name-expected.txt). The existence of those things will determine
the information that will be emitted into the tests's metadata file.
Returns:
A dict. The key is the string test name and the value is an integer
bitmap of statuses for the test.
"""
tests_needing_metadata = defaultdict(int)
for test_name in self.port.tests(paths=["external/wpt"]):
# First check for expectations. If a test is skipped then we do not
# look for more statuses
expectation_line = self.expectations.get_expectations(test_name)
self._handle_test_with_expectation(test_name, expectation_line,
tests_needing_metadata)
if self._test_was_skipped(test_name, tests_needing_metadata):
# Do not consider other statuses if a test is skipped
continue
# Check if the test has a baseline
if self.process_baselines:
test_baseline = self.port.expected_text(test_name)
if not test_baseline:
continue
self._handle_test_with_baseline(test_name, test_baseline,
tests_needing_metadata)
return tests_needing_metadata
def _handle_test_with_expectation(self, test_name, expectation_line,
status_dict):
"""Handles a single test expectation and updates |status_dict|."""
test_statuses = expectation_line.results
annotations = expectation_line.trailing_comments
if ResultType.Skip in test_statuses:
# Skips are handled alone, so don't look at any other statuses
status_dict[test_name] |= SKIP_TEST
return
# Guard against the only test_status being Pass (without any
# annotations), we don't want to create metadata for such a test.
if (len(test_statuses) == 1 and ResultType.Pass in test_statuses
and not annotations):
return
status_bitmap = 0
if ResultType.Pass in test_statuses:
status_bitmap |= TEST_PASS
if ResultType.Failure in test_statuses:
status_bitmap |= TEST_FAIL
if ResultType.Timeout in test_statuses:
status_bitmap |= TEST_TIMEOUT
if ResultType.Crash in test_statuses:
status_bitmap |= TEST_CRASH
if self.handle_annotations and annotations:
if "wpt_subtest_failure" in annotations:
status_bitmap |= SUBTEST_FAIL
if "wpt_precondition_failed" in annotations:
status_bitmap |= TEST_PRECONDITION_FAILED
# Update status bitmap for this test
status_dict[test_name] |= status_bitmap
def _test_was_skipped(self, test_name, status_dict):
"""Returns whether |test_name| is marked as skipped in |status_dict|."""
return test_name in status_dict and (
status_dict[test_name] & SKIP_TEST)
def _handle_test_with_baseline(self, test_name, test_baseline,
status_dict):
"""Handles a single test baseline and updates |status_dict|."""
status_bitmap = 0
if re.search(r"^(FAIL|NOTRUN|TIMEOUT)", test_baseline, re.MULTILINE):
status_bitmap |= SUBTEST_FAIL
if re.search(r"^Harness Error\.", test_baseline, re.MULTILINE):
status_bitmap |= HARNESS_ERROR
if status_bitmap > 0:
status_dict[test_name] |= status_bitmap
else:
# Treat this as an error because we don't want it to happen.
# Either the non-FAIL statuses need to be handled here, or the
# baseline is all PASS which should just be deleted.
_log.error("Test %s has a non-FAIL baseline" % test_name)
def _metadata_filename_from_test_file(self, wpt_test_file):
"""Returns the filename of the metadata (.ini) file for the test.
Args:
wpt_test_file: The file on disk that the specified test lives in.
For multi-global tests this is usually a ".js" file.
Returns:
The fully-qualified string path of the metadata file for this test.
"""
assert "?" not in wpt_test_file
test_file_parts = wpt_test_file.split("/")
return os.path.join(self.metadata_output_dir,
*test_file_parts) + ".ini"
def _metadata_inline_test_name_from_test_name(self, wpt_test_name):
"""Returns the test name to use *inside* of a metadata file.
The inline name inside the metadata file is the logical name of the
test without any subdirectories.
For multi-global tests this means that it must have the specific scope
of the test (eg: worker, window, etc). This name must also include any
variants that are set.
Args:
wpt_test_name: The fully-qualified test name which contains all
subdirectories as well as scope (for multi-globals), and
variants.
Returns:
The string test name inside of the metadata file.
"""
# To generate the inline test name we basically want to strip away the
# subdirectories from the test name, being careful not to accidentally
# clobber the variant.
variant_split = wpt_test_name.split("?")
test_path = variant_split[0]
test_name_part = test_path.split("/")[-1]
variant = "?" + variant_split[1] if len(variant_split) == 2 else ""
return test_name_part + variant
def get_metadata_filename_and_contents(self,
chromium_test_name,
test_status_bitmap=0):
"""Determines the metadata filename and contents for the specified test.
The metadata filename is derived from the test name but will differ if
the expectation is for a single test or for a directory of tests. The
contents of the metadata file will also differ for those two cases.
Args:
chromium_test_name: A Chromium test name from the expectation file,
which starts with `external/wpt`.
test_status_bitmap: An integer containing additional data about the
status, such as enumerating flaky statuses, or whether a test has
a combination of harness error and subtest failure.
Returns:
A pair of strings, the first is the path to the metadata file and
the second is the contents to write to that file. Or None if the
test does not need a metadata file.
"""
# Ignore expectations for non-WPT tests
if (not chromium_test_name
or not chromium_test_name.startswith('external/wpt')):
return None, None
# Split the test name by directory. We omit the first 2 entries because
# they are 'external' and 'wpt' and these don't exist in the WPT's test
# names.
wpt_test_name_parts = chromium_test_name.split("/")[2:]
# The WPT test name differs from the Chromium test name in that the WPT
# name omits `external/wpt`.
wpt_test_name = "/".join(wpt_test_name_parts)
# Check if this is a test file or a test directory
is_test_dir = chromium_test_name.endswith("/")
metadata_filename = None
metadata_file_contents = None
if is_test_dir:
# A test directory gets one metadata file called __dir__.ini and all
# tests in that dir are skipped.
metadata_filename = os.path.join(self.metadata_output_dir,
*wpt_test_name_parts)
metadata_filename = os.path.join(metadata_filename, "__dir__.ini")
_log.debug("Creating a dir-wide ini file %s", metadata_filename)
metadata_file_contents = self._get_dir_disabled_string()
else:
# For individual tests, we create one file per test, with the name
# of the test in the file as well.
test_file_path = self.wpt_manifest.file_path_for_test_url(
wpt_test_name)
if not test_file_path:
_log.info("Could not find file for test %s, skipping" %
wpt_test_name)
return None, None
metadata_filename = self._metadata_filename_from_test_file(
test_file_path)
_log.debug("Creating a test ini file %s with status_bitmap %s",
metadata_filename, test_status_bitmap)
inline_test_name = self._metadata_inline_test_name_from_test_name(
wpt_test_name)
metadata_file_contents = self._get_test_failed_string(
inline_test_name, test_status_bitmap)
return metadata_filename, metadata_file_contents
def _get_dir_disabled_string(self):
return "disabled: wpt_metadata_builder.py\n"
def _get_test_disabled_string(self, test_name):
return "[%s]\n disabled: wpt_metadata_builder.py\n" % test_name
def _get_test_failed_string(self, inline_test_name, test_status_bitmap):
# The contents of the metadata file is two lines:
# 1. the inline name of the WPT test pathinside square brackets. This
# name contains the test scope (for multi-globals) and variants.
# 2. an indented line with the test status and reason
result = "[%s]\n" % inline_test_name
# A skipped test is a little special in that it doesn't happen along with
# any other status. So we compare directly against SKIP_TEST and also
# return right away.
if test_status_bitmap == SKIP_TEST:
result += " disabled: wpt_metadata_builder.py\n"
return result
# Other test statuses can exist together. But ensure we have at least one.
expected_string = self.status_bitmap_to_string(test_status_bitmap)
if expected_string:
result += expected_string
return result
class BaseWptScriptAdapter(common.BaseIsolatedScriptArgsAdapter):
"""The base class for script adapters that use wptrunner to execute web
platform tests. This contains any code shared between these scripts, such
as integrating output with the results viewer. Subclasses contain other
(usually platform-specific) logic."""
def __init__(self):
super(BaseWptScriptAdapter, self).__init__()
self.fs = FileSystem()
host = Host()
self.port = host.port_factory.get()
self.wpt_manifest = self.port.wpt_manifest("external/wpt")
def generate_test_output_args(self, output):
return ['--log-chromium', output]
def generate_sharding_args(self, total_shards, shard_index):
return ['--total-chunks=%d' % total_shards,
# shard_index is 0-based but WPT's this-chunk to be 1-based
'--this-chunk=%d' % (shard_index + 1),
# The default sharding strategy is to shard by directory. But
# we want to hash each test to determine which shard runs it.
# This allows running individual directories that have few
# tests across many shards.
'--chunk-type=hash']
def do_post_test_run_tasks(self):
# Move json results into layout-test-results directory
results_dir = os.path.dirname(self.options.isolated_script_test_output)
layout_test_results = os.path.join(results_dir, 'layout-test-results')
if os.path.exists(layout_test_results):
self.fs.rmtree(layout_test_results)
self.fs.maybe_make_directory(layout_test_results)
# Perform post-processing of wptrunner output
self.process_wptrunner_output()
self.fs.copyfile(self.options.isolated_script_test_output,
os.path.join(layout_test_results, 'full_results.json'))
# create full_results_jsonp.js file which is used to
# load results into the results viewer
self.fs.write_text_file(
os.path.join(layout_test_results, 'full_results_jsonp.js'),
'ADD_FULL_RESULTS(%s);' % self.fs.read_text_file(
self.options.isolated_script_test_output))
# copy layout test results viewer to layout-test-results directory
self.fs.copyfile(
os.path.join(WEB_TESTS_DIR, 'fast', 'harness', 'results.html'),
os.path.join(layout_test_results, 'results.html'))
def process_wptrunner_output(self):
"""Post-process the output generated by wptrunner.
This output contains a single large json file containing the raw content
or artifacts which need to be extracted into their own files and removed
from the json file (to avoid duplication)."""
output_json = json.loads(
self.fs.read_text_file(self.options.isolated_script_test_output))
test_json = output_json["tests"]
results_dir = os.path.dirname(self.options.isolated_script_test_output)
self._process_test_leaves(results_dir, output_json["path_delimiter"],
test_json, "")
# Write output_json back to the same file after modifying it in memory
self.fs.write_text_file(self.options.isolated_script_test_output,
json.dumps(output_json))
def _process_test_leaves(self, results_dir, delim, root_node, path_so_far):
"""Finds and processes each test leaf below the specified root.
This will recursively traverse the trie of results in the json output,
keeping track of the path to each test and identifying leaves by the
presence of certain attributes.
Args:
results_dir: str path to the dir that results are stored
delim: str delimiter to be used for test names
root_node: dict representing the root of the trie we're currently
looking at
path_so_far: str the path to the current root_node in the trie
"""
if "actual" in root_node:
# Found a leaf, process it
if "artifacts" not in root_node:
return
log_artifact = root_node["artifacts"].pop("log", None)
if log_artifact:
artifact_subpath = self._write_log_artifact(
test_failures.FILENAME_SUFFIX_ACTUAL,
results_dir, path_so_far, log_artifact)
root_node["artifacts"]["actual_text"] = [artifact_subpath]
# Try to locate the expected output of this test, if it exists.
expected_subpath = self._maybe_write_expected_output(
results_dir, path_so_far)
if expected_subpath:
root_node["artifacts"]["expected_text"] = [expected_subpath]
screenshot_artifact = root_node["artifacts"].pop("screenshots",
None)
if screenshot_artifact:
screenshot_paths_dict = self._write_screenshot_artifact(
results_dir, path_so_far, screenshot_artifact)
for screenshot_key, path in screenshot_paths_dict.items():
root_node["artifacts"][screenshot_key] = [path]
crashlog_artifact = root_node["artifacts"].pop("wpt_crash_log",
None)
if crashlog_artifact:
artifact_subpath = self._write_log_artifact(
test_failures.FILENAME_SUFFIX_CRASH_LOG,
results_dir, path_so_far, crashlog_artifact)
return
# We're not at a leaf node, continue traversing the trie.
for key in root_node:
# Append the key to the path, separated by the delimiter. However if
# the path is empty, skip the delimiter to avoid a leading slash in
# the path.
new_path = path_so_far + delim + key if path_so_far else key
self._process_test_leaves(results_dir, delim, root_node[key],
new_path)
def _maybe_write_expected_output(self, results_dir, test_name):
"""Attempts to create an expected output artifact for the test.
The expected output of tests is checked-in to the source tree beside the
test itself, with a .ini extension. Not all tests have expected output.
Args:
results_dir: str path to the dir to write the output to
test_name: str name of the test to write expected output for
Returns:
string path to the artifact file that the expected output was
written to, relative to the directory that the original output is
located. Returns None if there is no expected output for this test.
"""
test_file_subpath = self.wpt_manifest.file_path_for_test_url(test_name)
if not test_file_subpath:
# Not all tests in the output have a corresponding test file. This
# could be print-reftests (which are unsupported by the blinkpy
# manifest) or .any.js tests (which appear in the output even though
# they do not actually run - they have corresponding tests like
# .any.worker.html which are covered here).
return None
test_file_path = os.path.join(EXTERNAL_WPT_TESTS_DIR, test_file_subpath)
expected_ini_path = test_file_path + ".ini"
if not os.path.exists(expected_ini_path):
return None
# This test has checked-in expected output. It needs to be copied to the
# results viewer directory and renamed from <test>.ini to
# <test>-expected.txt
# Note: Here we read-in the checked-in ini file and pass its contents to
# |_write_log_artifact| to reuse code. This is probably less efficient
# than just copying, but makes for cleaner code.
contents = self.fs.read_text_file(expected_ini_path)
return self._write_log_artifact(test_failures.FILENAME_SUFFIX_EXPECTED,
results_dir, test_name, [contents])
def _write_log_artifact(self, suffix, results_dir, test_name, log_artifact):
"""Writes a log artifact to disk.
A log artifact contains some form of output for a test. It is written to
a txt file with a suffix generated from the log type.
Args:
suffix: str suffix of the artifact to write, e.g.
test_failures.FILENAME_SUFFIX_ACTUAL
results_dir: str path to the directory that results live in
test_name: str name of the test that this artifact is for
log_artifact: list of strings, the log entries for this test from
the json output.
Returns:
string path to the artifact file that the log was written to,
relative to the directory that the original output is located.
"""
log_artifact_sub_path = (
os.path.join("layout-test-results",
self.port.output_filename(test_name, suffix, ".txt"))
)
log_artifact_full_path = os.path.join(results_dir,
log_artifact_sub_path)
if not os.path.exists(os.path.dirname(log_artifact_full_path)):
self.fs.maybe_make_directory(
os.path.dirname(log_artifact_full_path))
self.fs.write_text_file(log_artifact_full_path,
"\n".join(log_artifact))
return log_artifact_sub_path
def _write_screenshot_artifact(self, results_dir, test_name,
screenshot_artifact):
"""Write screenshot artifact to disk.
The screenshot artifact is a list of strings, each of which has the
format <url>:<base64-encoded PNG>. Each url-png pair is a screenshot of
either the test, or one of its refs. We can identify which screenshot is
for the test by comparing the url piece to the test name.
Args:
results_dir: str path to the directory that results live in
test:name str name of the test that this artifact is for
screenshot_artifact: list of strings, each being a url-png pair as
described above.
Returns:
A dict mapping the screenshot key (ie: actual, expected) to the
path of the file for that screenshot
"""
result={}
for screenshot_pair in screenshot_artifact:
screenshot_split = screenshot_pair.split(":")
url = screenshot_split[0]
# The url produced by wptrunner will have a leading / which we trim
# away for easier comparison to the test_name below.
if url.startswith("/"):
url = url[1:]
image_bytes = base64.b64decode(screenshot_split[1].strip())
screenshot_key = "expected_image"
file_suffix = test_failures.FILENAME_SUFFIX_EXPECTED
if test_name == url:
screenshot_key = "actual_image"
file_suffix = test_failures.FILENAME_SUFFIX_ACTUAL
screenshot_sub_path = (
os.path.join("layout-test-results",
self.port.output_filename(
test_name, file_suffix, ".png"))
)
result[screenshot_key] = screenshot_sub_path
screenshot_full_path = os.path.join(results_dir,screenshot_sub_path)
if not os.path.exists(os.path.dirname(screenshot_full_path)):
self.fs.maybe_make_directory(
os.path.dirname(screenshot_full_path))
# Note: we are writing raw bytes to this file
self.fs.write_binary_file(screenshot_full_path, image_bytes)
return result
