class WPTMetadataBuilder(object): def __init__(self, expectations, port): """ Args: expectations: a blinkpy.web_tests.models.test_expectations.TestExpectations object port: a blinkpy.web_tests.port.Port object """ self.expectations = expectations self.port = port # TODO(lpz): Use self.fs everywhere in this class and add tests self.fs = FileSystem() self.wpt_manifest = self.port.wpt_manifest("external/wpt") self.metadata_output_dir = "" self.checked_in_metadata_dir = "" self.process_baselines = True self.handle_annotations = True def run(self, args=None): """Main entry point to parse flags and execute the script.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--metadata-output-dir", help="The directory to output the metadata files into.") parser.add_argument( "--checked-in-metadata-dir", help="Root directory of any checked-in WPT metadata files to use. " "If set, these files will take precedence over legacy expectations " "and baselines when both exist for a test.") parser.add_argument( '-v', '--verbose', action='store_true', help='More verbose logging.') parser.add_argument( "--process-baselines", action="store_true", default=True, dest="process_baselines", help="Whether to translate baseline (-expected.txt) files into WPT " "metadata files. This translation is lossy and results in any " "subtest being accepted by wptrunner.") parser.add_argument("--no-process-baselines", action="store_false", dest="process_baselines") parser.add_argument( "--handle-annotations", action="store_true", default=True, dest="handle_annotations", help="Whether to handle annotations in expectations files. These " "are trailing comments that give additional details for how " "to translate an expectation into WPT metadata.") parser.add_argument("--no-handle-annotations", action="store_false", dest="handle_annotations") args = parser.parse_args(args) log_level = logging.DEBUG if args.verbose else logging.INFO configure_logging(logging_level=log_level, include_time=True) self.metadata_output_dir = args.metadata_output_dir self.checked_in_metadata_dir = args.checked_in_metadata_dir self.process_baselines = args.process_baselines self.handle_annotations = args.handle_annotations self._build_metadata_and_write() return 0 @staticmethod def status_bitmap_to_string(test_status_bitmap): statuses = [] result = "" if test_status_bitmap & SUBTEST_FAIL: result += " blink_expect_any_subtest_status: True # wpt_metadata_builder.py\n" if test_status_bitmap & HARNESS_ERROR: statuses.append("ERROR") if test_status_bitmap & TEST_PASS: # We need both PASS and OK. Reftests will PASS while testharness # tests are OK. statuses.append("PASS") statuses.append("OK") if test_status_bitmap & TEST_FAIL: # We need both FAIL and ERROR. Reftests will FAIL while testharness # tests have ERRORs. statuses.append("FAIL") statuses.append("ERROR") if test_status_bitmap & TEST_TIMEOUT: statuses.append("TIMEOUT") if test_status_bitmap & TEST_CRASH: statuses.append("CRASH") if test_status_bitmap & TEST_PRECONDITION_FAILED: statuses.append("PRECONDITION_FAILED") if statuses: result += " expected: [%s]\n" % ", ".join(statuses) return result def _build_metadata_and_write(self): """Build the metadata files and write them to disk.""" if os.path.exists(self.metadata_output_dir): _log.debug("Output dir exists, deleting: %s", self.metadata_output_dir) import shutil shutil.rmtree(self.metadata_output_dir) tests_for_metadata = self.get_tests_needing_metadata() _log.info("Found %d tests requiring metadata", len(tests_for_metadata)) for test_name, test_status_bitmap in tests_for_metadata.items(): filename, file_contents = self.get_metadata_filename_and_contents( test_name, test_status_bitmap) if not filename or not file_contents: continue self._write_to_file(filename, file_contents) if self.checked_in_metadata_dir and os.path.exists( self.checked_in_metadata_dir): _log.info("Copying checked-in WPT metadata on top of translated " "files.") self._copy_checked_in_metadata() else: _log.warning("Not using checked-in WPT metadata, path is empty or " "does not exist: %s" % self.checked_in_metadata_dir) # Finally, output a stamp file with the same name as the output # directory. The stamp file is empty, it's only used for its mtime. # This makes the GN build system happy (see crbug.com/995112). with open(self.metadata_output_dir + ".stamp", "w"): pass def _copy_checked_in_metadata(self): """Copies checked-in metadata files to the metadata output directory.""" for filename in self.fs.files_under(self.checked_in_metadata_dir): # We match any .ini files in the path. This will find .ini files # other than just metadata (such as tox.ini), but that is ok # since wptrunner will just ignore those. if not fnmatch.fnmatch(filename, "*.ini"): continue # Found a checked-in .ini file. Copy it to the metadata output # directory in the same sub-path as where it is checked in. # So /checked/in/a/b/c.ini goes to /metadata/out/a/b/c.ini output_path = filename.replace(self.checked_in_metadata_dir, self.metadata_output_dir) if not self.fs.exists(self.fs.dirname(output_path)): self.fs.maybe_make_directory(self.fs.dirname(output_path)) _log.debug("Copying %s to %s" % (filename, output_path)) self.fs.copyfile(filename, output_path) def _write_to_file(self, filename, file_contents): # Write the contents to the file name if not os.path.exists(os.path.dirname(filename)): os.makedirs(os.path.dirname(filename)) # Note that we append to the metadata file in order to allow multiple # tests to be present in the same .ini file (ie: for multi-global tests) with open(filename, "a") as metadata_file: metadata_file.write(file_contents) def get_tests_needing_metadata(self): """Determines which tests need metadata files. This function loops over the tests to be run and checks whether each test has an expectation (eg: in TestExpectations) and/or a baseline (ie: test-name-expected.txt). The existence of those things will determine the information that will be emitted into the tests's metadata file. Returns: A dict. The key is the string test name and the value is an integer bitmap of statuses for the test. """ tests_needing_metadata = defaultdict(int) for test_name in self.port.tests(paths=["external/wpt"]): # First check for expectations. If a test is skipped then we do not # look for more statuses expectation_line = self.expectations.get_expectations(test_name) self._handle_test_with_expectation(test_name, expectation_line, tests_needing_metadata) if self._test_was_skipped(test_name, tests_needing_metadata): # Do not consider other statuses if a test is skipped continue # Check if the test has a baseline if self.process_baselines: test_baseline = self.port.expected_text(test_name) if not test_baseline: continue self._handle_test_with_baseline(test_name, test_baseline, tests_needing_metadata) return tests_needing_metadata def _handle_test_with_expectation(self, test_name, expectation_line, status_dict): """Handles a single test expectation and updates |status_dict|.""" test_statuses = expectation_line.results annotations = expectation_line.trailing_comments if ResultType.Skip in test_statuses: # Skips are handled alone, so don't look at any other statuses status_dict[test_name] |= SKIP_TEST return # Guard against the only test_status being Pass (without any # annotations), we don't want to create metadata for such a test. if (len(test_statuses) == 1 and ResultType.Pass in test_statuses and not annotations): return status_bitmap = 0 if ResultType.Pass in test_statuses: status_bitmap |= TEST_PASS if ResultType.Failure in test_statuses: status_bitmap |= TEST_FAIL if ResultType.Timeout in test_statuses: status_bitmap |= TEST_TIMEOUT if ResultType.Crash in test_statuses: status_bitmap |= TEST_CRASH if self.handle_annotations and annotations: if "wpt_subtest_failure" in annotations: status_bitmap |= SUBTEST_FAIL if "wpt_precondition_failed" in annotations: status_bitmap |= TEST_PRECONDITION_FAILED # Update status bitmap for this test status_dict[test_name] |= status_bitmap def _test_was_skipped(self, test_name, status_dict): """Returns whether |test_name| is marked as skipped in |status_dict|.""" return test_name in status_dict and ( status_dict[test_name] & SKIP_TEST) def _handle_test_with_baseline(self, test_name, test_baseline, status_dict): """Handles a single test baseline and updates |status_dict|.""" status_bitmap = 0 if re.search(r"^(FAIL|NOTRUN|TIMEOUT)", test_baseline, re.MULTILINE): status_bitmap |= SUBTEST_FAIL if re.search(r"^Harness Error\.", test_baseline, re.MULTILINE): status_bitmap |= HARNESS_ERROR if status_bitmap > 0: status_dict[test_name] |= status_bitmap else: # Treat this as an error because we don't want it to happen. # Either the non-FAIL statuses need to be handled here, or the # baseline is all PASS which should just be deleted. _log.error("Test %s has a non-FAIL baseline" % test_name) def _metadata_filename_from_test_file(self, wpt_test_file): """Returns the filename of the metadata (.ini) file for the test. Args: wpt_test_file: The file on disk that the specified test lives in. For multi-global tests this is usually a ".js" file. Returns: The fully-qualified string path of the metadata file for this test. """ assert "?" not in wpt_test_file test_file_parts = wpt_test_file.split("/") return os.path.join(self.metadata_output_dir, *test_file_parts) + ".ini" def _metadata_inline_test_name_from_test_name(self, wpt_test_name): """Returns the test name to use *inside* of a metadata file. The inline name inside the metadata file is the logical name of the test without any subdirectories. For multi-global tests this means that it must have the specific scope of the test (eg: worker, window, etc). This name must also include any variants that are set. Args: wpt_test_name: The fully-qualified test name which contains all subdirectories as well as scope (for multi-globals), and variants. Returns: The string test name inside of the metadata file. """ # To generate the inline test name we basically want to strip away the # subdirectories from the test name, being careful not to accidentally # clobber the variant. variant_split = wpt_test_name.split("?") test_path = variant_split[0] test_name_part = test_path.split("/")[-1] variant = "?" + variant_split[1] if len(variant_split) == 2 else "" return test_name_part + variant def get_metadata_filename_and_contents(self, chromium_test_name, test_status_bitmap=0): """Determines the metadata filename and contents for the specified test. The metadata filename is derived from the test name but will differ if the expectation is for a single test or for a directory of tests. The contents of the metadata file will also differ for those two cases. Args: chromium_test_name: A Chromium test name from the expectation file, which starts with `external/wpt`. test_status_bitmap: An integer containing additional data about the status, such as enumerating flaky statuses, or whether a test has a combination of harness error and subtest failure. Returns: A pair of strings, the first is the path to the metadata file and the second is the contents to write to that file. Or None if the test does not need a metadata file. """ # Ignore expectations for non-WPT tests if (not chromium_test_name or not chromium_test_name.startswith('external/wpt')): return None, None # Split the test name by directory. We omit the first 2 entries because # they are 'external' and 'wpt' and these don't exist in the WPT's test # names. wpt_test_name_parts = chromium_test_name.split("/")[2:] # The WPT test name differs from the Chromium test name in that the WPT # name omits `external/wpt`. wpt_test_name = "/".join(wpt_test_name_parts) # Check if this is a test file or a test directory is_test_dir = chromium_test_name.endswith("/") metadata_filename = None metadata_file_contents = None if is_test_dir: # A test directory gets one metadata file called __dir__.ini and all # tests in that dir are skipped. metadata_filename = os.path.join(self.metadata_output_dir, *wpt_test_name_parts) metadata_filename = os.path.join(metadata_filename, "__dir__.ini") _log.debug("Creating a dir-wide ini file %s", metadata_filename) metadata_file_contents = self._get_dir_disabled_string() else: # For individual tests, we create one file per test, with the name # of the test in the file as well. test_file_path = self.wpt_manifest.file_path_for_test_url( wpt_test_name) if not test_file_path: _log.info("Could not find file for test %s, skipping" % wpt_test_name) return None, None metadata_filename = self._metadata_filename_from_test_file( test_file_path) _log.debug("Creating a test ini file %s with status_bitmap %s", metadata_filename, test_status_bitmap) inline_test_name = self._metadata_inline_test_name_from_test_name( wpt_test_name) metadata_file_contents = self._get_test_failed_string( inline_test_name, test_status_bitmap) return metadata_filename, metadata_file_contents def _get_dir_disabled_string(self): return "disabled: wpt_metadata_builder.py\n" def _get_test_disabled_string(self, test_name): return "[%s]\n disabled: wpt_metadata_builder.py\n" % test_name def _get_test_failed_string(self, inline_test_name, test_status_bitmap): # The contents of the metadata file is two lines: # 1. the inline name of the WPT test pathinside square brackets. This # name contains the test scope (for multi-globals) and variants. # 2. an indented line with the test status and reason result = "[%s]\n" % inline_test_name # A skipped test is a little special in that it doesn't happen along with # any other status. So we compare directly against SKIP_TEST and also # return right away. if test_status_bitmap == SKIP_TEST: result += " disabled: wpt_metadata_builder.py\n" return result # Other test statuses can exist together. But ensure we have at least one. expected_string = self.status_bitmap_to_string(test_status_bitmap) if expected_string: result += expected_string return result
def test_getcwd(self): fs = FileSystem() self.assertTrue(fs.exists(fs.getcwd()))
def test_exists__true(self): fs = FileSystem() self.assertTrue(fs.exists(self._this_file))
def test_exists__false(self): fs = FileSystem() self.assertFalse(fs.exists(self._missing_file))
class GitTestWithRealFilesystemAndExecutive(unittest.TestCase): def setUp(self): self.executive = Executive() self.filesystem = FileSystem() self.original_cwd = self.filesystem.getcwd() # Set up fresh git repository with one commit. self.untracking_checkout_path = self._mkdtemp( suffix='-git_unittest_untracking') self._run(['git', 'init', self.untracking_checkout_path]) self._chdir(self.untracking_checkout_path) # Explicitly create the default branch instead of relying on # init.defaultBranch. We don't use the new --initial-branch flag with # `git init` to keep the tests compatible with older versions of git. self._run(['git', 'checkout', '-b', 'master']) self._set_user_config() self._write_text_file('foo_file', 'foo') self._run(['git', 'add', 'foo_file']) self._run(['git', 'commit', '-am', 'dummy commit']) self.untracking_git = Git( cwd=self.untracking_checkout_path, filesystem=self.filesystem, executive=self.executive) # Then set up a second git repo that tracks the first one. self.tracking_git_checkout_path = self._mkdtemp( suffix='-git_unittest_tracking') self._run([ 'git', 'clone', '--quiet', self.untracking_checkout_path, self.tracking_git_checkout_path ]) self._chdir(self.tracking_git_checkout_path) self._set_user_config() self.tracking_git = Git( cwd=self.tracking_git_checkout_path, filesystem=self.filesystem, executive=self.executive) def tearDown(self): self._chdir(self.original_cwd) self._run(['rm', '-rf', self.tracking_git_checkout_path]) self._run(['rm', '-rf', self.untracking_checkout_path]) def _set_user_config(self): self._run(['git', 'config', '--local', 'user.name', 'Fake']) self._run( ['git', 'config', '--local', 'user.email', '*****@*****.**']) def _chdir(self, path): self.filesystem.chdir(path) def _mkdir(self, path): assert not self.filesystem.exists(path) self.filesystem.maybe_make_directory(path) def _mkdtemp(self, **kwargs): return str(self.filesystem.mkdtemp(**kwargs)) def _run(self, *args, **kwargs): return self.executive.run_command(*args, **kwargs) def _write_text_file(self, path, contents): self.filesystem.write_text_file(path, contents) def test_add_list(self): self._chdir(self.untracking_checkout_path) git = self.untracking_git self._mkdir('added_dir') self._write_text_file('added_dir/added_file', 'new stuff') self._run(['ls', 'added_dir']) self._run(['pwd']) self._run(['cat', 'added_dir/added_file']) git.add_list(['added_dir/added_file']) self.assertIn('added_dir/added_file', git.added_files()) def test_delete_recursively(self): self._chdir(self.untracking_checkout_path) git = self.untracking_git self._mkdir('added_dir') self._write_text_file('added_dir/added_file', 'new stuff') git.add_list(['added_dir/added_file']) self.assertIn('added_dir/added_file', git.added_files()) git.delete_list(['added_dir/added_file']) self.assertNotIn('added_dir', git.added_files()) def test_delete_recursively_or_not(self): self._chdir(self.untracking_checkout_path) git = self.untracking_git self._mkdir('added_dir') self._write_text_file('added_dir/added_file', 'new stuff') self._write_text_file('added_dir/another_added_file', 'more new stuff') git.add_list(['added_dir/added_file', 'added_dir/another_added_file']) self.assertIn('added_dir/added_file', git.added_files()) self.assertIn('added_dir/another_added_file', git.added_files()) git.delete_list(['added_dir/added_file']) self.assertIn('added_dir/another_added_file', git.added_files()) def test_exists(self): self._chdir(self.untracking_checkout_path) git = self.untracking_git self._chdir(git.checkout_root) self.assertFalse(git.exists('foo.txt')) self._write_text_file('foo.txt', 'some stuff') self.assertFalse(git.exists('foo.txt')) git.add_list(['foo.txt']) git.commit_locally_with_message('adding foo') self.assertTrue(git.exists('foo.txt')) git.delete_list(['foo.txt']) git.commit_locally_with_message('deleting foo') self.assertFalse(git.exists('foo.txt')) def test_move(self): self._chdir(self.untracking_checkout_path) git = self.untracking_git self._write_text_file('added_file', 'new stuff') git.add_list(['added_file']) git.move('added_file', 'moved_file') self.assertIn('moved_file', git.added_files()) def test_move_recursive(self): self._chdir(self.untracking_checkout_path) git = self.untracking_git self._mkdir('added_dir') self._write_text_file('added_dir/added_file', 'new stuff') self._write_text_file('added_dir/another_added_file', 'more new stuff') git.add_list(['added_dir']) git.move('added_dir', 'moved_dir') self.assertIn('moved_dir/added_file', git.added_files()) self.assertIn('moved_dir/another_added_file', git.added_files()) def test_remote_branch_ref(self): # This tests a protected method. pylint: disable=protected-access self.assertEqual(self.tracking_git._remote_branch_ref(), 'refs/remotes/origin/master') self._chdir(self.untracking_checkout_path) self.assertRaises(ScriptError, self.untracking_git._remote_branch_ref) def test_create_patch(self): self._chdir(self.tracking_git_checkout_path) git = self.tracking_git self._write_text_file('test_file_commit1', 'contents') self._run(['git', 'add', 'test_file_commit1']) git.commit_locally_with_message('message') patch = git.create_patch() self.assertNotRegexpMatches(patch, r'Subversion Revision:') def test_patches_have_filenames_with_prefixes(self): self._chdir(self.tracking_git_checkout_path) git = self.tracking_git self._write_text_file('test_file_commit1', 'contents') self._run(['git', 'add', 'test_file_commit1']) git.commit_locally_with_message('message') # Even if diff.noprefix is enabled, create_patch() produces diffs with prefixes. self._run(['git', 'config', 'diff.noprefix', 'true']) patch = git.create_patch() self.assertRegexpMatches( patch, r'^diff --git a/test_file_commit1 b/test_file_commit1') def test_rename_files(self): self._chdir(self.tracking_git_checkout_path) git = self.tracking_git git.move('foo_file', 'bar_file') git.commit_locally_with_message('message') patch = git.create_patch(changed_files=git.changed_files()) self.assertTrue('rename from' in patch) def test_commit_position_from_git_log(self): # This tests a protected method. pylint: disable=protected-access git_log = """ commit 624c3081c0 Author: foobarbaz1 <*****@*****.**> Date: Mon Sep 28 19:10:30 2015 -0700 Test foo bar baz qux 123. BUG=000000 Review URL: https://codereview.chromium.org/999999999 Cr-Commit-Position: refs/heads/master@{#1234567} """ self._chdir(self.tracking_git_checkout_path) git = self.tracking_git self.assertEqual(git._commit_position_from_git_log(git_log), 1234567)