def test_update(self): host = MockHost() filesystem = host.filesystem finder = PathFinder(filesystem) flag_expectations_file = finder.path_from_layout_tests( 'FlagExpectations', 'foo') filesystem.write_text_file( flag_expectations_file, 'something/pass-unexpectedly-mac.html [ Fail ]') self._setup_mock_results(host.buildbot) cmd = ['update', '--flag=--foo'] TryFlag(cmd, host, MockGitCL(host, self.mock_try_results)).run() def results_url(build): return '%s/%s/%s/layout-test-results/results.html' % ( 'https://test-results.appspot.com/data/layout_results', build.builder_name, build.build_number) self.assertEqual( host.stdout.getvalue(), '\n'.join([ 'Fetching results...', '-- Linux: %s' % results_url(self.linux_build), '-- Mac: %s' % results_url(self.mac_build), '-- Win: %s' % results_url(self.win_build), '', '### 1 unexpected passes:', '', 'Bug(none) [ Mac ] something/pass-unexpectedly-mac.html [ Pass ]', '', '### 2 unexpected failures:', '', 'Bug(none) something/fail-everywhere.html [ Failure ]', 'Bug(none) [ Linux Win ] something/fail-win-and-linux.html [ Failure ]', '' ]))
def _run_trigger_test(self, regenerate): host = MockHost() git = host.git() git_cl = MockGitCL(host) finder = PathFinder(host.filesystem) flag_file = finder.path_from_layout_tests( 'additional-driver-flag.setting') flag_expectations_file = finder.path_from_layout_tests( 'FlagExpectations', 'foo') cmd = ['trigger', '--flag=--foo'] if regenerate: cmd.append('--regenerate') TryFlag(cmd, host, git_cl).run() expected_added_paths = {flag_file} expected_commits = [[ 'Flag try job: force --foo for run-webkit-tests.' ]] if regenerate: expected_added_paths.add(flag_expectations_file) expected_commits.append( ['Flag try job: clear expectations for --foo.']) self.assertEqual(git.added_paths, expected_added_paths) self.assertEqual(git.local_commits(), expected_commits) self.assertEqual( git_cl.calls, [[ 'git', 'cl', 'upload', '--bypass-hooks', '-f', '-m', 'Flag try job for --foo.' ], [ 'git', 'cl', 'try', '-m', 'tryserver.chromium.linux', '-b', 'linux_chromium_rel_ng' ], [ 'git', 'cl', 'try', '-m', 'tryserver.chromium.mac', '-b', 'mac_chromium_rel_ng' ], [ 'git', 'cl', 'try', '-m', 'tryserver.chromium.win', '-b', 'win7_chromium_rel_ng' ]])
def _add_base_manifest_to_mock_filesystem(self, filesystem): path_finder = PathFinder(filesystem) external_dir = path_finder.path_from_layout_tests('external') filesystem.maybe_make_directory(filesystem.join(external_dir, 'wpt')) manifest_base_path = filesystem.join(external_dir, 'WPT_BASE_MANIFEST.json') filesystem.files[manifest_base_path] = '{}'
def ensure_manifest(host): """Generates the MANIFEST.json file if it does not exist.""" finder = PathFinder(host.filesystem) manifest_path = finder.path_from_layout_tests('external', 'wpt', 'MANIFEST.json') base_manifest_path = finder.path_from_layout_tests('external', 'WPT_BASE_MANIFEST.json') if not host.filesystem.exists(base_manifest_path): _log.error('Manifest base not found at "%s".', base_manifest_path) host.filesystem.write_text_file(base_manifest_path, '{}') if not host.filesystem.exists(manifest_path): _log.debug('Manifest not found, copying from base "%s".', base_manifest_path) host.filesystem.copyfile(base_manifest_path, manifest_path) wpt_path = manifest_path = finder.path_from_layout_tests('external', 'wpt') WPTManifest.generate_manifest(host, wpt_path) _log.debug('Manifest generation completed.')
def test_update_irrelevant_unexpected_pass(self): host = MockHost() filesystem = host.filesystem finder = PathFinder(filesystem) flag_expectations_file = finder.path_from_layout_tests( 'FlagExpectations', 'foo') self._setup_mock_results(host.buildbot) cmd = ['update', '--flag=--foo'] # Unexpected passes that don't have flag-specific failure expectations # should not be reported. filesystem.write_text_file(flag_expectations_file, '') TryFlag(cmd, host, MockGitCL(host, self.mock_try_results)).run() self.assertTrue('### 0 unexpected passes' in host.stdout.getvalue())
class DirectoryOwnersExtractor(object): def __init__(self, filesystem=None): self.filesystem = filesystem or FileSystem() self.finder = PathFinder(filesystem) self.owner_map = None def list_owners(self, changed_files): """Looks up the owners for the given set of changed files. Args: changed_files: A list of file paths relative to the repository root. Returns: A dict mapping tuples of owner email addresses to lists of owned directories (paths relative to the root of layout tests). """ email_map = collections.defaultdict(set) external_root_owners = self.finder.path_from_layout_tests('external', 'OWNERS') for relpath in changed_files: # Try to find the first *non-empty* OWNERS file. absolute_path = self.finder.path_from_chromium_base(relpath) owners = None owners_file = self.find_owners_file(absolute_path) while owners_file: owners = self.extract_owners(owners_file) if owners: break # Found an empty OWNERS file. Try again from the parent directory. absolute_path = self.filesystem.dirname(self.filesystem.dirname(owners_file)) owners_file = self.find_owners_file(absolute_path) # Skip LayoutTests/external/OWNERS. if not owners or owners_file == external_root_owners: continue owned_directory = self.filesystem.dirname(owners_file) owned_directory_relpath = self.filesystem.relpath(owned_directory, self.finder.layout_tests_dir()) email_map[tuple(owners)].add(owned_directory_relpath) return {owners: sorted(owned_directories) for owners, owned_directories in email_map.iteritems()} def find_owners_file(self, start_path): """Finds the first enclosing OWNERS file for a given path. Starting from the given path, walks up the directory tree until the first OWNERS file is found or LayoutTests/external is reached. Args: start_path: A relative path from the root of the repository, or an absolute path. The path can be a file or a directory. Returns: The absolute path to the first OWNERS file found; None if not found or if start_path is outside of LayoutTests/external. """ abs_start_path = (start_path if self.filesystem.isabs(start_path) else self.finder.path_from_chromium_base(start_path)) directory = (abs_start_path if self.filesystem.isdir(abs_start_path) else self.filesystem.dirname(abs_start_path)) external_root = self.finder.path_from_layout_tests('external') if not directory.startswith(external_root): return None # Stop at LayoutTests, which is the parent of external_root. while directory != self.finder.layout_tests_dir(): owners_file = self.filesystem.join(directory, 'OWNERS') if self.filesystem.isfile(self.finder.path_from_chromium_base(owners_file)): return owners_file directory = self.filesystem.dirname(directory) return None def extract_owners(self, owners_file): """Extracts owners from an OWNERS file. Args: owners_file: An absolute path to an OWNERS file. Returns: A list of valid owners (email addresses). """ contents = self._read_text_file(owners_file) email_regexp = re.compile(BASIC_EMAIL_REGEXP) addresses = [] for line in contents.splitlines(): line = line.strip() if email_regexp.match(line): addresses.append(line) return addresses def extract_component(self, owners_file): """Extracts the component from an OWNERS file. Args: owners_file: An absolute path to an OWNERS file. Returns: A string, or None if not found. """ contents = self._read_text_file(owners_file) search = re.search(COMPONENT_REGEXP, contents, re.MULTILINE) if search: return search.group(1) return None def is_wpt_notify_enabled(self, owners_file): """Checks if the OWNERS file enables WPT-NOTIFY. Args: owners_file: An absolute path to an OWNERS file. Returns: A boolean. """ contents = self._read_text_file(owners_file) return bool(re.search(WPT_NOTIFY_REGEXP, contents, re.MULTILINE)) @memoized def _read_text_file(self, path): return self.filesystem.read_text_file(path)
class TestCopier(object): def __init__(self, host, source_repo_path): """Initializes variables to prepare for copying and converting files. Args: host: An instance of Host. source_repo_path: Path to the local checkout of web-platform-tests. """ self.host = host assert self.host.filesystem.exists(source_repo_path) self.source_repo_path = source_repo_path self.filesystem = self.host.filesystem self.path_finder = PathFinder(self.filesystem) self.layout_tests_dir = self.path_finder.layout_tests_dir() self.destination_directory = self.filesystem.normpath( self.filesystem.join( self.layout_tests_dir, DEST_DIR_NAME, self.filesystem.basename(self.source_repo_path))) self.import_in_place = ( self.source_repo_path == self.destination_directory) self.dir_above_repo = self.filesystem.dirname(self.source_repo_path) self.import_list = [] # This is just a FYI list of CSS properties that still need to be prefixed, # which may be output after importing. self._prefixed_properties = {} def do_import(self): _log.info('Importing %s into %s', self.source_repo_path, self.destination_directory) self.find_importable_tests() self.import_tests() def find_importable_tests(self): """Walks through the source directory to find what tests should be imported. This function sets self.import_list, which contains information about how many tests are being imported, and their source and destination paths. """ paths_to_skip = self.find_paths_to_skip() for root, dirs, files in self.filesystem.walk(self.source_repo_path): cur_dir = root.replace(self.dir_above_repo + '/', '') + '/' _log.debug('Scanning %s...', cur_dir) dirs_to_skip = ('.git', ) if dirs: for name in dirs_to_skip: if name in dirs: dirs.remove(name) for path in paths_to_skip: path_base = path.replace(DEST_DIR_NAME + '/', '') path_base = path_base.replace(cur_dir, '') path_full = self.filesystem.join(root, path_base) if path_base in dirs: _log.info('Skipping: %s', path_full) dirs.remove(path_base) if self.import_in_place: self.filesystem.rmtree(path_full) copy_list = [] for filename in files: path_full = self.filesystem.join(root, filename) path_base = path_full.replace(self.source_repo_path + '/', '') path_base = self.destination_directory.replace( self.layout_tests_dir + '/', '') + '/' + path_base if path_base in paths_to_skip: if self.import_in_place: _log.debug('Pruning: %s', path_base) self.filesystem.remove(path_full) continue else: continue # FIXME: This block should really be a separate function, but the early-continues make that difficult. if is_basename_skipped(filename): _log.debug('Skipping: %s', path_full) _log.debug( ' Reason: This file may cause Chromium presubmit to fail.' ) continue copy_list.append({'src': path_full, 'dest': filename}) if copy_list: # Only add this directory to the list if there's something to import self.import_list.append({ 'dirname': root, 'copy_list': copy_list }) def find_paths_to_skip(self): paths_to_skip = set() port = self.host.port_factory.get() w3c_import_expectations_path = self.path_finder.path_from_layout_tests( 'W3CImportExpectations') w3c_import_expectations = self.filesystem.read_text_file( w3c_import_expectations_path) parser = TestExpectationParser(port, all_tests=(), is_lint_mode=False) expectation_lines = parser.parse(w3c_import_expectations_path, w3c_import_expectations) for line in expectation_lines: if 'SKIP' in line.expectations: if line.specifiers: _log.warning( 'W3CImportExpectations:%s should not have any specifiers', line.line_numbers) continue paths_to_skip.add(line.name) return paths_to_skip def import_tests(self): """Reads |self.import_list|, and converts and copies files to their destination.""" for dir_to_copy in self.import_list: if not dir_to_copy['copy_list']: continue orig_path = dir_to_copy['dirname'] relative_dir = self.filesystem.relpath(orig_path, self.source_repo_path) dest_dir = self.filesystem.join(self.destination_directory, relative_dir) if not self.filesystem.exists(dest_dir): self.filesystem.maybe_make_directory(dest_dir) for file_to_copy in dir_to_copy['copy_list']: self.copy_file(file_to_copy, dest_dir) _log.info('') _log.info('Import complete') _log.info('') if self._prefixed_properties: _log.info('Properties needing prefixes (by count):') for prefixed_property in sorted( self._prefixed_properties, key=lambda p: self._prefixed_properties[p]): _log.info(' %s: %s', prefixed_property, self._prefixed_properties[prefixed_property]) def copy_file(self, file_to_copy, dest_dir): """Converts and copies a file, if it should be copied. Args: file_to_copy: A dict in a file copy list constructed by find_importable_tests, which represents one file to copy, including the keys: "src": Absolute path to the source location of the file. "destination": File name of the destination file. And possibly also the keys "reference_support_info" or "is_jstest". dest_dir: Path to the directory where the file should be copied. """ source_path = self.filesystem.normpath(file_to_copy['src']) dest_path = self.filesystem.join(dest_dir, file_to_copy['dest']) if self.filesystem.isdir(source_path): _log.error('%s refers to a directory', source_path) return if not self.filesystem.exists(source_path): _log.error('%s not found. Possible error in the test.', source_path) return if not self.filesystem.exists(self.filesystem.dirname(dest_path)): if not self.import_in_place: self.filesystem.maybe_make_directory( self.filesystem.dirname(dest_path)) relpath = self.filesystem.relpath(dest_path, self.layout_tests_dir) # FIXME: Maybe doing a file diff is in order here for existing files? # In other words, there's no sense in overwriting identical files, but # there's no harm in copying the identical thing. _log.debug(' copying %s', relpath) if not self.import_in_place: self.filesystem.copyfile(source_path, dest_path) if self.filesystem.read_binary_file(source_path)[:2] == '#!': self.filesystem.make_executable(dest_path)
class TryFlag(object): def __init__(self, argv, host, git_cl): self._args = parse_args(argv) self._host = host self._git_cl = git_cl self._expectations_model = TestExpectationsModel() self._test_configuration_converter = TestConfigurationConverter( set(BUILDER_CONFIGS.values())) self._filesystem = self._host.filesystem self._path_finder = PathFinder(self._filesystem) self._git = self._host.git() def _set_flag(self, flag): path = self._path_finder.path_from_layout_tests(FLAG_FILE) self._filesystem.write_text_file(path, flag + '\n') self._git.add_list([path]) self._git.commit_locally_with_message( 'Flag try job: force %s for run-webkit-tests.' % flag) def _clear_expectations(self, flag): path = self._path_finder.path_from_layout_tests( 'FlagExpectations', flag.lstrip('-')) self._filesystem.write_text_file(path, '') self._git.add_list([path]) self._git.commit_locally_with_message( 'Flag try job: clear expectations for %s.' % flag) def trigger(self): flag = self._args.flag if flag: self._set_flag(flag) if self._args.regenerate: self._clear_expectations(flag) self._git_cl.run([ 'upload', '--bypass-hooks', '-f', '-m', 'Flag try job for %s.' % flag ]) for builder in sorted(BUILDER_MASTERS.keys()): master = BUILDER_MASTERS[builder] self._git_cl.trigger_try_jobs([builder], master) def _create_expectation_line(self, result, test_configuration): test_name = result.test_name() line = TestExpectationLine() line.name = test_name line.path = test_name line.matching_tests = [test_name] line.filename = '' if self._args.bug: line.bugs = ['crbug.com/%s' % self._args.bug] else: line.bugs = ['Bug(none)'] line.expectations = result.actual_results().split() line.parsed_expectations = [ TestExpectations.expectation_from_string(expectation) for expectation in line.expectations ] line.specifiers = [test_configuration.version] line.matching_configurations = set([test_configuration]) return line def _process_result(self, build, result): if not result.did_run_as_expected(): self._expectations_model.add_expectation_line( self._create_expectation_line( result, BUILDER_CONFIGS[build.builder_name]), model_all_expectations=True) def update(self): self._host.print_('Fetching results...') # TODO: Get jobs from the _tryflag branch. Current branch for now. jobs = self._git_cl.latest_try_jobs(BUILDER_CONFIGS.keys()) buildbot = self._host.buildbot for build in sorted(jobs.keys()): self._host.print_( '-- %s: %s/results.html' % (BUILDER_CONFIGS[build.builder_name].version, buildbot.results_url(build.builder_name, build.build_number))) results = buildbot.fetch_results(build, True) results.for_each_test( lambda result, b=build: self._process_result(b, result)) # TODO: Write to flag expectations file. For now, stdout. :) unexpected_failures = [] unexpected_passes = [] for line in self._expectations_model.all_lines(): if TestExpectations.EXPECTATIONS[ 'pass'] in line.parsed_expectations: unexpected_passes.append(line) else: unexpected_failures.append(line) self._print_all(unexpected_passes, 'unexpected passes') self._print_all(unexpected_failures, 'unexpected failures') def _print_all(self, lines, description): self._host.print_('\n### %s %s:\n' % (len(lines), description)) for line in lines: self._host.print_( line.to_string(self._test_configuration_converter)) def run(self): action = self._args.action if action == 'trigger': self.trigger() elif action == 'update': self.update() else: print >> self._host.stderr, 'specify "trigger" or "update"' return 1 return 0
class Manager(object): """A class for managing running a series of layout tests.""" HTTP_SUBDIR = 'http' PERF_SUBDIR = 'perf' WEBSOCKET_SUBDIR = 'websocket' ARCHIVED_RESULTS_LIMIT = 25 def __init__(self, port, options, printer): """Initializes test runner data structures. Args: port: An object implementing platform-specific functionality. options: An options argument which contains command line options. printer: A Printer object to record updates to. """ self._port = port self._filesystem = port.host.filesystem self._options = options self._printer = printer self._expectations = None self._http_server_started = False self._wptserve_started = False self._websockets_server_started = False self._results_directory = self._port.results_directory() self._finder = LayoutTestFinder(self._port, self._options) self._path_finder = PathFinder(port.host.filesystem) self._runner = LayoutTestRunner(self._options, self._port, self._printer, self._results_directory, self._test_is_slow) def run(self, args): """Runs the tests and return a RunDetails object with the results.""" start_time = time.time() self._printer.write_update('Collecting tests ...') running_all_tests = False if not args or any('external' in path for path in args): self._printer.write_update( 'Generating MANIFEST.json for web-platform-tests ...') WPTManifest.ensure_manifest(self._port.host) self._printer.write_update('Completed generating manifest.') self._printer.write_update('Collecting tests ...') try: paths, all_test_names, running_all_tests = self._collect_tests( args) except IOError: # This is raised if --test-list doesn't exist return test_run_results.RunDetails( exit_code=exit_codes.NO_TESTS_EXIT_STATUS) # Create a sorted list of test files so the subset chunk, # if used, contains alphabetically consecutive tests. if self._options.order == 'natural': all_test_names.sort(key=self._port.test_key) elif self._options.order == 'random': all_test_names.sort() random.Random(self._options.seed).shuffle(all_test_names) test_names, tests_in_other_chunks = self._finder.split_into_chunks( all_test_names) self._printer.write_update('Parsing expectations ...') self._expectations = test_expectations.TestExpectations( self._port, test_names) tests_to_run, tests_to_skip = self._prepare_lists(paths, test_names) self._expectations.remove_tests_from_expectations( tests_in_other_chunks) self._printer.print_found(len(all_test_names), len(test_names), len(tests_to_run), self._options.repeat_each, self._options.iterations) # Check to make sure we're not skipping every test. if not tests_to_run: _log.critical('No tests to run.') return test_run_results.RunDetails( exit_code=exit_codes.NO_TESTS_EXIT_STATUS) exit_code = self._set_up_run(tests_to_run) if exit_code: return test_run_results.RunDetails(exit_code=exit_code) # Don't retry failures if an explicit list of tests was passed in. if self._options.retry_failures is None: should_retry_failures = len(paths) < len(test_names) else: should_retry_failures = self._options.retry_failures try: self._start_servers(tests_to_run) if self._options.watch: run_results = self._run_test_loop(tests_to_run, tests_to_skip) else: run_results = self._run_test_once(tests_to_run, tests_to_skip, should_retry_failures) initial_results, all_retry_results, enabled_pixel_tests_in_retry = run_results finally: self._stop_servers() self._clean_up_run() # Some crash logs can take a long time to be written out so look # for new logs after the test run finishes. self._printer.write_update('Looking for new crash logs ...') self._look_for_new_crash_logs(initial_results, start_time) for retry_attempt_results in all_retry_results: self._look_for_new_crash_logs(retry_attempt_results, start_time) self._printer.write_update('Summarizing results ...') summarized_full_results = test_run_results.summarize_results( self._port, self._expectations, initial_results, all_retry_results, enabled_pixel_tests_in_retry) summarized_failing_results = test_run_results.summarize_results( self._port, self._expectations, initial_results, all_retry_results, enabled_pixel_tests_in_retry, only_include_failing=True) exit_code = summarized_failing_results['num_regressions'] if exit_code > exit_codes.MAX_FAILURES_EXIT_STATUS: _log.warning('num regressions (%d) exceeds max exit status (%d)', exit_code, exit_codes.MAX_FAILURES_EXIT_STATUS) exit_code = exit_codes.MAX_FAILURES_EXIT_STATUS if not self._options.dry_run: self._write_json_files(summarized_full_results, summarized_failing_results, initial_results, running_all_tests) self._upload_json_files() self._copy_results_html_file(self._results_directory, 'results.html') self._copy_results_html_file(self._results_directory, 'legacy-results.html') if initial_results.keyboard_interrupted: exit_code = exit_codes.INTERRUPTED_EXIT_STATUS else: if initial_results.interrupted: exit_code = exit_codes.EARLY_EXIT_STATUS if self._options.show_results and ( exit_code or initial_results.total_failures): self._port.show_results_html_file( self._filesystem.join(self._results_directory, 'results.html')) self._printer.print_results(time.time() - start_time, initial_results) return test_run_results.RunDetails(exit_code, summarized_full_results, summarized_failing_results, initial_results, all_retry_results, enabled_pixel_tests_in_retry) def _run_test_loop(self, tests_to_run, tests_to_skip): # Don't show results in a new browser window because we're already # printing the link to diffs in the loop self._options.show_results = False while True: initial_results, all_retry_results, enabled_pixel_tests_in_retry = self._run_test_once( tests_to_run, tests_to_skip, should_retry_failures=False) for name in initial_results.failures_by_name: failure = initial_results.failures_by_name[name][0] if isinstance(failure, test_failures.FailureTextMismatch): full_test_path = self._filesystem.join( self._results_directory, name) filename, _ = self._filesystem.splitext(full_test_path) pretty_diff_path = 'file://' + filename + '-pretty-diff.html' self._printer.writeln('Link to pretty diff:') self._printer.writeln(pretty_diff_path + '\n') self._printer.writeln('Finished running tests') user_input = self._port.host.user.prompt( 'Interactive watch mode: (q)uit (r)etry\n').lower() if user_input == 'q' or user_input == 'quit': return (initial_results, all_retry_results, enabled_pixel_tests_in_retry) def _run_test_once(self, tests_to_run, tests_to_skip, should_retry_failures): enabled_pixel_tests_in_retry = False num_workers = self._port.num_workers(int( self._options.child_processes)) initial_results = self._run_tests(tests_to_run, tests_to_skip, self._options.repeat_each, self._options.iterations, num_workers) # Don't retry failures when interrupted by user or failures limit exception. should_retry_failures = should_retry_failures and not ( initial_results.interrupted or initial_results.keyboard_interrupted) tests_to_retry = self._tests_to_retry(initial_results) all_retry_results = [] if should_retry_failures and tests_to_retry: enabled_pixel_tests_in_retry = self._force_pixel_tests_if_needed() for retry_attempt in xrange(1, self._options.num_retries + 1): if not tests_to_retry: break _log.info('') _log.info( 'Retrying %s, attempt %d of %d...', grammar.pluralize('unexpected failure', len(tests_to_retry)), retry_attempt, self._options.num_retries) retry_results = self._run_tests(tests_to_retry, tests_to_skip=set(), repeat_each=1, iterations=1, num_workers=num_workers, retry_attempt=retry_attempt) all_retry_results.append(retry_results) tests_to_retry = self._tests_to_retry(retry_results) if enabled_pixel_tests_in_retry: self._options.pixel_tests = False return (initial_results, all_retry_results, enabled_pixel_tests_in_retry) def _collect_tests(self, args): return self._finder.find_tests( args, test_list=self._options.test_list, fastest_percentile=self._options.fastest) def _is_http_test(self, test): return (test.startswith(self.HTTP_SUBDIR + self._port.TEST_PATH_SEPARATOR) or self._is_websocket_test(test) or self._port.TEST_PATH_SEPARATOR + self.HTTP_SUBDIR + self._port.TEST_PATH_SEPARATOR in test) def _is_websocket_test(self, test): if self._port.should_use_wptserve(test): return False return self.WEBSOCKET_SUBDIR + self._port.TEST_PATH_SEPARATOR in test def _http_tests(self, test_names): return set(test for test in test_names if self._is_http_test(test)) def _is_perf_test(self, test): return self.PERF_SUBDIR == test or ( self.PERF_SUBDIR + self._port.TEST_PATH_SEPARATOR) in test def _prepare_lists(self, paths, test_names): tests_to_skip = self._finder.skip_tests(paths, test_names, self._expectations, self._http_tests(test_names)) tests_to_run = [ test for test in test_names if test not in tests_to_skip ] return tests_to_run, tests_to_skip def _test_input_for_file(self, test_file): return TestInput( test_file, self._options.slow_time_out_ms if self._test_is_slow(test_file) else self._options.time_out_ms, self._test_requires_lock(test_file)) def _test_requires_lock(self, test_file): """Returns True if the test needs to be locked when running multiple instances of this test runner. Perf tests are locked because heavy load caused by running other tests in parallel might cause some of them to time out. """ return self._is_http_test(test_file) or self._is_perf_test(test_file) def _test_is_slow(self, test_file): expectations = self._expectations.model().get_expectations(test_file) return (test_expectations.SLOW in expectations or self._port.is_slow_wpt_test(test_file)) def _needs_servers(self, test_names): return any( self._test_requires_lock(test_name) for test_name in test_names) def _rename_results_folder(self): try: timestamp = time.strftime( "%Y-%m-%d-%H-%M-%S", time.localtime( self._filesystem.mtime( self._filesystem.join(self._results_directory, 'results.html')))) except (IOError, OSError) as error: # It might be possible that results.html was not generated in previous run, because the test # run was interrupted even before testing started. In those cases, don't archive the folder. # Simply override the current folder contents with new results. import errno if error.errno in (errno.EEXIST, errno.ENOENT): self._printer.write_update( 'No results.html file found in previous run, skipping it.') return None archived_name = ''.join( (self._filesystem.basename(self._results_directory), '_', timestamp)) archived_path = self._filesystem.join( self._filesystem.dirname(self._results_directory), archived_name) self._filesystem.move(self._results_directory, archived_path) def _delete_dirs(self, dir_list): for dir_path in dir_list: self._filesystem.rmtree(dir_path) def _limit_archived_results_count(self): results_directory_path = self._filesystem.dirname( self._results_directory) file_list = self._filesystem.listdir(results_directory_path) results_directories = [] for name in file_list: file_path = self._filesystem.join(results_directory_path, name) if self._filesystem.isdir( file_path) and self._results_directory in file_path: results_directories.append(file_path) results_directories.sort(key=self._filesystem.mtime) self._printer.write_update('Clobbering excess archived results in %s' % results_directory_path) self._delete_dirs(results_directories[:-self.ARCHIVED_RESULTS_LIMIT]) def _set_up_run(self, test_names): self._printer.write_update('Checking build ...') if self._options.build: exit_code = self._port.check_build(self._needs_servers(test_names), self._printer) if exit_code: _log.error('Build check failed') return exit_code if self._options.clobber_old_results: self._clobber_old_results() elif self._filesystem.exists(self._results_directory): self._limit_archived_results_count() # Rename the existing results folder for archiving. self._rename_results_folder() # Create the output directory if it doesn't already exist. self._port.host.filesystem.maybe_make_directory( self._results_directory) exit_code = self._port.setup_test_run() if exit_code: _log.error('Build setup failed') return exit_code # Check that the system dependencies (themes, fonts, ...) are correct. if not self._options.nocheck_sys_deps: self._printer.write_update('Checking system dependencies ...') exit_code = self._port.check_sys_deps( self._needs_servers(test_names)) if exit_code: return exit_code return exit_codes.OK_EXIT_STATUS def _run_tests(self, tests_to_run, tests_to_skip, repeat_each, iterations, num_workers, retry_attempt=0): test_inputs = [] for _ in xrange(iterations): # TODO(crbug.com/650747): We may want to switch the two loops below # to make the behavior consistent with gtest runner (--gtest_repeat # is an alias for --repeat-each now), which looks like "ABCABCABC". # And remember to update the help text when we do so. for test in tests_to_run: for _ in xrange(repeat_each): test_inputs.append(self._test_input_for_file(test)) return self._runner.run_tests(self._expectations, test_inputs, tests_to_skip, num_workers, retry_attempt) def _start_servers(self, tests_to_run): if any(self._port.is_wpt_test(test) for test in tests_to_run): self._printer.write_update('Starting WPTServe ...') self._port.start_wptserve() self._wptserve_started = True if self._port.requires_http_server() or any( self._is_http_test(test) for test in tests_to_run): self._printer.write_update('Starting HTTP server ...') self._port.start_http_server( additional_dirs={}, number_of_drivers=self._options.max_locked_shards) self._http_server_started = True if any(self._is_websocket_test(test) for test in tests_to_run): self._printer.write_update('Starting WebSocket server ...') self._port.start_websocket_server() self._websockets_server_started = True def _stop_servers(self): if self._wptserve_started: self._printer.write_update('Stopping WPTServe ...') self._wptserve_started = False self._port.stop_wptserve() if self._http_server_started: self._printer.write_update('Stopping HTTP server ...') self._http_server_started = False self._port.stop_http_server() if self._websockets_server_started: self._printer.write_update('Stopping WebSocket server ...') self._websockets_server_started = False self._port.stop_websocket_server() def _clean_up_run(self): _log.debug('Flushing stdout') sys.stdout.flush() _log.debug('Flushing stderr') sys.stderr.flush() _log.debug('Cleaning up port') self._port.clean_up_test_run() def _force_pixel_tests_if_needed(self): if self._options.pixel_tests: return False self._options.pixel_tests = True return True def _look_for_new_crash_logs(self, run_results, start_time): """Looks for and writes new crash logs, at the end of the test run. Since crash logs can take a long time to be written out if the system is under stress, do a second pass at the end of the test run. Args: run_results: The results of the test run. start_time: Time the tests started at. We're looking for crash logs after that time. """ crashed_processes = [] for test, result in run_results.unexpected_results_by_name.iteritems(): if result.type != test_expectations.CRASH: continue for failure in result.failures: if not isinstance(failure, test_failures.FailureCrash): continue if failure.has_log: continue crashed_processes.append( [test, failure.process_name, failure.pid]) sample_files = self._port.look_for_new_samples(crashed_processes, start_time) if sample_files: for test, sample_file in sample_files.iteritems(): writer = TestResultWriter(self._filesystem, self._port, self._port.results_directory(), test) writer.copy_sample_file(sample_file) crash_logs = self._port.look_for_new_crash_logs( crashed_processes, start_time) if crash_logs: for test, (crash_log, crash_site) in crash_logs.iteritems(): writer = TestResultWriter(self._filesystem, self._port, self._port.results_directory(), test) writer.write_crash_log(crash_log) run_results.unexpected_results_by_name[ test].crash_site = crash_site def _clobber_old_results(self): dir_above_results_path = self._filesystem.dirname( self._results_directory) self._printer.write_update('Clobbering old results in %s.' % dir_above_results_path) if not self._filesystem.exists(dir_above_results_path): return file_list = self._filesystem.listdir(dir_above_results_path) results_directories = [] for name in file_list: file_path = self._filesystem.join(dir_above_results_path, name) if self._filesystem.isdir( file_path) and self._results_directory in file_path: results_directories.append(file_path) self._delete_dirs(results_directories) # Port specific clean-up. self._port.clobber_old_port_specific_results() def _tests_to_retry(self, run_results): # TODO(ojan): This should also check that result.type != test_expectations.MISSING # since retrying missing expectations is silly. But that's a bit tricky since we # only consider the last retry attempt for the count of unexpected regressions. return [ result.test_name for result in run_results.unexpected_results_by_name.values() if result.type != test_expectations.PASS ] def _write_json_files(self, summarized_full_results, summarized_failing_results, initial_results, running_all_tests): _log.debug("Writing JSON files in %s.", self._results_directory) # FIXME: Upload stats.json to the server and delete times_ms. times_trie = json_results_generator.test_timings_trie( initial_results.results_by_name.values()) times_json_path = self._filesystem.join(self._results_directory, 'times_ms.json') json_results_generator.write_json(self._filesystem, times_trie, times_json_path) # Save out the times data so we can use it for --fastest in the future. if running_all_tests: bot_test_times_path = self._port.bot_test_times_path() self._filesystem.maybe_make_directory( self._filesystem.dirname(bot_test_times_path)) json_results_generator.write_json(self._filesystem, times_trie, bot_test_times_path) stats_trie = self._stats_trie(initial_results) stats_path = self._filesystem.join(self._results_directory, 'stats.json') self._filesystem.write_text_file(stats_path, json.dumps(stats_trie)) full_results_path = self._filesystem.join(self._results_directory, 'full_results.json') json_results_generator.write_json(self._filesystem, summarized_full_results, full_results_path) full_results_jsonp_path = self._filesystem.join( self._results_directory, 'full_results_jsonp.js') json_results_generator.write_json(self._filesystem, summarized_full_results, full_results_jsonp_path, callback='ADD_FULL_RESULTS') full_results_path = self._filesystem.join(self._results_directory, 'failing_results.json') # We write failing_results.json out as jsonp because we need to load it # from a file url for results.html and Chromium doesn't allow that. json_results_generator.write_json(self._filesystem, summarized_failing_results, full_results_path, callback='ADD_RESULTS') # Write out the JSON files suitable for other tools to process. # As the output can be quite large (as there are 60k+ tests) we also # support only outputting the failing results. if self._options.json_failing_test_results: # FIXME(tansell): Make sure this includes an *unexpected* results # (IE Passing when expected to be failing.) json_results_generator.write_json( self._filesystem, summarized_failing_results, self._options.json_failing_test_results) if self._options.json_test_results: json_results_generator.write_json(self._filesystem, summarized_full_results, self._options.json_test_results) _log.debug('Finished writing JSON files.') def _upload_json_files(self): if not self._options.test_results_server: return if not self._options.master_name: _log.error( '--test-results-server was set, but --master-name was not. Not uploading JSON files.' ) return _log.debug('Uploading JSON files for builder: %s', self._options.builder_name) attrs = [('builder', self._options.builder_name), ('testtype', self._options.step_name), ('master', self._options.master_name)] files = [ (name, self._filesystem.join(self._results_directory, name)) for name in ['failing_results.json', 'full_results.json', 'times_ms.json'] ] url = 'https://%s/testfile/upload' % self._options.test_results_server # Set uploading timeout in case appengine server is having problems. # 120 seconds are more than enough to upload test results. uploader = FileUploader(url, 120) try: response = uploader.upload_as_multipart_form_data( self._filesystem, files, attrs) if response: if response.code == 200: _log.debug('JSON uploaded.') else: _log.debug('JSON upload failed, %d: "%s"', response.code, response.read()) else: _log.error('JSON upload failed; no response returned') except IOError as err: _log.error('Upload failed: %s', err) def _copy_results_html_file(self, destination_dir, filename): """Copies a file from the template directory to the results directory.""" template_dir = self._path_finder.path_from_layout_tests( 'fast', 'harness') source_path = self._filesystem.join(template_dir, filename) destination_path = self._filesystem.join(destination_dir, filename) # Note that the results.html template file won't exist when # we're using a MockFileSystem during unit tests, so make sure # it exists before we try to copy it. if self._filesystem.exists(source_path): self._filesystem.copyfile(source_path, destination_path) def _stats_trie(self, initial_results): def _worker_number(worker_name): return int(worker_name.split('/')[1]) if worker_name else -1 stats = {} for result in initial_results.results_by_name.values(): if result.type != test_expectations.SKIP: stats[result.test_name] = { 'results': (_worker_number(result.worker_name), result.test_number, result.pid, int(result.test_run_time * 1000), int(result.total_run_time * 1000)) } stats_trie = {} for name, value in stats.iteritems(): json_results_generator.add_path_to_trie(name, value, stats_trie) return stats_trie
class ImportNotifier(object): def __init__(self, host, chromium_git, local_wpt): self.host = host self.git = chromium_git self.local_wpt = local_wpt self.default_port = host.port_factory.get() self.finder = PathFinder(host.filesystem) self.owners_extractor = DirectoryOwnersExtractor(host.filesystem) self.new_failures_by_directory = defaultdict(list) def main(self, wpt_revision_start, wpt_revision_end, rebaselined_tests, test_expectations, issue, patchset, dry_run=True, service_account_key_json=None): """Files bug reports for new failures. Args: wpt_revision_start: The start of the imported WPT revision range (exclusive), i.e. the last imported revision. wpt_revision_end: The end of the imported WPT revision range (inclusive), i.e. the current imported revision. rebaselined_tests: A list of test names that have been rebaselined. test_expectations: A dictionary mapping names of tests that cannot be rebaselined to a list of new test expectation lines. issue: The issue number of the import CL (a string). patchset: The patchset number of the import CL (a string). dry_run: If True, no bugs will be actually filed to crbug.com. service_account_key_json: The path to a JSON private key of a service account for accessing Monorail. If None, try to load from the default location, i.e. the path stored in the environment variable GOOGLE_APPLICATION_CREDENTIALS. Note: "test names" are paths of the tests relative to LayoutTests. """ gerrit_url = SHORT_GERRIT_PREFIX + issue gerrit_url_with_ps = gerrit_url + '/' + patchset + '/' changed_test_baselines = self.find_changed_baselines_of_tests(rebaselined_tests) self.examine_baseline_changes(changed_test_baselines, gerrit_url_with_ps) self.examine_new_test_expectations(test_expectations) bugs = self.create_bugs_from_new_failures(wpt_revision_start, wpt_revision_end, gerrit_url) self.file_bugs(bugs, dry_run, service_account_key_json) def find_changed_baselines_of_tests(self, rebaselined_tests): """Finds the corresponding changed baselines of each test. Args: rebaselined_tests: A list of test names that have been rebaselined. Returns: A dictionary mapping test names to paths of their baselines changed in this import CL (paths relative to the root of Chromium repo). """ test_baselines = {} changed_files = self.git.changed_files() for test_name in rebaselined_tests: test_without_ext, _ = self.host.filesystem.splitext(test_name) changed_baselines = [] # TODO(robertma): Refactor this into layout_tests.port.base. baseline_name = test_without_ext + '-expected.txt' for changed_file in changed_files: if changed_file.endswith(baseline_name): changed_baselines.append(changed_file) if changed_baselines: test_baselines[test_name] = changed_baselines return test_baselines def examine_baseline_changes(self, changed_test_baselines, gerrit_url_with_ps): """Examines all changed baselines to find new failures. Args: changed_test_baselines: A dictionary mapping test names to paths of changed baselines. gerrit_url_with_ps: Gerrit URL of this CL with the patchset number. """ for test_name, changed_baselines in changed_test_baselines.iteritems(): directory = self.find_owned_directory(test_name) if not directory: _log.warning('Cannot find OWNERS of %s', test_name) continue for baseline in changed_baselines: if self.more_failures_in_baseline(baseline): self.new_failures_by_directory[directory].append( TestFailure(TestFailure.BASELINE_CHANGE, test_name, baseline_path=baseline, gerrit_url_with_ps=gerrit_url_with_ps) ) def more_failures_in_baseline(self, baseline): diff = self.git.run(['diff', '-U0', 'origin/master', '--', baseline]) delta_failures = 0 for line in diff.splitlines(): if line.startswith('+FAIL'): delta_failures += 1 if line.startswith('-FAIL'): delta_failures -= 1 return delta_failures > 0 def examine_new_test_expectations(self, test_expectations): """Examines new test expectations to find new failures. Args: test_expectations: A dictionary mapping names of tests that cannot be rebaselined to a list of new test expectation lines. """ for test_name, expectation_lines in test_expectations.iteritems(): directory = self.find_owned_directory(test_name) if not directory: _log.warning('Cannot find OWNERS of %s', test_name) continue for expectation_line in expectation_lines: self.new_failures_by_directory[directory].append( TestFailure(TestFailure.NEW_EXPECTATION, test_name, expectation_line=expectation_line) ) def create_bugs_from_new_failures(self, wpt_revision_start, wpt_revision_end, gerrit_url): """Files bug reports for new failures. Args: wpt_revision_start: The start of the imported WPT revision range (exclusive), i.e. the last imported revision. wpt_revision_end: The end of the imported WPT revision range (inclusive), i.e. the current imported revision. gerrit_url: Gerrit URL of the CL. Return: A list of MonorailIssue objects that should be filed. """ imported_commits = self.local_wpt.commits_in_range(wpt_revision_start, wpt_revision_end) bugs = [] for directory, failures in self.new_failures_by_directory.iteritems(): summary = '[WPT] New failures introduced in {} by import {}'.format(directory, gerrit_url) full_directory = self.host.filesystem.join(self.finder.layout_tests_dir(), directory) owners_file = self.host.filesystem.join(full_directory, 'OWNERS') is_wpt_notify_enabled = self.owners_extractor.is_wpt_notify_enabled(owners_file) owners = self.owners_extractor.extract_owners(owners_file) # owners may be empty but not None. cc = owners + ['*****@*****.**'] component = self.owners_extractor.extract_component(owners_file) # component could be None. components = [component] if component else None prologue = ('WPT import {} introduced new failures in {}:\n\n' 'List of new failures:\n'.format(gerrit_url, directory)) failure_list = '' for failure in failures: failure_list += str(failure) + '\n' epilogue = '\nThis import contains upstream changes from {} to {}:\n'.format( wpt_revision_start, wpt_revision_end ) commit_list = self.format_commit_list(imported_commits, full_directory) description = prologue + failure_list + epilogue + commit_list bug = MonorailIssue.new_chromium_issue(summary, description, cc, components) _log.info(str(bug)) if is_wpt_notify_enabled: _log.info("WPT-NOTIFY enabled in this directory; adding the bug to the pending list.") bugs.append(bug) else: _log.info("WPT-NOTIFY disabled in this directory; discarding the bug.") return bugs def format_commit_list(self, imported_commits, directory): """Formats the list of imported WPT commits. Imports affecting the given directory will be highlighted. Args: imported_commits: A list of (SHA, commit subject) pairs. directory: An absolute path of a directory in the Chromium repo, for which the list is formatted. Returns: A multi-line string. """ path_from_wpt = self.host.filesystem.relpath( directory, self.finder.path_from_layout_tests('external', 'wpt')) commit_list = '' for sha, subject in imported_commits: line = '{}: {}'.format(subject, GITHUB_COMMIT_PREFIX + sha) if self.local_wpt.is_commit_affecting_directory(sha, path_from_wpt): line += ' [affecting this directory]' commit_list += line + '\n' return commit_list def find_owned_directory(self, test_name): """Finds the lowest directory that contains the test and has OWNERS. Args: The name of the test (a path relative to LayoutTests). Returns: The path of the found directory relative to LayoutTests. """ # Always use non-virtual test names when looking up OWNERS. if self.default_port.lookup_virtual_test_base(test_name): test_name = self.default_port.lookup_virtual_test_base(test_name) # find_owners_file takes either a relative path from the *root* of the # repository, or an absolute path. abs_test_path = self.finder.path_from_layout_tests(test_name) owners_file = self.owners_extractor.find_owners_file(self.host.filesystem.dirname(abs_test_path)) if not owners_file: return None owned_directory = self.host.filesystem.dirname(owners_file) short_directory = self.host.filesystem.relpath(owned_directory, self.finder.layout_tests_dir()) return short_directory def file_bugs(self, bugs, dry_run, service_account_key_json=None): """Files a list of bugs to Monorail. Args: bugs: A list of MonorailIssue objects. dry_run: A boolean, whether we are in dry run mode. service_account_key_json: Optional, see docs for main(). """ # TODO(robertma): Better error handling in this method. if dry_run: _log.info('[dry_run] Would have filed the %d bugs in the pending list.', len(bugs)) return _log.info('Filing %d bugs in the pending list to Monorail', len(bugs)) api = self._get_monorail_api(service_account_key_json) for index, bug in enumerate(bugs, start=1): response = api.insert_issue(bug) _log.info('[%d] Filed bug: %s', index, MonorailIssue.crbug_link(response['id'])) def _get_monorail_api(self, service_account_key_json): return MonorailAPI(service_account_key_json=service_account_key_json)
class TestImporter(object): def __init__(self, host, wpt_github=None): self.host = host self.executive = host.executive self.fs = host.filesystem self.finder = PathFinder(self.fs) self.verbose = False self.git_cl = None self.wpt_github = wpt_github def main(self, argv=None): options = self.parse_args(argv) self.verbose = options.verbose log_level = logging.DEBUG if self.verbose else logging.INFO logging.basicConfig(level=log_level, format='%(message)s') if not self.checkout_is_okay(): return 1 credentials = read_credentials(self.host, options.credentials_json) gh_user = credentials.get('GH_USER') gh_token = credentials.get('GH_TOKEN') self.wpt_github = self.wpt_github or WPTGitHub(self.host, gh_user, gh_token) local_wpt = LocalWPT(self.host, gh_token=gh_token) self.git_cl = GitCL( self.host, auth_refresh_token_json=options.auth_refresh_token_json) _log.debug('Noting the current Chromium commit.') # TODO(qyearsley): Use Git (self.host.git) to run git commands. _, show_ref_output = self.run(['git', 'show-ref', 'HEAD']) chromium_commit = show_ref_output.split()[0] local_wpt.fetch() if options.revision is not None: _log.info('Checking out %s', options.revision) self.run(['git', 'checkout', options.revision], cwd=local_wpt.path) _log.info('Noting the revision we are importing.') _, show_ref_output = self.run(['git', 'show-ref', 'origin/master'], cwd=local_wpt.path) import_commit = 'wpt@%s' % show_ref_output.split()[0] commit_message = self._commit_message(chromium_commit, import_commit) if not options.ignore_exportable_commits: commits = self.apply_exportable_commits_locally(local_wpt) if commits is None: _log.error('Could not apply some exportable commits cleanly.') _log.error('Aborting import to prevent clobbering commits.') return 1 commit_message = self._commit_message( chromium_commit, import_commit, locally_applied_commits=commits) dest_path = self.finder.path_from_layout_tests('external', 'wpt') self._clear_out_dest_path(dest_path) _log.info('Copying the tests from the temp repo to the destination.') test_copier = TestCopier(self.host, local_wpt.path) test_copier.do_import() self.run(['git', 'add', '--all', 'external/wpt']) self._delete_orphaned_baselines(dest_path) # TODO(qyearsley): Consider updating manifest after adding baselines. self._generate_manifest(dest_path) # TODO(qyearsley): Consider running the imported tests with # `run-webkit-tests --reset-results external/wpt` to get some baselines # before the try jobs are started. _log.info( 'Updating TestExpectations for any removed or renamed tests.') self.update_all_test_expectations_files(self._list_deleted_tests(), self._list_renamed_tests()) has_changes = self._has_changes() if not has_changes: _log.info('Done: no changes to import.') return 0 self.run(['git', 'commit', '--all', '-F', '-'], stdin=commit_message) self._commit_changes(commit_message) _log.info('Changes imported and committed.') if not options.auto_update: return 0 self._upload_cl() _log.info('Issue: %s', self.git_cl.run(['issue']).strip()) if not self.update_expectations_for_cl(): return 1 if not self.run_commit_queue_for_cl(): return 1 return 0 def update_expectations_for_cl(self): """Performs the expectation-updating part of an auto-import job. This includes triggering try jobs and waiting; then, if applicable, writing new baselines and TestExpectation lines, committing, and uploading a new patchset. This assumes that there is CL associated with the current branch. Returns True if everything is OK to continue, or False on failure. """ _log.info('Triggering try jobs for updating expectations.') self.git_cl.trigger_try_jobs() try_results = self.git_cl.wait_for_try_jobs( poll_delay_seconds=POLL_DELAY_SECONDS, timeout_seconds=TIMEOUT_SECONDS) if not try_results: _log.error('No initial try job results, aborting.') self.git_cl.run(['set-close']) return False if try_results and self.git_cl.some_failed(try_results): self.fetch_new_expectations_and_baselines() if self.host.git().has_working_directory_changes(): message = 'Update test expectations and baselines.' self.check_run(['git', 'commit', '-a', '-m', message]) self._upload_patchset(message) return True def run_commit_queue_for_cl(self): """Triggers CQ and either commits or aborts; returns True on success.""" _log.info('Triggering CQ try jobs.') self.git_cl.run(['try']) try_results = self.git_cl.wait_for_try_jobs( poll_delay_seconds=POLL_DELAY_SECONDS, timeout_seconds=TIMEOUT_SECONDS) try_results = self.git_cl.filter_latest(try_results) if not try_results: self.git_cl.run(['set-close']) _log.error('No CQ try job results, aborting.') return False if try_results and self.git_cl.all_success(try_results): _log.info('CQ appears to have passed; trying to commit.') self.git_cl.run(['upload', '-f', '--send-mail']) # Turn off WIP mode. self.git_cl.run(['set-commit']) _log.info('Update completed.') return True self.git_cl.run(['set-close']) _log.error('CQ appears to have failed; aborting.') return False def parse_args(self, argv): parser = argparse.ArgumentParser() parser.description = __doc__ parser.add_argument( '-v', '--verbose', action='store_true', help='log extra details that may be helpful when debugging') parser.add_argument( '--ignore-exportable-commits', action='store_true', help='do not check for exportable commits that would be clobbered') parser.add_argument('-r', '--revision', help='target wpt revision') parser.add_argument( '--auto-update', action='store_true', help='upload a CL, update expectations, and trigger CQ') parser.add_argument( '--auth-refresh-token-json', help='authentication refresh token JSON file used for try jobs, ' 'generally not necessary on developer machines') parser.add_argument('--credentials-json', help='A JSON file with GitHub credentials, ' 'generally not necessary on developer machines') return parser.parse_args(argv) def checkout_is_okay(self): git_diff_retcode, _ = self.run(['git', 'diff', '--quiet', 'HEAD'], exit_on_failure=False) if git_diff_retcode: _log.warning('Checkout is dirty; aborting.') return False _, local_commits = self.run( ['git', 'log', '--oneline', 'origin/master..HEAD']) if local_commits: _log.warning('Checkout has local commits before import.') return True def apply_exportable_commits_locally(self, local_wpt): """Applies exportable Chromium changes to the local WPT repo. The purpose of this is to avoid clobbering changes that were made in Chromium but not yet merged upstream. By applying these changes to the local copy of web-platform-tests before copying files over, we make it so that the resulting change in Chromium doesn't undo the previous Chromium change. Args: A LocalWPT instance for our local copy of WPT. Returns: A list of commits applied (could be empty), or None if any of the patches could not be applied cleanly. """ commits = self.exportable_but_not_exported_commits(local_wpt) for commit in commits: _log.info('Applying exportable commit locally:') _log.info(commit.url()) _log.info('Subject: %s', commit.subject().strip()) # TODO(qyearsley): We probably don't need to know about # corresponding PRs at all anymore, although this information # could still be useful for reference. pull_request = self.wpt_github.pr_for_chromium_commit(commit) if pull_request: _log.info( 'PR: https://github.com/w3c/web-platform-tests/pull/%d', pull_request.number) else: _log.warning('No pull request found.') applied = local_wpt.apply_patch(commit.format_patch()) if not applied: return None self.run(['git', 'commit', '--all', '-F', '-'], stdin='Applying patch %s' % commit.sha, cwd=local_wpt.path) return commits def exportable_but_not_exported_commits(self, local_wpt): """Returns a list of commits that would be clobbered by importer.""" return exportable_commits_over_last_n_commits(self.host, local_wpt, self.wpt_github) def _generate_manifest(self, dest_path): """Generates MANIFEST.json for imported tests. Args: dest_path: Path to the destination WPT directory. Runs the (newly-updated) manifest command if it's found, and then stages the generated MANIFEST.json in the git index, ready to commit. """ _log.info('Generating MANIFEST.json') WPTManifest.generate_manifest(self.host, dest_path) manifest_path = self.fs.join(dest_path, 'MANIFEST.json') assert self.fs.exists(manifest_path) manifest_base_path = self.fs.normpath( self.fs.join(dest_path, '..', 'WPT_BASE_MANIFEST.json')) self.copyfile(manifest_path, manifest_base_path) self.run(['git', 'add', manifest_base_path]) def _clear_out_dest_path(self, dest_path): _log.info('Cleaning out tests from %s.', dest_path) should_remove = lambda fs, dirname, basename: ( not self.is_baseline(basename) and # See http://crbug.com/702283 for context. basename != 'OWNERS') files_to_delete = self.fs.files_under(dest_path, file_filter=should_remove) for subpath in files_to_delete: self.remove(self.finder.path_from_layout_tests( 'external', subpath)) def _commit_changes(self, commit_message): _log.info('Committing changes.') self.run(['git', 'commit', '--all', '-F', '-'], stdin=commit_message) def _has_changes(self): return_code, _ = self.run(['git', 'diff', '--quiet', 'HEAD'], exit_on_failure=False) return return_code == 1 def _commit_message(self, chromium_commit_sha, import_commit_sha, locally_applied_commits=None): message = 'Import {}\n\nUsing wpt-import in Chromium {}.\n'.format( import_commit_sha, chromium_commit_sha) if locally_applied_commits is not None: message += 'With Chromium commits locally applied on WPT:\n' message += '\n'.join( str(commit) for commit in locally_applied_commits) message += '\nNo-Export: true' return message def _delete_orphaned_baselines(self, dest_path): _log.info('Deleting any orphaned baselines.') is_baseline_filter = lambda fs, dirname, basename: self.is_baseline( basename) previous_baselines = self.fs.files_under( dest_path, file_filter=is_baseline_filter) for sub_path in previous_baselines: full_baseline_path = self.fs.join(dest_path, sub_path) if not self._has_corresponding_test(full_baseline_path): self.fs.remove(full_baseline_path) def _has_corresponding_test(self, full_baseline_path): base = full_baseline_path.replace('-expected.txt', '') return any( self.fs.exists(base + ext) for ext in Port.supported_file_extensions) @staticmethod def is_baseline(basename): # TODO(qyearsley): Find a better, centralized place for this. # Also, the name for this method should be is_text_baseline. return basename.endswith('-expected.txt') def run(self, cmd, exit_on_failure=True, cwd=None, stdin=''): _log.debug('Running command: %s', ' '.join(cmd)) cwd = cwd or self.finder.path_from_layout_tests() proc = self.executive.popen(cmd, stdout=self.executive.PIPE, stderr=self.executive.PIPE, stdin=self.executive.PIPE, cwd=cwd) out, err = proc.communicate(stdin) if proc.returncode or self.verbose: _log.info('# ret> %d', proc.returncode) if out: for line in out.splitlines(): _log.info('# out> %s', line) if err: for line in err.splitlines(): _log.info('# err> %s', line) if exit_on_failure and proc.returncode: self.host.exit(proc.returncode) return proc.returncode, out def check_run(self, command): return_code, out = self.run(command) if return_code: raise Exception('%s failed with exit code %d.' % ' '.join(command), return_code) return out def copyfile(self, source, destination): _log.debug('cp %s %s', source, destination) self.fs.copyfile(source, destination) def remove(self, dest): _log.debug('rm %s', dest) self.fs.remove(dest) def _upload_patchset(self, message): self.git_cl.run(['upload', '-f', '-t', message, '--gerrit']) def _upload_cl(self): _log.info('Uploading change list.') directory_owners = self.get_directory_owners() description = self._cl_description(directory_owners) self.git_cl.run([ 'upload', '-f', '--gerrit', '-m', description, '--tbrs', '*****@*****.**', ] + self._cc_part(directory_owners)) @staticmethod def _cc_part(directory_owners): cc_part = [] for owner_tuple in sorted(directory_owners): cc_part.extend('--cc=' + owner for owner in owner_tuple) return cc_part def get_directory_owners(self): """Returns a mapping of email addresses to owners of changed tests.""" _log.info('Gathering directory owners emails to CC.') changed_files = self.host.git().changed_files() extractor = DirectoryOwnersExtractor(self.fs) extractor.read_owner_map() return extractor.list_owners(changed_files) def _cl_description(self, directory_owners): """Returns a CL description string. Args: directory_owners: A dict of tuples of owner names to lists of directories. """ description = self.check_run(['git', 'log', '-1', '--format=%B']) build_link = current_build_link(self.host) if build_link: description += 'Build: %s\n\n' % build_link description += ( 'Note to sheriffs: This CL imports external tests and adds\n' 'expectations for those tests; if this CL is large and causes\n' 'a few new failures, please fix the failures by adding new\n' 'lines to TestExpectations rather than reverting. See:\n' 'https://chromium.googlesource.com' '/chromium/src/+/master/docs/testing/web_platform_tests.md\n\n') if directory_owners: description += self._format_directory_owners( directory_owners) + '\n\n' # Move any No-Export tag to the end of the description. description = description.replace('No-Export: true', '') description = description.replace('\n\n\n\n', '\n\n') description += 'No-Export: true' return description @staticmethod def _format_directory_owners(directory_owners): message_lines = ['Directory owners for changes in this CL:'] for owner_tuple, directories in sorted(directory_owners.items()): message_lines.append(', '.join(owner_tuple) + ':') message_lines.extend(' ' + d for d in directories) return '\n'.join(message_lines) def fetch_new_expectations_and_baselines(self): """Modifies expectation lines and baselines based on try job results. Assuming that there are some try job results available, this adds new expectation lines to TestExpectations and downloads new baselines based on the try job results. This is the same as invoking the `wpt-update-expectations` script. """ _log.info( 'Adding test expectations lines to LayoutTests/TestExpectations.') expectation_updater = WPTExpectationsUpdater(self.host) expectation_updater.run(args=[]) def update_all_test_expectations_files(self, deleted_tests, renamed_tests): """Updates all test expectations files for tests that have been deleted or renamed. This is only for deleted or renamed tests in the initial import, not for tests that have failures in try jobs. """ port = self.host.port_factory.get() for path, file_contents in port.all_expectations_dict().iteritems(): parser = TestExpectationParser(port, all_tests=None, is_lint_mode=False) expectation_lines = parser.parse(path, file_contents) self._update_single_test_expectations_file(path, expectation_lines, deleted_tests, renamed_tests) def _update_single_test_expectations_file(self, path, expectation_lines, deleted_tests, renamed_tests): """Updates a single test expectations file.""" # FIXME: This won't work for removed or renamed directories with test # expectations that are directories rather than individual tests. new_lines = [] changed_lines = [] for expectation_line in expectation_lines: if expectation_line.name in deleted_tests: continue if expectation_line.name in renamed_tests: expectation_line.name = renamed_tests[expectation_line.name] # Upon parsing the file, a "path does not exist" warning is expected # to be there for tests that have been renamed, and if there are warnings, # then the original string is used. If the warnings are reset, then the # expectation line is re-serialized when output. expectation_line.warnings = [] changed_lines.append(expectation_line) new_lines.append(expectation_line) new_file_contents = TestExpectations.list_to_string( new_lines, reconstitute_only_these=changed_lines) self.host.filesystem.write_text_file(path, new_file_contents) def _list_deleted_tests(self): """List of layout tests that have been deleted.""" out = self.check_run([ 'git', 'diff', 'origin/master', '-M100%', '--diff-filter=D', '--name-only' ]) deleted_tests = [] for line in out.splitlines(): test = self.finder.layout_test_name(line) if test: deleted_tests.append(test) return deleted_tests def _list_renamed_tests(self): """Lists tests that have been renamed. Returns a dict mapping source name to destination name. """ out = self.check_run([ 'git', 'diff', 'origin/master', '-M100%', '--diff-filter=R', '--name-status' ]) renamed_tests = {} for line in out.splitlines(): _, source_path, dest_path = line.split() source_test = self.finder.layout_test_name(source_path) dest_test = self.finder.layout_test_name(dest_path) if source_test and dest_test: renamed_tests[source_test] = dest_test return renamed_tests
class TestImporter(object): def __init__(self, host): self.host = host self.executive = host.executive self.fs = host.filesystem self.finder = PathFinder(self.fs) self.verbose = False self.git_cl = None def main(self, argv=None): options = self.parse_args(argv) self.verbose = options.verbose log_level = logging.DEBUG if self.verbose else logging.INFO logging.basicConfig(level=log_level, format='%(message)s') if not self.checkout_is_okay(options.allow_local_commits): return 1 self.git_cl = GitCL(self.host, auth_refresh_token_json=options.auth_refresh_token_json) _log.debug('Noting the current Chromium commit.') _, show_ref_output = self.run(['git', 'show-ref', 'HEAD']) chromium_commit = show_ref_output.split()[0] dest_dir_name = WPT_DEST_NAME repo_url = WPT_REPO_URL # TODO(qyearsley): Simplify this to use LocalWPT.fetch when csswg-test # is merged into web-platform-tests (crbug.com/706118). temp_repo_path = self.finder.path_from_layout_tests(dest_dir_name) _log.info('Cloning repo: %s', repo_url) _log.info('Local path: %s', temp_repo_path) self.run(['git', 'clone', repo_url, temp_repo_path]) if not options.ignore_exportable_commits: commits = self.exportable_but_not_exported_commits(temp_repo_path) if commits: # If there are exportable commits, then there's no more work # to do for now. This isn't really an error case; we expect # to hit this case some of the time. _log.info('There were exportable but not-yet-exported commits:') for commit in commits: _log.info(' https://chromium.googlesource.com/chromium/src/+/%s', commit.sha) _log.info('Aborting import to prevent clobbering these commits.') self.clean_up_temp_repo(temp_repo_path) return 0 import_commit = self.update(dest_dir_name, temp_repo_path, options.revision) self.clean_up_temp_repo(temp_repo_path) self._copy_resources() has_changes = self._has_changes() if not has_changes: _log.info('Done: no changes to import.') return 0 commit_message = self._commit_message(chromium_commit, import_commit) self._commit_changes(commit_message) _log.info('Done: changes imported and committed.') if options.auto_update: commit_successful = self.do_auto_update() if not commit_successful: return 1 return 0 def parse_args(self, argv): parser = argparse.ArgumentParser() parser.description = __doc__ parser.add_argument('-v', '--verbose', action='store_true', help='log what we are doing') parser.add_argument('--allow-local-commits', action='store_true', help='allow script to run even if we have local commits') parser.add_argument('-r', dest='revision', action='store', help='Target revision.') parser.add_argument('--auto-update', action='store_true', help='uploads CL and initiates commit queue.') parser.add_argument('--auth-refresh-token-json', help='authentication refresh token JSON file, ' 'used for authentication for try jobs, ' 'generally not necessary on developer machines') parser.add_argument('--ignore-exportable-commits', action='store_true', help='Continue even if there are exportable commits that may be overwritten.') # TODO(qyearsley): Change this back to parse_args once this script # is no longer being called with the "wpt" argument. See crbug.com/706118. args, _ = parser.parse_known_args(argv) return args def checkout_is_okay(self, allow_local_commits): git_diff_retcode, _ = self.run(['git', 'diff', '--quiet', 'HEAD'], exit_on_failure=False) if git_diff_retcode: _log.warning('Checkout is dirty; aborting.') return False local_commits = self.run(['git', 'log', '--oneline', 'origin/master..HEAD'])[1] if local_commits and not allow_local_commits: _log.warning('Checkout has local commits; aborting. Use --allow-local-commits to allow this.') return False temp_repo_path = self.finder.path_from_layout_tests(WPT_DEST_NAME) if self.fs.exists(temp_repo_path): _log.warning('%s exists; aborting.', temp_repo_path) return False return True def exportable_but_not_exported_commits(self, wpt_path): """Checks for commits that might be overwritten by importing. Args: wpt_path: The path to a local checkout of web-platform-tests. Returns: A list of commits in the Chromium repo that are exportable but not yet exported to the web-platform-tests repo. """ local_wpt = LocalWPT(self.host, path=wpt_path) assert self.host.filesystem.exists(wpt_path) _, chromium_commit = local_wpt.most_recent_chromium_commit() return exportable_commits_since(chromium_commit.sha, self.host, local_wpt) def clean_up_temp_repo(self, temp_repo_path): _log.info('Deleting temp repo directory %s.', temp_repo_path) self.fs.rmtree(temp_repo_path) def _copy_resources(self): """Copies resources from wpt to LayoutTests/resources. We copy idlharness.js and testharness.js in wpt to LayoutTests/resources in order to use them in non-imported tests. If this method is changed, the lists of files expected to be identical in LayoutTests/PRESUBMIT.py should also be changed. """ resources_to_copy_from_wpt = [ ('idlharness.js', 'resources'), ('testharness.js', 'resources'), ] for filename, wpt_subdir in resources_to_copy_from_wpt: source = self.finder.path_from_layout_tests('external', WPT_DEST_NAME, wpt_subdir, filename) destination = self.finder.path_from_layout_tests('resources', filename) self.copyfile(source, destination) self.run(['git', 'add', destination]) def _generate_manifest(self, dest_path): """Generates MANIFEST.json for imported tests. Args: dest_path: Path to the destination WPT directory. Runs the (newly-updated) manifest command if it's found, and then stages the generated MANIFEST.json in the git index, ready to commit. """ _log.info('Generating MANIFEST.json') WPTManifest.generate_manifest(self.host, dest_path) manifest_path = self.fs.join(dest_path, 'MANIFEST.json') assert self.fs.exists(manifest_path) manifest_base_path = self.fs.normpath( self.fs.join(dest_path, '..', 'WPT_BASE_MANIFEST.json')) self.copyfile(manifest_path, manifest_base_path) self.run(['git', 'add', manifest_base_path]) def update(self, dest_dir_name, temp_repo_path, revision): """Updates an imported repository. Args: dest_dir_name: The destination directory name. temp_repo_path: Path to local checkout of W3C test repo. revision: A W3C test repo commit hash, or None. Returns: A string for the commit description "<destination>@<commitish>". """ if revision is not None: _log.info('Checking out %s', revision) self.run(['git', 'checkout', revision], cwd=temp_repo_path) self.run(['git', 'submodule', 'update', '--init', '--recursive'], cwd=temp_repo_path) _log.info('Noting the revision we are importing.') _, show_ref_output = self.run(['git', 'show-ref', 'origin/master'], cwd=temp_repo_path) master_commitish = show_ref_output.split()[0] dest_path = self.finder.path_from_layout_tests('external', dest_dir_name) self._clear_out_dest_path(dest_path) _log.info('Importing the tests.') test_copier = TestCopier(self.host, temp_repo_path) test_copier.do_import() self.run(['git', 'add', '--all', 'external/%s' % dest_dir_name]) self._delete_orphaned_baselines(dest_path) self._generate_manifest(dest_path) _log.info('Updating TestExpectations for any removed or renamed tests.') self.update_all_test_expectations_files(self._list_deleted_tests(), self._list_renamed_tests()) return '%s@%s' % (dest_dir_name, master_commitish) def _clear_out_dest_path(self, dest_path): _log.info('Cleaning out tests from %s.', dest_path) should_remove = lambda fs, dirname, basename: ( not self.is_baseline(basename) and # See http://crbug.com/702283 for context. basename != 'OWNERS') files_to_delete = self.fs.files_under(dest_path, file_filter=should_remove) for subpath in files_to_delete: self.remove(self.finder.path_from_layout_tests('external', subpath)) def _commit_changes(self, commit_message): _log.info('Committing changes.') self.run(['git', 'commit', '--all', '-F', '-'], stdin=commit_message) def _has_changes(self): return_code, _ = self.run(['git', 'diff', '--quiet', 'HEAD'], exit_on_failure=False) return return_code == 1 def _commit_message(self, chromium_commit, import_commit): return ('Import %s\n\n' 'Using wpt-import in Chromium %s.\n\n' 'NOEXPORT=true' % (import_commit, chromium_commit)) def _delete_orphaned_baselines(self, dest_path): _log.info('Deleting any orphaned baselines.') is_baseline_filter = lambda fs, dirname, basename: self.is_baseline(basename) previous_baselines = self.fs.files_under(dest_path, file_filter=is_baseline_filter) for sub_path in previous_baselines: full_baseline_path = self.fs.join(dest_path, sub_path) if not self._has_corresponding_test(full_baseline_path): self.fs.remove(full_baseline_path) def _has_corresponding_test(self, full_baseline_path): base = full_baseline_path.replace('-expected.txt', '') return any(self.fs.exists(base + ext) for ext in Port.supported_file_extensions) @staticmethod def is_baseline(basename): # TODO(qyearsley): Find a better, centralized place for this. # Also, the name for this method should be is_text_baseline. return basename.endswith('-expected.txt') def run(self, cmd, exit_on_failure=True, cwd=None, stdin=''): _log.debug('Running command: %s', ' '.join(cmd)) cwd = cwd or self.finder.path_from_layout_tests() proc = self.executive.popen(cmd, stdout=self.executive.PIPE, stderr=self.executive.PIPE, stdin=self.executive.PIPE, cwd=cwd) out, err = proc.communicate(stdin) if proc.returncode or self.verbose: _log.info('# ret> %d', proc.returncode) if out: for line in out.splitlines(): _log.info('# out> %s', line) if err: for line in err.splitlines(): _log.info('# err> %s', line) if exit_on_failure and proc.returncode: self.host.exit(proc.returncode) return proc.returncode, out def check_run(self, command): return_code, out = self.run(command) if return_code: raise Exception('%s failed with exit code %d.' % ' '.join(command), return_code) return out def copyfile(self, source, destination): _log.debug('cp %s %s', source, destination) self.fs.copyfile(source, destination) def remove(self, dest): _log.debug('rm %s', dest) self.fs.remove(dest) def do_auto_update(self): """Attempts to upload a CL, make any required adjustments, and commit. This function assumes that the imported repo has already been updated, and that change has been committed. There may be newly-failing tests, so before being able to commit these new changes, we may need to update TestExpectations or download new baselines. Returns: True if successfully committed, False otherwise. """ self._upload_cl() _log.info('Issue: %s', self.git_cl.run(['issue']).strip()) # First, try on Blink try bots in order to get any new baselines. # TODO(qyearsley): Make this faster by triggering all try jobs in # one invocation. _log.info('Triggering try jobs.') self.git_cl.trigger_try_jobs() try_results = self.git_cl.wait_for_try_jobs( poll_delay_seconds=POLL_DELAY_SECONDS, timeout_seconds=TIMEOUT_SECONDS) if not try_results: self.git_cl.run(['set-close']) return False if try_results and self.git_cl.has_failing_try_results(try_results): self.fetch_new_expectations_and_baselines() message = 'Update test expectations and baselines.' self.check_run(['git', 'commit', '-a', '-m', message]) self._upload_patchset(message) # Trigger CQ and wait for CQ try jobs to finish. self.git_cl.run(['set-commit', '--gerrit']) try_results = self.git_cl.wait_for_try_jobs( poll_delay_seconds=POLL_DELAY_SECONDS, timeout_seconds=TIMEOUT_SECONDS) _log.info('Try results: %s', try_results) # If the CQ passed, then the issue will be closed already. status = self.git_cl.run(['status', '--field', 'status']).strip() _log.info('CL status: "%s"', status) if status not in ('lgtm', 'closed'): _log.error('CQ appears to have failed; aborting.') self.git_cl.run(['set-close']) return False _log.info('Update completed.') return True def _upload_cl(self): _log.info('Uploading change list.') directory_owners = self.get_directory_owners() description = self._cl_description(directory_owners) self.git_cl.run([ 'upload', '-f', '--gerrit', '-m', description, ] + self._cc_part(directory_owners)) def _upload_patchset(self, message): self.git_cl.run(['upload', '-f', '-t', message, '--gerrit']) @staticmethod def _cc_part(directory_owners): cc_part = [] for owner_tuple in sorted(directory_owners): cc_part.extend('--cc=' + owner for owner in owner_tuple) return cc_part def get_directory_owners(self): """Returns a mapping of email addresses to owners of changed tests.""" _log.info('Gathering directory owners emails to CC.') changed_files = self.host.git().changed_files() extractor = DirectoryOwnersExtractor(self.fs) extractor.read_owner_map() return extractor.list_owners(changed_files) def _cl_description(self, directory_owners): """Returns a CL description string. Args: directory_owners: A dict of tuples of owner names to lists of directories. """ description = self.check_run(['git', 'log', '-1', '--format=%B']) build_link = current_build_link(self.host) if build_link: description += 'Build: %s\n\n' % build_link description += ( 'Background: https://chromium.googlesource.com' '/chromium/src/+/master/docs/testing/web_platform_tests.md\n\n' 'Note to sheriffs: If this CL causes a small number of new layout\n' 'test failures, it may be easier to add lines to TestExpectations\n' 'rather than reverting.\n') if directory_owners: description += self._format_directory_owners(directory_owners) + '\n\n' description += '[email protected]\n' # Move any NOEXPORT tag to the end of the description. description = description.replace('NOEXPORT=true', '') description = description.replace('\n\n\n\n', '\n\n') description += 'NOEXPORT=true' return description @staticmethod def _format_directory_owners(directory_owners): message_lines = ['Directory owners for changes in this CL:'] for owner_tuple, directories in sorted(directory_owners.items()): message_lines.append(', '.join(owner_tuple) + ':') message_lines.extend(' ' + d for d in directories) return '\n'.join(message_lines) def fetch_new_expectations_and_baselines(self): """Adds new expectations and downloads baselines based on try job results, then commits and uploads the change.""" _log.info('Adding test expectations lines to LayoutTests/TestExpectations.') expectation_updater = WPTExpectationsUpdater(self.host) expectation_updater.run(args=[]) def update_all_test_expectations_files(self, deleted_tests, renamed_tests): """Updates all test expectations files for tests that have been deleted or renamed.""" port = self.host.port_factory.get() for path, file_contents in port.all_expectations_dict().iteritems(): parser = TestExpectationParser(port, all_tests=None, is_lint_mode=False) expectation_lines = parser.parse(path, file_contents) self._update_single_test_expectations_file(path, expectation_lines, deleted_tests, renamed_tests) def _update_single_test_expectations_file(self, path, expectation_lines, deleted_tests, renamed_tests): """Updates single test expectations file.""" # FIXME: This won't work for removed or renamed directories with test expectations # that are directories rather than individual tests. new_lines = [] changed_lines = [] for expectation_line in expectation_lines: if expectation_line.name in deleted_tests: continue if expectation_line.name in renamed_tests: expectation_line.name = renamed_tests[expectation_line.name] # Upon parsing the file, a "path does not exist" warning is expected # to be there for tests that have been renamed, and if there are warnings, # then the original string is used. If the warnings are reset, then the # expectation line is re-serialized when output. expectation_line.warnings = [] changed_lines.append(expectation_line) new_lines.append(expectation_line) new_file_contents = TestExpectations.list_to_string(new_lines, reconstitute_only_these=changed_lines) self.host.filesystem.write_text_file(path, new_file_contents) def _list_deleted_tests(self): """Returns a list of layout tests that have been deleted.""" out = self.check_run(['git', 'diff', 'origin/master', '-M100%', '--diff-filter=D', '--name-only']) deleted_tests = [] for line in out.splitlines(): test = self.finder.layout_test_name(line) if test: deleted_tests.append(test) return deleted_tests def _list_renamed_tests(self): """Returns a dict mapping source to dest name for layout tests that have been renamed.""" out = self.check_run(['git', 'diff', 'origin/master', '-M100%', '--diff-filter=R', '--name-status']) renamed_tests = {} for line in out.splitlines(): _, source_path, dest_path = line.split() source_test = self.finder.layout_test_name(source_path) dest_test = self.finder.layout_test_name(dest_path) if source_test and dest_test: renamed_tests[source_test] = dest_test return renamed_tests
class TestImporter(object): def __init__(self, host, wpt_github=None): self.host = host self.wpt_github = wpt_github self.executive = host.executive self.fs = host.filesystem self.finder = PathFinder(self.fs) self.chromium_git = self.host.git(self.finder.chromium_base()) self.dest_path = self.finder.path_from_layout_tests('external', 'wpt') # A common.net.git_cl.GitCL instance. self.git_cl = None # Another Git instance with local WPT as CWD, which can only be # instantiated after the working directory is created. self.wpt_git = None # The WPT revision we are importing and the one imported last time. self.wpt_revision = None self.last_wpt_revision = None # A set of rebaselined tests and a dictionary of new test expectations # mapping failing tests to platforms to # wpt_expectations_updater.SimpleTestResult. self.rebaselined_tests = set() self.new_test_expectations = {} self.verbose = False def main(self, argv=None): # TODO(robertma): Test this method! Split it to make it easier to test # if necessary. options = self.parse_args(argv) self.verbose = options.verbose log_level = logging.DEBUG if self.verbose else logging.INFO configure_logging(logging_level=log_level, include_time=True) if options.verbose: # Print out the full output when executive.run_command fails. self.host.executive.error_output_limit = None if not self.checkout_is_okay(): return 1 credentials = read_credentials(self.host, options.credentials_json) gh_user = credentials.get('GH_USER') gh_token = credentials.get('GH_TOKEN') if not gh_user or not gh_token: _log.warning('You have not set your GitHub credentials. This ' 'script may fail with a network error when making ' 'an API request to GitHub.') _log.warning('See https://chromium.googlesource.com/chromium/src' '/+/master/docs/testing/web_platform_tests.md' '#GitHub-credentials for instructions on how to set ' 'your credentials up.') self.wpt_github = self.wpt_github or WPTGitHub(self.host, gh_user, gh_token) self.git_cl = GitCL( self.host, auth_refresh_token_json=options.auth_refresh_token_json) _log.debug('Noting the current Chromium revision.') chromium_revision = self.chromium_git.latest_git_commit() # Instantiate Git after local_wpt.fetch() to make sure the path exists. local_wpt = LocalWPT(self.host, gh_token=gh_token) local_wpt.fetch() self.wpt_git = self.host.git(local_wpt.path) if options.revision is not None: _log.info('Checking out %s', options.revision) self.wpt_git.run(['checkout', options.revision]) _log.debug('Noting the revision we are importing.') self.wpt_revision = self.wpt_git.latest_git_commit() self.last_wpt_revision = self._get_last_imported_wpt_revision() import_commit = 'wpt@%s' % self.wpt_revision _log.info('Importing %s to Chromium %s', import_commit, chromium_revision) if options.ignore_exportable_commits: commit_message = self._commit_message(chromium_revision, import_commit) else: commits = self.apply_exportable_commits_locally(local_wpt) if commits is None: _log.error('Could not apply some exportable commits cleanly.') _log.error('Aborting import to prevent clobbering commits.') return 1 commit_message = self._commit_message( chromium_revision, import_commit, locally_applied_commits=commits) self._clear_out_dest_path() _log.info('Copying the tests from the temp repo to the destination.') test_copier = TestCopier(self.host, local_wpt.path) test_copier.do_import() # TODO(robertma): Implement `add --all` in Git (it is different from `commit --all`). self.chromium_git.run(['add', '--all', self.dest_path]) self._generate_manifest() # TODO(crbug.com/800570 robertma): Re-enable it once we fix the bug. # self._delete_orphaned_baselines() # TODO(qyearsley): Consider running the imported tests with # `run-webkit-tests --reset-results external/wpt` to get some baselines # before the try jobs are started. _log.info( 'Updating TestExpectations for any removed or renamed tests.') self.update_all_test_expectations_files(self._list_deleted_tests(), self._list_renamed_tests()) if not self.chromium_git.has_working_directory_changes(): _log.info('Done: no changes to import.') return 0 if self._only_wpt_manifest_changed(): _log.info( 'Only WPT_BASE_MANIFEST.json was updated; skipping the import.' ) return 0 self._commit_changes(commit_message) _log.info('Changes imported and committed.') if not options.auto_update: return 0 self._upload_cl() _log.info('Issue: %s', self.git_cl.run(['issue']).strip()) if not self.update_expectations_for_cl(): return 1 if not self.run_commit_queue_for_cl(): return 1 if not self.send_notifications(local_wpt, options.auto_file_bugs, options.monorail_auth_json): return 1 return 0 def update_expectations_for_cl(self): """Performs the expectation-updating part of an auto-import job. This includes triggering try jobs and waiting; then, if applicable, writing new baselines and TestExpectation lines, committing, and uploading a new patchset. This assumes that there is CL associated with the current branch. Returns True if everything is OK to continue, or False on failure. """ _log.info('Triggering try jobs for updating expectations.') self.git_cl.trigger_try_jobs(self.blink_try_bots()) cl_status = self.git_cl.wait_for_try_jobs( poll_delay_seconds=POLL_DELAY_SECONDS, timeout_seconds=TIMEOUT_SECONDS) if not cl_status: _log.error('No initial try job results, aborting.') self.git_cl.run(['set-close']) return False if cl_status.status == 'closed': _log.error('The CL was closed, aborting.') return False _log.info('All jobs finished.') try_results = cl_status.try_job_results if try_results and self.git_cl.some_failed(try_results): self.fetch_new_expectations_and_baselines() if self.chromium_git.has_working_directory_changes(): self._generate_manifest() message = 'Update test expectations and baselines.' self._commit_changes(message) self._upload_patchset(message) return True def run_commit_queue_for_cl(self): """Triggers CQ and either commits or aborts; returns True on success.""" _log.info('Triggering CQ try jobs.') self.git_cl.run(['try']) cl_status = self.git_cl.wait_for_try_jobs( poll_delay_seconds=POLL_DELAY_SECONDS, timeout_seconds=TIMEOUT_SECONDS, cq_only=True) if not cl_status: self.git_cl.run(['set-close']) _log.error('Timed out waiting for CQ; aborting.') return False if cl_status.status == 'closed': _log.error('The CL was closed; aborting.') return False _log.info('All jobs finished.') cq_try_results = cl_status.try_job_results if not cq_try_results: _log.error('No CQ try results found in try results') self.git_cl.run(['set-close']) return False if self.git_cl.all_success(cq_try_results): _log.info('CQ appears to have passed; trying to commit.') self.git_cl.run(['upload', '-f', '--send-mail']) # Turn off WIP mode. self.git_cl.run(['set-commit']) self.git_cl.wait_for_closed_status() _log.info('Update completed.') return True self.git_cl.run(['set-close']) _log.error('CQ appears to have failed; aborting.') return False def blink_try_bots(self): """Returns the collection of builders used for updating expectations.""" return self.host.builders.all_try_builder_names() def parse_args(self, argv): parser = argparse.ArgumentParser() parser.description = __doc__ parser.add_argument( '-v', '--verbose', action='store_true', help='log extra details that may be helpful when debugging') parser.add_argument( '--ignore-exportable-commits', action='store_true', help='do not check for exportable commits that would be clobbered') parser.add_argument('-r', '--revision', help='target wpt revision') parser.add_argument( '--auto-update', action='store_true', help='upload a CL, update expectations, and trigger CQ') parser.add_argument( '--auto-file-bugs', action='store_true', help='file new failures automatically to crbug.com') parser.add_argument( '--auth-refresh-token-json', help='authentication refresh token JSON file used for try jobs, ' 'generally not necessary on developer machines') parser.add_argument('--credentials-json', help='A JSON file with GitHub credentials, ' 'generally not necessary on developer machines') parser.add_argument( '--monorail-auth-json', help='A JSON file containing the private key of a service account ' 'to access Monorail (crbug.com), only needed when ' '--auto-file-bugs is used') return parser.parse_args(argv) def checkout_is_okay(self): if self.chromium_git.has_working_directory_changes(): _log.warning('Checkout is dirty; aborting.') return False # TODO(robertma): Add a method in Git to query a range of commits. local_commits = self.chromium_git.run( ['log', '--oneline', 'origin/master..HEAD']) if local_commits: _log.warning('Checkout has local commits before import.') return True def apply_exportable_commits_locally(self, local_wpt): """Applies exportable Chromium changes to the local WPT repo. The purpose of this is to avoid clobbering changes that were made in Chromium but not yet merged upstream. By applying these changes to the local copy of web-platform-tests before copying files over, we make it so that the resulting change in Chromium doesn't undo the previous Chromium change. Args: A LocalWPT instance for our local copy of WPT. Returns: A list of commits applied (could be empty), or None if any of the patches could not be applied cleanly. """ commits = self.exportable_but_not_exported_commits(local_wpt) for commit in commits: _log.info('Applying exportable commit locally:') _log.info(commit.url()) _log.info('Subject: %s', commit.subject().strip()) # TODO(qyearsley): We probably don't need to know about # corresponding PRs at all anymore, although this information # could still be useful for reference. pull_request = self.wpt_github.pr_for_chromium_commit(commit) if pull_request: _log.info( 'PR: https://github.com/w3c/web-platform-tests/pull/%d', pull_request.number) else: _log.warning('No pull request found.') error = local_wpt.apply_patch(commit.format_patch()) if error: _log.error('Commit cannot be applied cleanly:') _log.error(error) return None self.wpt_git.commit_locally_with_message('Applying patch %s' % commit.sha) return commits def exportable_but_not_exported_commits(self, local_wpt): """Returns a list of commits that would be clobbered by importer. The list contains all exportable but not exported commits, not filtered by whether they can apply cleanly. """ # The errors returned by exportable_commits_over_last_n_commits are # irrelevant and ignored here, because it tests patches *individually* # while the importer tries to reapply these patches *cumulatively*. commits, _ = exportable_commits_over_last_n_commits( self.host, local_wpt, self.wpt_github, require_clean=False, verify_merged_pr=True) return commits def _generate_manifest(self): """Generates MANIFEST.json for imported tests. Runs the (newly-updated) manifest command if it's found, and then stages the generated MANIFEST.json in the git index, ready to commit. """ _log.info('Generating MANIFEST.json') WPTManifest.generate_manifest(self.host, self.dest_path) manifest_path = self.fs.join(self.dest_path, 'MANIFEST.json') assert self.fs.exists(manifest_path) manifest_base_path = self.fs.normpath( self.fs.join(self.dest_path, '..', 'WPT_BASE_MANIFEST.json')) self.copyfile(manifest_path, manifest_base_path) self.chromium_git.add_list([manifest_base_path]) def _clear_out_dest_path(self): """Removes all files that are synced with upstream from Chromium WPT. Instead of relying on TestCopier to overwrite these files, cleaning up first ensures if upstream deletes some files, we also delete them. """ _log.info('Cleaning out tests from %s.', self.dest_path) should_remove = lambda fs, dirname, basename: (is_file_exportable( fs.relpath(fs.join(dirname, basename), self.finder.chromium_base()) )) files_to_delete = self.fs.files_under(self.dest_path, file_filter=should_remove) for subpath in files_to_delete: self.remove(self.finder.path_from_layout_tests( 'external', subpath)) def _commit_changes(self, commit_message): _log.info('Committing changes.') self.chromium_git.commit_locally_with_message(commit_message) def _only_wpt_manifest_changed(self): changed_files = self.chromium_git.changed_files() wpt_base_manifest = self.fs.relpath( self.fs.join(self.dest_path, '..', 'WPT_BASE_MANIFEST.json'), self.finder.chromium_base()) return changed_files == [wpt_base_manifest] def _commit_message(self, chromium_commit_sha, import_commit_sha, locally_applied_commits=None): message = 'Import {}\n\nUsing wpt-import in Chromium {}.\n'.format( import_commit_sha, chromium_commit_sha) if locally_applied_commits: message += 'With Chromium commits locally applied on WPT:\n' message += '\n'.join( str(commit) for commit in locally_applied_commits) message += '\nNo-Export: true' return message def _delete_orphaned_baselines(self): _log.info('Deleting any orphaned baselines.') is_baseline_filter = lambda fs, dirname, basename: is_testharness_baseline( basename) baselines = self.fs.files_under(self.dest_path, file_filter=is_baseline_filter) # TODO(qyearsley): Factor out the manifest path to a common location. # TODO(qyearsley): Factor out the manifest reading from here and Port # to WPTManifest. manifest_path = self.finder.path_from_layout_tests( 'external', 'wpt', 'MANIFEST.json') manifest = WPTManifest(self.fs.read_text_file(manifest_path)) wpt_urls = manifest.all_urls() # Currently baselines for tests with query strings are merged, # so that the tests foo.html?r=1 and foo.html?r=2 both have the same # baseline, foo-expected.txt. # TODO(qyearsley): Remove this when this behavior is fixed. wpt_urls = [url.split('?')[0] for url in wpt_urls] wpt_dir = self.finder.path_from_layout_tests('external', 'wpt') for full_path in baselines: rel_path = self.fs.relpath(full_path, wpt_dir) if not self._has_corresponding_test(rel_path, wpt_urls): self.fs.remove(full_path) def _has_corresponding_test(self, rel_path, wpt_urls): # TODO(qyearsley): Ensure that this works with platform baselines and # virtual baselines, and add unit tests. base = '/' + rel_path.replace('-expected.txt', '') return any( (base + ext) in wpt_urls for ext in Port.supported_file_extensions) def copyfile(self, source, destination): _log.debug('cp %s %s', source, destination) self.fs.copyfile(source, destination) def remove(self, dest): _log.debug('rm %s', dest) self.fs.remove(dest) def _upload_patchset(self, message): self.git_cl.run(['upload', '-f', '-t', message, '--gerrit']) def _upload_cl(self): _log.info('Uploading change list.') directory_owners = self.get_directory_owners() description = self._cl_description(directory_owners) sheriff_email = self.tbr_reviewer() temp_file, temp_path = self.fs.open_text_tempfile() temp_file.write(description) temp_file.close() self.git_cl.run([ 'upload', '-f', '--gerrit', '--message-file', temp_path, '--tbrs', sheriff_email, # Note: we used to CC all the directory owners, but have stopped # in search of a better notification mechanism. (crbug.com/765334) '--cc', '*****@*****.**', ]) self.fs.remove(temp_path) def get_directory_owners(self): """Returns a mapping of email addresses to owners of changed tests.""" _log.info('Gathering directory owners emails to CC.') changed_files = self.chromium_git.changed_files() extractor = DirectoryOwnersExtractor(self.fs) return extractor.list_owners(changed_files) def _cl_description(self, directory_owners): """Returns a CL description string. Args: directory_owners: A dict of tuples of owner names to lists of directories. """ # TODO(robertma): Add a method in Git for getting the commit body. description = self.chromium_git.run(['log', '-1', '--format=%B']) build_link = current_build_link(self.host) if build_link: description += 'Build: %s\n\n' % build_link description += ( 'Note to sheriffs: This CL imports external tests and adds\n' 'expectations for those tests; if this CL is large and causes\n' 'a few new failures, please fix the failures by adding new\n' 'lines to TestExpectations rather than reverting. See:\n' 'https://chromium.googlesource.com' '/chromium/src/+/master/docs/testing/web_platform_tests.md\n\n') if directory_owners: description += self._format_directory_owners( directory_owners) + '\n\n' # Move any No-Export tag to the end of the description. description = description.replace('No-Export: true', '') description = description.replace('\n\n\n\n', '\n\n') description += 'No-Export: true' return description @staticmethod def _format_directory_owners(directory_owners): message_lines = ['Directory owners for changes in this CL:'] for owner_tuple, directories in sorted(directory_owners.items()): message_lines.append(', '.join(owner_tuple) + ':') message_lines.extend(' ' + d for d in directories) return '\n'.join(message_lines) def tbr_reviewer(self): """Returns the user name or email address to use as the reviewer. This tries to fetch the current ecosystem infra sheriff, but falls back in case of error. Either a user name (which is assumed to have a chromium.org email address) or a full email address (for other cases) is returned. """ username = '' try: username = self._fetch_ecosystem_infra_sheriff_username() except (IOError, KeyError, ValueError) as error: _log.error('Exception while fetching current sheriff: %s', error) return username or TBR_FALLBACK def _fetch_ecosystem_infra_sheriff_username(self): content = self.host.web.get_binary(ROTATIONS_URL) data = json.loads(content) today = datetime.date.fromtimestamp(self.host.time()).isoformat() index = data['rotations'].index('ecosystem_infra') calendar = data['calendar'] for entry in calendar: if entry['date'] == today: if not entry['participants'][index]: _log.info('No sheriff today.') return '' return entry['participants'][index][0] _log.error('No entry found for date %s in rotations table.', today) return '' def fetch_new_expectations_and_baselines(self): """Modifies expectation lines and baselines based on try job results. Assuming that there are some try job results available, this adds new expectation lines to TestExpectations and downloads new baselines based on the try job results. This is the same as invoking the `wpt-update-expectations` script. """ _log.info( 'Adding test expectations lines to LayoutTests/TestExpectations.') expectation_updater = WPTExpectationsUpdater(self.host) self.rebaselined_tests, self.new_test_expectations = expectation_updater.update_expectations( ) def update_all_test_expectations_files(self, deleted_tests, renamed_tests): """Updates all test expectations files for tests that have been deleted or renamed. This is only for deleted or renamed tests in the initial import, not for tests that have failures in try jobs. """ port = self.host.port_factory.get() for path, file_contents in port.all_expectations_dict().iteritems(): parser = TestExpectationParser(port, all_tests=None, is_lint_mode=False) expectation_lines = parser.parse(path, file_contents) self._update_single_test_expectations_file(path, expectation_lines, deleted_tests, renamed_tests) def _update_single_test_expectations_file(self, path, expectation_lines, deleted_tests, renamed_tests): """Updates a single test expectations file.""" # FIXME: This won't work for removed or renamed directories with test # expectations that are directories rather than individual tests. new_lines = [] changed_lines = [] for expectation_line in expectation_lines: if expectation_line.name in deleted_tests: continue if expectation_line.name in renamed_tests: expectation_line.name = renamed_tests[expectation_line.name] # Upon parsing the file, a "path does not exist" warning is expected # to be there for tests that have been renamed, and if there are warnings, # then the original string is used. If the warnings are reset, then the # expectation line is re-serialized when output. expectation_line.warnings = [] changed_lines.append(expectation_line) new_lines.append(expectation_line) new_file_contents = TestExpectations.list_to_string( new_lines, reconstitute_only_these=changed_lines) self.host.filesystem.write_text_file(path, new_file_contents) def _list_deleted_tests(self): """List of layout tests that have been deleted.""" # TODO(robertma): Improve Git.changed_files so that we can use it here. out = self.chromium_git.run([ 'diff', 'origin/master', '-M100%', '--diff-filter=D', '--name-only' ]) deleted_tests = [] for path in out.splitlines(): test = self._relative_to_layout_test_dir(path) if test: deleted_tests.append(test) return deleted_tests def _list_renamed_tests(self): """Lists tests that have been renamed. Returns a dict mapping source name to destination name. """ out = self.chromium_git.run([ 'diff', 'origin/master', '-M100%', '--diff-filter=R', '--name-status' ]) renamed_tests = {} for line in out.splitlines(): _, source_path, dest_path = line.split() source_test = self._relative_to_layout_test_dir(source_path) dest_test = self._relative_to_layout_test_dir(dest_path) if source_test and dest_test: renamed_tests[source_test] = dest_test return renamed_tests def _relative_to_layout_test_dir(self, path_relative_to_repo_root): """Returns a path that's relative to the layout tests directory.""" abs_path = self.finder.path_from_chromium_base( path_relative_to_repo_root) if not abs_path.startswith(self.finder.layout_tests_dir()): return None return self.fs.relpath(abs_path, self.finder.layout_tests_dir()) def _get_last_imported_wpt_revision(self): """Finds the last imported WPT revision.""" # TODO(robertma): Only match commit subjects. output = self.chromium_git.most_recent_log_matching( '^Import wpt@', self.finder.chromium_base()) # No line-start anchor (^) below because of the formatting of output. result = re.search(r'Import wpt@(\w+)', output) if result: return result.group(1) else: _log.error('Cannot find last WPT import.') return None def send_notifications(self, local_wpt, auto_file_bugs, monorail_auth_json): issue = self.git_cl.run(['status', '--field=id']).strip() patchset = self.git_cl.run(['status', '--field=patch']).strip() # Construct the notifier here so that any errors won't affect the import. notifier = ImportNotifier(self.host, self.chromium_git, local_wpt) notifier.main(self.last_wpt_revision, self.wpt_revision, self.rebaselined_tests, self.new_test_expectations, issue, patchset, dry_run=not auto_file_bugs, service_account_key_json=monorail_auth_json) return True
def absolute_chromium_wpt_dir(host): finder = PathFinder(host.filesystem) return finder.path_from_layout_tests('external', 'wpt')
def test_path_from_layout_tests(self): finder = PathFinder(MockFileSystem()) self.assertEqual( finder.path_from_layout_tests('external', 'wpt'), '/mock-checkout/third_party/WebKit/LayoutTests/external/wpt')
class DirectoryOwnersExtractor(object): def __init__(self, filesystem=None): self.filesystem = filesystem or FileSystem self.finder = PathFinder(filesystem) self.owner_map = None def read_owner_map(self): """Reads the W3CImportExpectations file and returns a map of directories to owners.""" input_path = self.finder.path_from_layout_tests( 'W3CImportExpectations') input_contents = self.filesystem.read_text_file(input_path) self.owner_map = self.lines_to_owner_map(input_contents.splitlines()) def lines_to_owner_map(self, lines): current_owners = [] owner_map = {} for line in lines: owners = self.extract_owners(line) if owners: current_owners = owners directory = self.extract_directory(line) if not owners and not directory: current_owners = [] if current_owners and directory: owner_map[directory] = current_owners return owner_map @staticmethod def extract_owners(line): """Extracts owner email addresses listed on a line.""" match = re.match(r'##? Owners?: (?P<addresses>.*)', line) if not match or not match.group('addresses'): return None email_part = match.group('addresses') addresses = [email_part] if ',' not in email_part else re.split( r',\s*', email_part) addresses = [s for s in addresses if re.match(r'\S+@\S+', s)] return addresses or None @staticmethod def extract_directory(line): match = re.match(r'# ?(?P<directory>\S+) \[ (Pass|Skip) \]', line) if match and match.group('directory'): return match.group('directory') match = re.match(r'(?P<directory>\S+) \[ Pass \]', line) if match and match.group('directory'): return match.group('directory') return None def list_owners(self, changed_files): """Looks up the owners for the given set of changed files. Args: changed_files: A list of file paths relative to the repository root. Returns: A dict mapping tuples of owner email addresses to lists of owned directories. """ tests = [self.finder.layout_test_name(path) for path in changed_files] tests = [t for t in tests if t is not None] email_map = collections.defaultdict(list) for directory, owners in self.owner_map.iteritems(): owned_tests = [t for t in tests if t.startswith(directory)] if not owned_tests: continue email_map[tuple(owners)].append(directory) return email_map