def test_web_tests_dir_with_backslash_sep(self):
filesystem = MockFileSystem()
filesystem.sep = '\\'
filesystem.path_to_module = lambda _: (
'C:\\mock-checkout\\third_party\\blink\\tools\\blinkpy\\foo.py')
finder = PathFinder(filesystem)
self.assertEqual(finder.web_tests_dir(),
'C:\\mock-checkout\\third_party\\blink\\web_tests')
class DirectoryOwnersExtractor(object):
def __init__(self, host):
self.filesystem = host.filesystem
self.finder = PathFinder(self.filesystem)
self.executive = host.executive
self.owner_map = None
def list_owners(self, changed_files):
"""Looks up the owners for the given set of changed files.
Args:
changed_files: A list of file paths relative to the repository root.
Returns:
A dict mapping tuples of owner email addresses to lists of
owned directories (paths relative to the root of web tests).
"""
email_map = collections.defaultdict(set)
external_root_owners = self.finder.path_from_web_tests(
'external', 'OWNERS')
for relpath in changed_files:
# Try to find the first *non-empty* OWNERS file.
absolute_path = self.finder.path_from_chromium_base(relpath)
owners = None
owners_file = self.find_owners_file(absolute_path)
while owners_file:
owners = self.extract_owners(owners_file)
if owners:
break
# Found an empty OWNERS file. Try again from the parent directory.
absolute_path = self.filesystem.dirname(
self.filesystem.dirname(owners_file))
owners_file = self.find_owners_file(absolute_path)
# Skip web_tests/external/OWNERS.
if not owners or owners_file == external_root_owners:
continue
owned_directory = self.filesystem.dirname(owners_file)
owned_directory_relpath = self.filesystem.relpath(
owned_directory, self.finder.web_tests_dir())
email_map[tuple(owners)].add(owned_directory_relpath)
return {
owners: sorted(owned_directories)
for owners, owned_directories in email_map.iteritems()
}
def find_owners_file(self, start_path):
"""Finds the first enclosing OWNERS file for a given path.
Starting from the given path, walks up the directory tree until the
first OWNERS file is found or web_tests/external is reached.
Args:
start_path: A relative path from the root of the repository, or an
absolute path. The path can be a file or a directory.
Returns:
The absolute path to the first OWNERS file found; None if not found
or if start_path is outside of web_tests/external.
"""
abs_start_path = (start_path if self.filesystem.isabs(start_path) else
self.finder.path_from_chromium_base(start_path))
directory = (abs_start_path if self.filesystem.isdir(abs_start_path)
else self.filesystem.dirname(abs_start_path))
external_root = self.finder.path_from_web_tests('external')
if not directory.startswith(external_root):
return None
# Stop at web_tests, which is the parent of external_root.
while directory != self.finder.web_tests_dir():
owners_file = self.filesystem.join(directory, 'OWNERS')
if self.filesystem.isfile(
self.finder.path_from_chromium_base(owners_file)):
return owners_file
directory = self.filesystem.dirname(directory)
return None
def extract_owners(self, owners_file):
"""Extracts owners from an OWNERS file.
Args:
owners_file: An absolute path to an OWNERS file.
Returns:
A list of valid owners (email addresses).
"""
contents = self._read_text_file(owners_file)
email_regexp = re.compile(BASIC_EMAIL_REGEXP)
addresses = []
for line in contents.splitlines():
line = line.strip()
if email_regexp.match(line):
addresses.append(line)
return addresses
def extract_component(self, owners_file):
"""Extracts the component from an OWNERS file.
Args:
owners_file: An absolute path to an OWNERS file.
Returns:
A string, or None if not found.
"""
dir_metadata = self._read_dir_metadata(owners_file)
if dir_metadata and dir_metadata.component:
return dir_metadata.component
contents = self._read_text_file(owners_file)
search = re.search(COMPONENT_REGEXP, contents, re.MULTILINE)
if search:
return search.group(1)
return None
def is_wpt_notify_enabled(self, owners_file):
"""Checks if the OWNERS file enables WPT-NOTIFY.
Args:
owners_file: An absolute path to an OWNERS file.
Returns:
A boolean.
"""
dir_metadata = self._read_dir_metadata(owners_file)
if dir_metadata and dir_metadata.should_notify is not None:
return dir_metadata.should_notify
contents = self._read_text_file(owners_file)
return bool(re.search(WPT_NOTIFY_REGEXP, contents, re.MULTILINE))
@memoized
def _read_text_file(self, path):
return self.filesystem.read_text_file(path)
@memoized
def _read_dir_metadata(self, path):
"""Read the content from a path.
Args:
path: An absolute path.
Returns:
A WPTDirMetadata object, or None if not found.
"""
root_path = self.finder.web_tests_dir()
dir_path = self.filesystem.dirname(path)
# dirmd starts with an absolute directory path, `dir_path`, traverses all
# parent directories and stops at `root_path` to find the first available DIR_METADATA
# file. `root_path` is the web_tests directory.
json_data = self.executive.run_command([
self.finder.path_from_depot_tools_base('dirmd'), 'compute',
'-root', root_path, dir_path
])
try:
data = json.loads(json_data)
except ValueError:
return None
relative_path = self.filesystem.relpath(dir_path, root_path)
return WPTDirMetadata(data, relative_path)
class ImportNotifier(object):
def __init__(self, host, chromium_git, local_wpt):
self.host = host
self.git = chromium_git
self.local_wpt = local_wpt
self._monorail_api = MonorailAPI
self.default_port = host.port_factory.get()
self.finder = PathFinder(host.filesystem)
self.owners_extractor = DirectoryOwnersExtractor(host)
self.new_failures_by_directory = defaultdict(list)
self.components_for_product = {ANDROID_WEBLAYER: ["Internals>WebLayer"]}
self.labels_for_product = {
ANDROID_WEBLAYER: ["Project-WebLayer-WebPlatformSupport", "WL-WPT-Compat"]
}
def main(self,
wpt_revision_start,
wpt_revision_end,
rebaselined_tests,
test_expectations,
new_override_expectations,
issue,
patchset,
dry_run=True,
service_account_key_json=None):
"""Files bug reports for new failures.
Args:
wpt_revision_start: The start of the imported WPT revision range
(exclusive), i.e. the last imported revision.
wpt_revision_end: The end of the imported WPT revision range
(inclusive), i.e. the current imported revision.
rebaselined_tests: A list of test names that have been rebaselined.
test_expectations: A dictionary mapping names of tests that cannot
be rebaselined to a list of new test expectation lines.
issue: The issue number of the import CL (a string).
patchset: The patchset number of the import CL (a string).
dry_run: If True, no bugs will be actually filed to crbug.com.
service_account_key_json: The path to a JSON private key of a
service account for accessing Monorail. If None, try to get an
access token from luci-auth.
Note: "test names" are paths of the tests relative to web_tests.
"""
gerrit_url = SHORT_GERRIT_PREFIX + issue
gerrit_url_with_ps = gerrit_url + '/' + patchset + '/'
changed_test_baselines = self.find_changed_baselines_of_tests(
rebaselined_tests)
self.examine_baseline_changes(changed_test_baselines,
gerrit_url_with_ps)
self.examine_new_test_expectations(test_expectations)
bugs = self.create_bugs_from_new_failures(wpt_revision_start,
wpt_revision_end, gerrit_url)
self.file_bugs(bugs, dry_run, service_account_key_json)
for product, expectation_lines in new_override_expectations.items():
bugs = self.create_bugs_for_product(wpt_revision_start,
wpt_revision_end,
gerrit_url,
product,
expectation_lines)
self.file_bugs(bugs, dry_run, service_account_key_json)
def find_changed_baselines_of_tests(self, rebaselined_tests):
"""Finds the corresponding changed baselines of each test.
Args:
rebaselined_tests: A list of test names that have been rebaselined.
Returns:
A dictionary mapping test names to paths of their baselines changed
in this import CL (paths relative to the root of Chromium repo).
"""
test_baselines = {}
changed_files = self.git.changed_files()
for test_name in rebaselined_tests:
test_without_ext, _ = self.host.filesystem.splitext(test_name)
changed_baselines = []
# TODO(robertma): Refactor this into web_tests.port.base.
baseline_name = test_without_ext + '-expected.txt'
for changed_file in changed_files:
if changed_file.endswith(baseline_name):
changed_baselines.append(changed_file)
if changed_baselines:
test_baselines[test_name] = changed_baselines
return test_baselines
def examine_baseline_changes(self, changed_test_baselines,
gerrit_url_with_ps):
"""Examines all changed baselines to find new failures.
Args:
changed_test_baselines: A dictionary mapping test names to paths of
changed baselines.
gerrit_url_with_ps: Gerrit URL of this CL with the patchset number.
"""
for test_name, changed_baselines in changed_test_baselines.items():
directory = self.find_owned_directory(test_name)
if not directory:
_log.warning('Cannot find OWNERS of %s', test_name)
continue
for baseline in changed_baselines:
if self.more_failures_in_baseline(baseline):
self.new_failures_by_directory[directory].append(
TestFailure(
TestFailure.BASELINE_CHANGE,
test_name,
baseline_path=baseline,
gerrit_url_with_ps=gerrit_url_with_ps))
def more_failures_in_baseline(self, baseline):
"""Determines if a testharness.js baseline file has new failures.
The file is assumed to have been modified in the current git checkout,
and so has a diff we can parse.
We recognize two types of failures: FAIL lines, which are output for a
specific subtest failing, and harness errors, which indicate an uncaught
error in the test. Increasing numbers of either are considered new
failures - this includes going from FAIL to error or vice-versa.
"""
diff = self.git.run(['diff', '-U0', 'origin/main', '--', baseline])
delta_failures = 0
delta_harness_errors = 0
for line in diff.splitlines():
if line.startswith('+FAIL'):
delta_failures += 1
if line.startswith('-FAIL'):
delta_failures -= 1
if line.startswith('+Harness Error.'):
delta_harness_errors += 1
if line.startswith('-Harness Error.'):
delta_harness_errors -= 1
return delta_failures > 0 or delta_harness_errors > 0
def examine_new_test_expectations(self, test_expectations):
"""Examines new test expectations to find new failures.
Args:
test_expectations: A dictionary mapping names of tests that cannot
be rebaselined to a list of new test expectation lines.
"""
for test_name, expectation_lines in test_expectations.items():
directory = self.find_owned_directory(test_name)
if not directory:
_log.warning('Cannot find OWNERS of %s', test_name)
continue
for expectation_line in expectation_lines:
self.new_failures_by_directory[directory].append(
TestFailure(
TestFailure.NEW_EXPECTATION,
test_name,
expectation_line=expectation_line))
def create_bugs_for_product(self, wpt_revision_start, wpt_revision_end,
gerrit_url, product, expectation_lines):
"""Files bug reports for new failures per product
Args:
wpt_revision_start: The start of the imported WPT revision range
(exclusive), i.e. the last imported revision.
wpt_revision_end: The end of the imported WPT revision range
(inclusive), i.e. the current imported revision.
gerrit_url: Gerrit URL of the CL.
product: the product for which to file bugs for.
expectation_lines: list of new expectations for this product
Return:
A MonorailIssue object that should be filed.
"""
bugs = []
summary = '[WPT] New failures introduced by import {}'.format(gerrit_url)
prologue = ('WPT import {} introduced new failures:\n\n'
'List of new failures:\n'.format(gerrit_url))
failure_list = ''
for _, failure in expectation_lines.items():
failure_list += str(failure) + '\n'
expectations_statement = (
'\nExpectations have been automatically added for '
'the failing results to keep the bots green. Please '
'investigate the new failures and triage as appropriate.\n')
range_statement = '\nThis import contains upstream changes from {} to {}:\n'.format(
wpt_revision_start, wpt_revision_end)
description = (prologue + failure_list + expectations_statement +
range_statement)
bug = MonorailIssue.new_chromium_issue(
summary,
description,
cc=[],
components=self.components_for_product[product],
labels=self.labels_for_product[product])
bugs.append(bug)
return bugs
def create_bugs_from_new_failures(self, wpt_revision_start,
wpt_revision_end, gerrit_url):
"""Files bug reports for new failures.
Args:
wpt_revision_start: The start of the imported WPT revision range
(exclusive), i.e. the last imported revision.
wpt_revision_end: The end of the imported WPT revision range
(inclusive), i.e. the current imported revision.
gerrit_url: Gerrit URL of the CL.
Return:
A list of MonorailIssue objects that should be filed.
"""
imported_commits = self.local_wpt.commits_in_range(
wpt_revision_start, wpt_revision_end)
bugs = []
for directory, failures in self.new_failures_by_directory.items():
summary = '[WPT] New failures introduced in {} by import {}'.format(
directory, gerrit_url)
full_directory = self.host.filesystem.join(
self.finder.web_tests_dir(), directory)
owners_file = self.host.filesystem.join(full_directory, 'OWNERS')
metadata_file = self.host.filesystem.join(full_directory,
'DIR_METADATA')
is_wpt_notify_enabled = False
try:
is_wpt_notify_enabled = self.owners_extractor.is_wpt_notify_enabled(
metadata_file)
except KeyError:
_log.info('KeyError when parsing %s' % metadata_file)
if not is_wpt_notify_enabled:
_log.info("WPT-NOTIFY disabled in %s." % full_directory)
continue
owners = self.owners_extractor.extract_owners(owners_file)
# owners may be empty but not None.
cc = owners
component = self.owners_extractor.extract_component(metadata_file)
# component could be None.
components = [component] if component else None
prologue = ('WPT import {} introduced new failures in {}:\n\n'
'List of new failures:\n'.format(
gerrit_url, directory))
failure_list = ''
for failure in failures:
failure_list += str(failure) + '\n'
expectations_statement = (
'\nExpectations or baseline files [0] have been automatically '
'added for the failing results to keep the bots green. Please '
'investigate the new failures and triage as appropriate.\n')
range_statement = '\nThis import contains upstream changes from {} to {}:\n'.format(
wpt_revision_start, wpt_revision_end)
commit_list = self.format_commit_list(imported_commits,
full_directory)
links_list = '\n[0]: https://chromium.googlesource.com/chromium/src/+/HEAD/docs/testing/web_test_expectations.md\n'
description = (prologue + failure_list + expectations_statement +
range_statement + commit_list + links_list)
bug = MonorailIssue.new_chromium_issue(summary,
description,
cc,
components,
labels=['Test-WebTest'])
_log.info(bug)
_log.info("WPT-NOTIFY enabled in %s; adding the bug to the pending list." % full_directory)
bugs.append(bug)
return bugs
def format_commit_list(self, imported_commits, directory):
"""Formats the list of imported WPT commits.
Imports affecting the given directory will be highlighted.
Args:
imported_commits: A list of (SHA, commit subject) pairs.
directory: An absolute path of a directory in the Chromium repo, for
which the list is formatted.
Returns:
A multi-line string.
"""
path_from_wpt = self.host.filesystem.relpath(
directory, self.finder.path_from_web_tests('external', 'wpt'))
commit_list = ''
for sha, subject in imported_commits:
# subject is a Unicode string and can contain non-ASCII characters.
line = u'{}: {}'.format(subject, GITHUB_COMMIT_PREFIX + sha)
if self.local_wpt.is_commit_affecting_directory(
sha, path_from_wpt):
line += ' [affecting this directory]'
commit_list += line + '\n'
return commit_list
def find_owned_directory(self, test_name):
"""Finds the lowest directory that contains the test and has OWNERS.
Args:
The name of the test (a path relative to web_tests).
Returns:
The path of the found directory relative to web_tests.
"""
# Always use non-virtual test names when looking up OWNERS.
if self.default_port.lookup_virtual_test_base(test_name):
test_name = self.default_port.lookup_virtual_test_base(test_name)
# find_owners_file takes either a relative path from the *root* of the
# repository, or an absolute path.
abs_test_path = self.finder.path_from_web_tests(test_name)
owners_file = self.owners_extractor.find_owners_file(
self.host.filesystem.dirname(abs_test_path))
if not owners_file:
return None
owned_directory = self.host.filesystem.dirname(owners_file)
short_directory = self.host.filesystem.relpath(
owned_directory, self.finder.web_tests_dir())
return short_directory
def file_bugs(self, bugs, dry_run, service_account_key_json=None):
"""Files a list of bugs to Monorail.
Args:
bugs: A list of MonorailIssue objects.
dry_run: A boolean, whether we are in dry run mode.
service_account_key_json: Optional, see docs for main().
"""
# TODO(robertma): Better error handling in this method.
if dry_run:
_log.info(
'[dry_run] Would have filed the %d bugs in the pending list.',
len(bugs))
return
_log.info('Filing %d bugs in the pending list to Monorail', len(bugs))
api = self._get_monorail_api(service_account_key_json)
for index, bug in enumerate(bugs, start=1):
response = api.insert_issue(bug)
_log.info('[%d] Filed bug: %s', index,
MonorailIssue.crbug_link(response['id']))
def _get_monorail_api(self, service_account_key_json):
if service_account_key_json:
return self._monorail_api(
service_account_key_json=service_account_key_json)
token = LuciAuth(self.host).get_access_token()
return self._monorail_api(access_token=token)
class WPTExpectationsUpdater(object):
MARKER_COMMENT = '# ====== New tests from wpt-importer added here ======'
UMBRELLA_BUG = 'crbug.com/626703'
def __init__(self, host, args=None):
self.host = host
self.port = self.host.port_factory.get()
self.finder = PathFinder(self.host.filesystem)
self.git_cl = GitCL(host)
self.git = self.host.git(self.finder.chromium_base())
self.configs_with_no_results = []
self.configs_with_all_pass = []
self.patchset = None
# Get options from command line arguments.
parser = argparse.ArgumentParser(description=__doc__)
self.add_arguments(parser)
self.options = parser.parse_args(args or [])
# Set up TestExpectations instance which contains all
# expectations files associated with the platform.
expectations_dict = {p: self.host.filesystem.read_text_file(p)
for p in self.expectations_files()}
self._test_expectations = TestExpectations(
self.port, expectations_dict=expectations_dict)
def expectations_files(self):
"""Returns list of expectations files.
Each expectation file in the list will be cleaned of expectations
for tests that were removed and will also have test names renamed
for tests that were renamed. Also the files may have their expectations
updated using builder results.
"""
return (self.port.all_expectations_dict().keys() +
PRODUCTS_TO_EXPECTATION_FILE_PATHS.values())
def run(self):
"""Does required setup before calling update_expectations().
Do not override this function!
"""
log_level = logging.DEBUG if self.options.verbose else logging.INFO
configure_logging(logging_level=log_level, include_time=True)
self.patchset = self.options.patchset
# Remove expectations for deleted tests and rename tests in expectations
# for renamed tests.
self.cleanup_test_expectations_files()
if not self.options.cleanup_test_expectations_only:
# Use try job results to update expectations and baselines
self.update_expectations()
return 0
def add_arguments(self, parser):
parser.add_argument(
'--patchset',
default=None,
help='Patchset number to fetch new baselines from.')
parser.add_argument(
'-v',
'--verbose',
action='store_true',
help='More verbose logging.')
parser.add_argument(
'--clean-up-affected-tests-only',
action='store_true',
help='Only cleanup expectations deleted or renamed in current CL. '
'If flag is not used then a full cleanup of deleted or '
'renamed tests will be done in expectations.')
parser.add_argument(
'--cleanup-test-expectations-only',
action='store_true',
help='Cleanup test expectations files and then exit script.')
# TODO(rmhasan): Move this argument to the
# AndroidWPTExpectationsUpdater add_arguments implementation.
# Also look into using sub parsers to separate android and
# desktop specific arguments.
parser.add_argument(
'--android-product', action='append', default=[],
help='Android products whose baselines will be updated.',
choices=PRODUCTS)
def update_expectations(self):
"""Downloads text new baselines and adds test expectations lines.
Returns:
A pair: A set of tests that are rebaselined, and a dictionary
mapping tests that couldn't be rebaselined to lists of expectation
lines written to TestExpectations.
"""
issue_number = self.get_issue_number()
if issue_number == 'None':
raise ScriptError('No issue on current branch.')
build_to_status = self.get_latest_try_jobs()
_log.debug('Latest try jobs: %r', build_to_status)
if not build_to_status:
raise ScriptError('No try job information was collected.')
# Here we build up a dict of failing test results for all platforms.
test_expectations = {}
for build, job_status in build_to_status.iteritems():
if job_status.result == 'SUCCESS':
self.configs_with_all_pass.extend(
self.get_builder_configs(build))
continue
result_dicts = self.get_failing_results_dicts(build)
for result_dict in result_dicts:
test_expectations = self.merge_dicts(
test_expectations, result_dict)
# At this point, test_expectations looks like: {
# 'test-with-failing-result': {
# config1: SimpleTestResult,
# config2: SimpleTestResult,
# config3: AnotherSimpleTestResult
# }
# }
# And then we merge results for different platforms that had the same results.
for test_name, platform_result in test_expectations.iteritems():
# platform_result is a dict mapping platforms to results.
test_expectations[test_name] = self.merge_same_valued_keys(
platform_result)
# At this point, test_expectations looks like: {
# 'test-with-failing-result': {
# (config1, config2): SimpleTestResult,
# (config3,): AnotherSimpleTestResult
# }
# }
rebaselined_tests, test_expectations = self.download_text_baselines(
test_expectations)
exp_lines_dict = self.write_to_test_expectations(test_expectations)
return rebaselined_tests, exp_lines_dict
def get_issue_number(self):
"""Returns current CL number. Can be replaced in unit tests."""
return self.git_cl.get_issue_number()
def get_latest_try_jobs(self):
"""Returns the latest finished try jobs as Build objects."""
return self.git_cl.latest_try_jobs(
builder_names=self._get_try_bots(), patchset=self.patchset)
def get_failing_results_dicts(self, build):
"""Returns a list of nested dicts of failing test results.
Retrieves a full list of web test results from a builder result URL.
Collects the builder name, platform and a list of tests that did not
run as expected.
Args:
build: A Build object.
Returns:
A list of dictionaries that have the following structure.
{
'test-with-failing-result': {
config: SimpleTestResult
}
}
If results could be fetched but none are failing,
this will return an empty list.
"""
test_results_list = self._get_web_test_results(build)
has_webdriver_tests = self.host.builders.has_webdriver_tests_for_builder(
build.builder_name)
if has_webdriver_tests:
master = self.host.builders.master_for_builder(build.builder_name)
test_results_list.append(
self.host.results_fetcher.fetch_webdriver_test_results(
build, master))
test_results_list = filter(None, test_results_list)
if not test_results_list:
_log.warning('No results for build %s', build)
self.configs_with_no_results.extend(self.get_builder_configs(build))
return []
failing_test_results = []
for results_set in test_results_list:
results_dict = self.generate_failing_results_dict(
build, results_set)
if results_dict:
failing_test_results.append(results_dict)
return failing_test_results
def _get_web_test_results(self, build):
"""Gets web tests results for a builder.
Args:
build: Named tuple containing builder name and number
Returns:
List of web tests results for each web test step
in build.
"""
return [self.host.results_fetcher.fetch_results(build)]
def get_builder_configs(self, build, *_):
return [DesktopConfig(port_name=self.port_name(build))]
@memoized
def port_name(self, build):
return self.host.builders.port_name_for_builder_name(
build.builder_name)
def generate_failing_results_dict(self, build, web_test_results):
"""Makes a dict with results for one platform.
Args:
builder: Builder instance containing builder information..
web_test_results: A list of WebTestResult objects.
Returns:
A dictionary with the structure: {
'test-name': {
('full-port-name',): SimpleTestResult
}
}
"""
test_dict = {}
configs = self.get_builder_configs(build, web_test_results)
_log.debug('Getting failing results dictionary'
' for %s step in latest %s build' % (
web_test_results.step_name(), build.builder_name))
if len(configs) > 1:
raise ScriptError('More than one configs were produced for'
' builder and web tests step combination')
if not configs:
raise ScriptError('No configuration was found for builder and web test'
' step combination ')
config = configs[0]
if config in self.configs_with_all_pass:
return {}
for result in web_test_results.didnt_run_as_expected_results():
# TODO(rmhasan) If a test fails unexpectedly then it runs multiple
# times until, it passes or a retry limit is reached. Even though
# it passed we there are still flaky failures that we are not
# creating test expectations for. Maybe we should add a mode
# which creates expectations for tests that are flaky but still
# pass in a web test step.
if result.did_pass():
continue
test_name = result.test_name()
if not self._is_wpt_test(test_name):
continue
test_dict[test_name] = {
config:
SimpleTestResult(
expected=result.expected_results(),
actual=result.actual_results(),
bug=self.UMBRELLA_BUG)
}
return test_dict
def _is_wpt_test(self, test_name):
"""Check if a web test is a WPT tests.
In blink web tests results, each test name is relative to
the web_tests directory instead of the wpt directory. We
need to use the port.is_wpt_test() function to find out if a test
is from the WPT suite.
Returns: True if a test is in the external/wpt subdirectory of
the web_tests directory."""
return self.port.is_wpt_test(test_name)
def merge_dicts(self, target, source, path=None):
"""Recursively merges nested dictionaries.
Args:
target: First dictionary, which is updated based on source.
source: Second dictionary, not modified.
path: A list of keys, only used for making error messages.
Returns:
The updated target dictionary.
"""
path = path or []
for key in source:
if key in target:
if (isinstance(target[key], dict)) and isinstance(
source[key], dict):
self.merge_dicts(target[key], source[key],
path + [str(key)])
elif target[key] == source[key]:
pass
else:
raise ValueError(
'The key: %s already exist in the target dictionary.' %
'.'.join(path))
else:
target[key] = source[key]
return target
def merge_same_valued_keys(self, dictionary):
"""Merges keys in dictionary with same value.
Traverses through a dict and compares the values of keys to one another.
If the values match, the keys are combined to a tuple and the previous
keys are removed from the dict.
Args:
dictionary: A dictionary with a dictionary as the value.
Returns:
A new dictionary with updated keys to reflect matching values of keys.
Example: {
'one': {'foo': 'bar'},
'two': {'foo': 'bar'},
'three': {'foo': 'bar'}
}
is converted to a new dictionary with that contains
{('one', 'two', 'three'): {'foo': 'bar'}}
"""
merged_dict = {}
matching_value_keys = set()
keys = sorted(dictionary.keys())
while keys:
current_key = keys[0]
found_match = False
if current_key == keys[-1]:
merged_dict[tuple([current_key])] = dictionary[current_key]
keys.remove(current_key)
break
for next_item in keys[1:]:
if dictionary[current_key] == dictionary[next_item]:
found_match = True
matching_value_keys.update([current_key, next_item])
if next_item == keys[-1]:
if found_match:
merged_dict[
tuple(matching_value_keys)] = dictionary[current_key]
keys = [
k for k in keys if k not in matching_value_keys
]
else:
merged_dict[tuple([current_key])] = dictionary[current_key]
keys.remove(current_key)
matching_value_keys = set()
return merged_dict
def get_expectations(self, result, test_name=''):
"""Returns a set of test expectations based on the result of a test.
Returns a set of one or more test expectations based on the expected
and actual results of a given test name. This function is to decide
expectations for tests that could not be rebaselined.
Args:
result: A SimpleTestResult.
test_name: The test name string (optional).
Returns:
A set of one or more test expectation strings with the first letter
capitalized. Example: {'Failure', 'Timeout'}.
"""
actual_results = set(result.actual.split())
# If the result is MISSING, this implies that the test was not
# rebaselined and has an actual result but no baseline. We can't
# add a Missing expectation (this is not allowed), but no other
# expectation is correct.
# We also want to skip any new manual tests that are not automated;
# see crbug.com/708241 for context.
if 'MISSING' in actual_results:
return {'Skip'}
if '-manual.' in test_name and 'TIMEOUT' in actual_results:
return {'Skip'}
expectations = set()
failure_types = {'TEXT', 'IMAGE+TEXT', 'IMAGE', 'AUDIO', 'FAIL'}
other_types = {'TIMEOUT', 'CRASH', 'PASS'}
for actual in actual_results:
if actual in failure_types:
expectations.add('Failure')
if actual in other_types:
expectations.add(actual.capitalize())
return expectations
def create_line_dict(self, merged_results):
"""Creates list of test expectations lines.
Traverses through the given |merged_results| dictionary and parses the
value to create one test expectations line per key.
Test expectation lines have the following format:
['BUG_URL [PLATFORM(S)] TEST_NAME [EXPECTATION(S)]']
Args:
merged_results: A dictionary with the format:
{
'test-with-failing-result': {
(config1, config2): SimpleTestResult,
(config3,): SimpleTestResult
}
}
Returns:
A dictionary from test names to a list of test expectation lines
(each SimpleTestResult turns into a line).
"""
line_dict = defaultdict(list)
for test_name, test_results in sorted(merged_results.iteritems()):
if not self._is_wpt_test(test_name):
_log.warning(
'Non-WPT test "%s" unexpectedly passed to create_line_dict.',
test_name)
continue
for configs, result in sorted(test_results.iteritems()):
line_dict[test_name].extend(
self._create_lines(test_name, configs, result))
return line_dict
def _create_lines(self, test_name, configs, result):
"""Constructs test expectation line strings.
Args:
test_name: The test name string.
configs: A list of full configs that the line should apply to.
result: A SimpleTestResult.
Returns:
A list of strings which each is a line of test expectation for given
|test_name|.
"""
lines = []
# The set of ports with no results is assumed to have have no
# overlap with the set of port names passed in here.
assert set(configs) & set(self.configs_with_no_results) == set()
# The ports with no results are generally ports of builders that
# failed, maybe for unrelated reasons. At this point, we add ports
# with no results to the list of platforms because we're guessing
# that this new expectation might be cross-platform and should
# also apply to any ports that we weren't able to get results for.
configs = tuple(list(configs) + self.configs_with_no_results)
expectations = '[ %s ]' % \
' '.join(self.get_expectations(result, test_name))
for specifier in self.normalized_specifiers(test_name, configs):
line_parts = []
if specifier:
line_parts.append('[ %s ]' % specifier)
# Escape literal asterisks for typ (https://crbug.com/1036130).
line_parts.append(test_name.replace('*', '\\*'))
line_parts.append(expectations)
# Only add the bug link if the expectations do not include WontFix.
if 'WontFix' not in expectations and result.bug:
line_parts.insert(0, result.bug)
lines.append(' '.join(line_parts))
return lines
def normalized_specifiers(self, test_name, configs):
"""Converts and simplifies ports into platform specifiers.
Args:
test_name: The test name string.
configs: A list of full configs that the line should apply to.
Returns:
A list of specifier string, e.g. ["Mac", "Win"].
[''] will be returned if the line should apply to all platforms.
"""
specifiers = []
for config in configs:
specifiers.append(
self.host.builders.version_specifier_for_port_name(
config.port_name))
if self.specifiers_can_extend_to_all_platforms(specifiers, test_name):
return ['']
specifiers = self.simplify_specifiers(
specifiers, self.port.configuration_specifier_macros())
if not specifiers:
return ['']
return specifiers
def specifiers_can_extend_to_all_platforms(self, specifiers, test_name):
"""Tests whether a list of specifiers can be extended to all platforms.
Tries to add skipped platform specifiers to the list and tests if the
extended list covers all platforms.
"""
extended_specifiers = specifiers + self.skipped_specifiers(test_name)
# If the list is simplified to empty, then all platforms are covered.
return not self.simplify_specifiers(
extended_specifiers, self.port.configuration_specifier_macros())
def skipped_specifiers(self, test_name):
"""Returns a list of platform specifiers for which the test is skipped."""
specifiers = []
for port in self.all_try_builder_ports():
if port.skips_test(test_name):
specifiers.append(
self.host.builders.version_specifier_for_port_name(
port.name()))
return specifiers
@memoized
def all_try_builder_ports(self):
"""Returns a list of Port objects for all try builders."""
return [
self.host.port_factory.get_from_builder_name(name)
for name in self._get_try_bots()
]
def simplify_specifiers(self, specifiers, specifier_macros):
"""Simplifies the specifier part of an expectation line if possible.
"Simplifying" means finding the shortest list of platform specifiers
that is equivalent to the given list of specifiers. This can be done
because there are "macro specifiers" that stand in for multiple version
specifiers, and an empty list stands in for "all platforms".
Args:
specifiers: A collection of specifiers (case insensitive).
specifier_macros: A dict mapping "macros" for groups of specifiers
to lists of version specifiers. e.g. {"win": ["win7", "win10"]}.
If there are versions in this dict for that have no corresponding
try bots, they are ignored.
Returns:
A shortened list of specifiers (capitalized). For example, ["win7",
"win10"] would be converted to ["Win"]. If the given list covers
all supported platforms, then an empty list is returned.
"""
specifiers = {s.lower() for s in specifiers}
covered_by_try_bots = self._platform_specifiers_covered_by_try_bots()
for macro, versions in specifier_macros.iteritems():
macro = macro.lower()
# Only consider version specifiers that have corresponding try bots.
versions = {
s.lower()
for s in versions if s.lower() in covered_by_try_bots
}
if len(versions) == 0:
continue
if versions <= specifiers:
specifiers -= versions
specifiers.add(macro)
if specifiers == {macro.lower() for macro in specifier_macros}:
return []
return sorted(specifier.capitalize() for specifier in specifiers)
def _platform_specifiers_covered_by_try_bots(self):
all_platform_specifiers = set()
for builder_name in self._get_try_bots():
all_platform_specifiers.add(
self.host.builders.platform_specifier_for_builder(
builder_name).lower())
return frozenset(all_platform_specifiers)
def write_to_test_expectations(self, test_expectations):
"""Writes the given lines to the TestExpectations file.
The place in the file where the new lines are inserted is after a marker
comment line. If this marker comment line is not found, then everything
including the marker line is appended to the end of the file.
All WontFix tests are inserted to NeverFixTests file instead of TextExpectations
file.
Args:
test_expectations: A dictionary mapping test names to a dictionary
mapping platforms and test results.
Returns:
Dictionary mapping test names to lists of test expectation strings.
"""
line_dict = self.create_line_dict(test_expectations)
if not line_dict:
_log.info(
'No lines to write to TestExpectations,'
' WebdriverExpectations or NeverFixTests.'
)
return {}
line_list = []
wont_fix_list = []
webdriver_list = []
for lines in line_dict.itervalues():
for line in lines:
if 'Skip' in line and '-manual.' in line:
wont_fix_list.append(line)
elif self.finder.webdriver_prefix() in line:
webdriver_list.append(line)
else:
line_list.append(line)
list_to_expectation = {
self.port.path_to_generic_test_expectations_file(): line_list,
self.port.path_to_webdriver_expectations_file(): webdriver_list
}
for expectations_file_path, lines in list_to_expectation.iteritems():
if not lines:
continue
_log.info('Lines to write to %s:\n %s', expectations_file_path,
'\n'.join(lines))
# Writes to TestExpectations file.
file_contents = self.host.filesystem.read_text_file(
expectations_file_path)
marker_comment_index = file_contents.find(self.MARKER_COMMENT)
if marker_comment_index == -1:
file_contents += '\n%s\n' % self.MARKER_COMMENT
file_contents += '\n'.join(lines)
else:
end_of_marker_line = (file_contents[marker_comment_index:].
find('\n')) + marker_comment_index
file_contents = (
file_contents[:end_of_marker_line + 1] + '\n'.join(lines) +
file_contents[end_of_marker_line:])
self.host.filesystem.write_text_file(expectations_file_path,
file_contents)
if wont_fix_list:
_log.info('Lines to write to NeverFixTests:\n %s',
'\n'.join(wont_fix_list))
# Writes to NeverFixTests file.
wont_fix_path = self.port.path_to_never_fix_tests_file()
wont_fix_file_content = self.host.filesystem.read_text_file(
wont_fix_path)
if not wont_fix_file_content.endswith('\n'):
wont_fix_file_content += '\n'
wont_fix_file_content += '\n'.join(wont_fix_list)
wont_fix_file_content += '\n'
self.host.filesystem.write_text_file(wont_fix_path,
wont_fix_file_content)
return line_dict
def cleanup_test_expectations_files(self):
"""Removes deleted tests from expectations files.
Removes expectations for deleted tests or renames test names in
expectation files for tests that were renamed. If the
--clean-up-affected-tests-only command line argument is used then
only tests deleted in the CL will have their expectations removed
through this script. If that command line argument is not used then
expectations for test files that no longer exist will be deleted.
"""
deleted_test_files = self._list_deleted_test_files()
renamed_test_files = self._list_renamed_test_files()
for path in self._test_expectations.expectations_dict:
_log.info(
'Updating %s for any removed or renamed tests.' %
self.host.filesystem.basename(path))
self._clean_single_test_expectations_file(
path, deleted_test_files, renamed_test_files)
self._test_expectations.commit_changes()
def _clean_single_test_expectations_file(
self, path, deleted_files, renamed_files):
"""Cleans up a single test expectations file.
Args:
path: Path of expectations file that is being cleaned up.
deleted_files: List of test file paths relative to the web tests
directory which were deleted.
renamed_files: Dictionary mapping test file paths to their new file
name after renaming.
"""
for line in self._test_expectations.get_updated_lines(path):
# if a test is a glob type expectation or empty line or comment then
# add it to the updated expectations file without modifications
if not line.test or line.is_glob:
continue
root_test_file = self._get_root_file(line.test)
if root_test_file in renamed_files:
self._test_expectations.remove_expectations(path, [line])
new_file_name = renamed_files[root_test_file]
if self.finder.is_webdriver_test_path(root_test_file):
_, subtest_suffix = self.port.split_webdriver_test_name(
line.test)
line.test = self.port.add_webdriver_subtest_suffix(
new_file_name, subtest_suffix)
elif '?' in line.test:
line.test = (
new_file_name + line.test[line.test.find('?'):])
else:
line.test = new_file_name
self._test_expectations.add_expectations(
path, [line], lineno=line.lineno)
elif root_test_file in deleted_files:
self._test_expectations.remove_expectations(
path, [line])
@memoized
def _get_root_file(self, test_name):
"""Strips arguments from a web test name in order to get the file name.
It also removes the arguments for web driver tests. For instances for
the test test1/example.html?Hello this function will return
test1/example.html. For a webdriver test it would include arguments and
would have the following format, {test file}>>{argument}.
Args:
test_name: Test name which may include test arguments.
Returns:
Returns the test file which is the root of a test.
"""
if self.finder.is_webdriver_test_path(test_name):
root_test_file, _ = (
self.port.split_webdriver_test_name(test_name))
elif '?' in test_name:
root_test_file = test_name[:test_name.find('?')]
else:
root_test_file = test_name
return root_test_file
def _list_deleted_test_files(self):
"""Returns a list of web tests that have been deleted.
If --clean-up-affected-tests-only is true then only test files deleted
in the current CL may be removed from expectations. Otherwise, any test
file may be removed from expectations if it has been deleted.
Returns: A list of web test files that have been deleted.
"""
if self.options.clean_up_affected_tests_only:
# TODO(robertma): Improve Git.changed_files so that we can use
# it here.
paths = set(self.git.run(
['diff', 'origin/master', '-M100%', '--diff-filter=D',
'--name-only']).splitlines())
deleted_tests = set()
for path in paths:
test = self._relative_to_web_test_dir(path)
if test:
deleted_tests.add(test)
else:
# Remove expectations for all test which have files that
# were deleted. Paths are already relative to the web_tests
# directory
deleted_tests = self._deleted_test_files_in_expectations()
return deleted_tests
def _list_renamed_test_files(self):
"""Returns a dictionary mapping tests to their new name.
Regardless of the command line arguments used this test will only
return a dictionary for tests affected in the current CL.
Returns a dictionary mapping source name to destination name.
"""
out = self.git.run(
['diff', 'origin/master', '-M100%', '--diff-filter=R',
'--name-status'])
renamed_tests = {}
for line in out.splitlines():
_, source_path, dest_path = line.split()
source_test = self._relative_to_web_test_dir(source_path)
dest_test = self._relative_to_web_test_dir(dest_path)
if source_test and dest_test:
renamed_tests[source_test] = dest_test
return renamed_tests
def _relative_to_web_test_dir(self, path_relative_to_repo_root):
"""Returns a path that's relative to the web tests directory."""
abs_path = self.finder.path_from_chromium_base(
path_relative_to_repo_root)
if not abs_path.startswith(self.finder.web_tests_dir()):
return None
return self.host.filesystem.relpath(
abs_path, self.finder.web_tests_dir())
def _deleted_test_files_in_expectations(self):
"""Returns a list of test files that were deleted.
Returns a list of test file names that are still in the expectations
files but no longer exists in the web tests directory.
"""
deleted_files = set()
existing_files = {
self._get_root_file(p)
for p in self.port.tests()}
for path in self._test_expectations.expectations_dict:
for line in self._test_expectations.get_updated_lines(path):
if not line.test or line.is_glob:
continue
root_test_file = self._get_root_file(line.test)
if root_test_file not in existing_files:
deleted_files.add(root_test_file)
return deleted_files
# TODO(robertma): Unit test this method.
def download_text_baselines(self, test_results):
"""Fetches new baseline files for tests that should be rebaselined.
Invokes `blink_tool.py rebaseline-cl` in order to download new baselines
(-expected.txt files) for testharness.js tests that did not crash or
time out. Then, the platform-specific test is removed from the overall
failure test dictionary and the resulting dictionary is returned.
Args:
test_results: A dictionary of failing test results, mapping test
names to lists of platforms to SimpleTestResult.
Returns:
A pair: A set of tests that are rebaselined, and a modified copy of
the test_results dictionary containing only tests that couldn't be
rebaselined.
"""
tests_to_rebaseline, test_results = self.get_tests_to_rebaseline(
test_results)
if not tests_to_rebaseline:
_log.info('No tests to rebaseline.')
return tests_to_rebaseline, test_results
_log.info('Tests to rebaseline:')
for test in tests_to_rebaseline:
_log.info(' %s', test)
blink_tool = self.finder.path_from_blink_tools('blink_tool.py')
command = [
'python',
blink_tool,
'rebaseline-cl',
'--verbose',
'--no-trigger-jobs',
'--fill-missing',
]
if self.patchset:
command.append('--patchset=' + str(self.patchset))
command += tests_to_rebaseline
self.host.executive.run_command(command)
return tests_to_rebaseline, test_results
def get_tests_to_rebaseline(self, test_results):
"""Filters failing tests that can be rebaselined.
Creates a list of tests to rebaseline depending on the tests' platform-
specific results. In general, this will be non-ref tests that failed
due to a baseline mismatch (rather than crash or timeout).
Args:
test_results: A dictionary of failing test results, mapping test
names to lists of platforms to SimpleTestResult.
Returns:
A pair: A set of tests to be rebaselined, and a modified copy of
the test_results dictionary. The tests to be rebaselined should
include testharness.js tests that failed due to a baseline mismatch.
"""
new_test_results = copy.deepcopy(test_results)
tests_to_rebaseline = set()
for test_name in test_results:
for platforms, result in test_results[test_name].iteritems():
if self.can_rebaseline(test_name, result):
del new_test_results[test_name][platforms]
tests_to_rebaseline.add(test_name)
return sorted(tests_to_rebaseline), new_test_results
def can_rebaseline(self, test_name, result):
"""Checks if a test can be rebaselined.
Args:
test_name: The test name string.
result: A SimpleTestResult.
"""
if self.is_reference_test(test_name):
return False
if any(x in result.actual for x in ('CRASH', 'TIMEOUT', 'MISSING')):
return False
if self.is_webdriver_test(test_name):
return False
return True
def is_reference_test(self, test_name):
"""Checks whether a given test is a reference test."""
return bool(self.port.reference_files(test_name))
def is_webdriver_test(self, test_name):
"""Checks whether a given test is a WebDriver test."""
return self.finder.is_webdriver_test_path(test_name)
@memoized
def _get_try_bots(self):
return self.host.builders.filter_builders(
is_try=True, exclude_specifiers={'android'})
def test_web_tests_dir(self):
finder = PathFinder(MockFileSystem())
self.assertEqual(finder.web_tests_dir(),
'/mock-checkout/' + RELATIVE_WEB_TESTS[:-1])
class TestImporter(object):
def __init__(self, host, wpt_github=None):
self.host = host
self.wpt_github = wpt_github
self.executive = host.executive
self.fs = host.filesystem
self.finder = PathFinder(self.fs)
self.chromium_git = self.host.git(self.finder.chromium_base())
self.dest_path = self.finder.path_from_web_tests('external', 'wpt')
# A common.net.git_cl.GitCL instance.
self.git_cl = None
# Another Git instance with local WPT as CWD, which can only be
# instantiated after the working directory is created.
self.wpt_git = None
# The WPT revision we are importing and the one imported last time.
self.wpt_revision = None
self.last_wpt_revision = None
# A set of rebaselined tests and a dictionary of new test expectations
# mapping failing tests to platforms to
# wpt_expectations_updater.SimpleTestResult.
self.rebaselined_tests = set()
self.new_test_expectations = {}
self.verbose = False
def main(self, argv=None):
# TODO(robertma): Test this method! Split it to make it easier to test
# if necessary.
options = self.parse_args(argv)
self.verbose = options.verbose
log_level = logging.DEBUG if self.verbose else logging.INFO
configure_logging(logging_level=log_level, include_time=True)
if options.verbose:
# Print out the full output when executive.run_command fails.
self.host.executive.error_output_limit = None
if options.auto_update and options.auto_upload:
_log.error(
'--auto-upload and --auto-update cannot be used together.')
return 1
if not self.checkout_is_okay():
return 1
credentials = read_credentials(self.host, options.credentials_json)
gh_user = credentials.get('GH_USER')
gh_token = credentials.get('GH_TOKEN')
if not gh_user or not gh_token:
_log.warning('You have not set your GitHub credentials. This '
'script may fail with a network error when making '
'an API request to GitHub.')
_log.warning('See https://chromium.googlesource.com/chromium/src'
'/+/master/docs/testing/web_platform_tests.md'
'#GitHub-credentials for instructions on how to set '
'your credentials up.')
self.wpt_github = self.wpt_github or WPTGitHub(self.host, gh_user,
gh_token)
self.git_cl = GitCL(
self.host, auth_refresh_token_json=options.auth_refresh_token_json)
_log.debug('Noting the current Chromium revision.')
chromium_revision = self.chromium_git.latest_git_commit()
# Instantiate Git after local_wpt.fetch() to make sure the path exists.
local_wpt = LocalWPT(self.host, gh_token=gh_token)
local_wpt.fetch()
self.wpt_git = self.host.git(local_wpt.path)
if options.revision is not None:
_log.info('Checking out %s', options.revision)
self.wpt_git.run(['checkout', options.revision])
_log.debug('Noting the revision we are importing.')
self.wpt_revision = self.wpt_git.latest_git_commit()
self.last_wpt_revision = self._get_last_imported_wpt_revision()
import_commit = 'wpt@%s' % self.wpt_revision
_log.info('Importing %s to Chromium %s', import_commit,
chromium_revision)
if options.ignore_exportable_commits:
commit_message = self._commit_message(chromium_revision,
import_commit)
else:
commits = self.apply_exportable_commits_locally(local_wpt)
if commits is None:
_log.error('Could not apply some exportable commits cleanly.')
_log.error('Aborting import to prevent clobbering commits.')
return 1
commit_message = self._commit_message(
chromium_revision,
import_commit,
locally_applied_commits=commits)
self._clear_out_dest_path()
_log.info('Copying the tests from the temp repo to the destination.')
test_copier = TestCopier(self.host, local_wpt.path)
test_copier.do_import()
# TODO(robertma): Implement `add --all` in Git (it is different from `commit --all`).
self.chromium_git.run(['add', '--all', self.dest_path])
self._generate_manifest()
# TODO(crbug.com/800570 robertma): Re-enable it once we fix the bug.
# self._delete_orphaned_baselines()
_log.info(
'Updating TestExpectations for any removed or renamed tests.')
self.update_all_test_expectations_files(self._list_deleted_tests(),
self._list_renamed_tests())
if not self.chromium_git.has_working_directory_changes():
_log.info('Done: no changes to import.')
return 0
if self._only_wpt_manifest_changed():
_log.info('Only manifest was updated; skipping the import.')
return 0
self._commit_changes(commit_message)
_log.info('Changes imported and committed.')
if not options.auto_upload and not options.auto_update:
return 0
self._upload_cl()
_log.info('Issue: %s', self.git_cl.run(['issue']).strip())
if not self.update_expectations_for_cl():
return 1
if not options.auto_update:
return 0
if not self.run_commit_queue_for_cl():
return 1
if not self.send_notifications(local_wpt, options.auto_file_bugs,
options.monorail_auth_json):
return 1
return 0
def update_expectations_for_cl(self):
"""Performs the expectation-updating part of an auto-import job.
This includes triggering try jobs and waiting; then, if applicable,
writing new baselines and TestExpectation lines, committing, and
uploading a new patchset.
This assumes that there is CL associated with the current branch.
Returns True if everything is OK to continue, or False on failure.
"""
_log.info('Triggering try jobs for updating expectations.')
self.git_cl.trigger_try_jobs(self.blink_try_bots())
cl_status = self.git_cl.wait_for_try_jobs(
poll_delay_seconds=POLL_DELAY_SECONDS,
timeout_seconds=TIMEOUT_SECONDS)
if not cl_status:
_log.error('No initial try job results, aborting.')
self.git_cl.run(['set-close'])
return False
if cl_status.status == 'closed':
_log.error('The CL was closed, aborting.')
return False
_log.info('All jobs finished.')
try_results = cl_status.try_job_results
if try_results and self.git_cl.some_failed(try_results):
self.fetch_new_expectations_and_baselines()
if self.chromium_git.has_working_directory_changes():
self._generate_manifest()
message = 'Update test expectations and baselines.'
self._commit_changes(message)
self._upload_patchset(message)
return True
def run_commit_queue_for_cl(self):
"""Triggers CQ and either commits or aborts; returns True on success."""
_log.info('Triggering CQ try jobs.')
self.git_cl.run(['try'])
cl_status = self.git_cl.wait_for_try_jobs(
poll_delay_seconds=POLL_DELAY_SECONDS,
timeout_seconds=TIMEOUT_SECONDS,
cq_only=True)
if not cl_status:
self.git_cl.run(['set-close'])
_log.error('Timed out waiting for CQ; aborting.')
return False
if cl_status.status == 'closed':
_log.error('The CL was closed; aborting.')
return False
_log.info('All jobs finished.')
cq_try_results = cl_status.try_job_results
if not cq_try_results:
_log.error('No CQ try results found in try results')
self.git_cl.run(['set-close'])
return False
if not self.git_cl.all_success(cq_try_results):
_log.error('CQ appears to have failed; aborting.')
self.git_cl.run(['set-close'])
return False
_log.info('CQ appears to have passed; trying to commit.')
self.git_cl.run(['upload', '-f', '--send-mail']) # Turn off WIP mode.
self.git_cl.run(['set-commit'])
if self.git_cl.wait_for_closed_status():
_log.info('Update completed.')
return True
_log.error('Cannot submit CL; aborting.')
try:
self.git_cl.run(['set-close'])
except ScriptError as e:
if e.output and 'Conflict: change is merged' in e.output:
_log.error('CL is already merged; treating as success.')
return True
else:
raise e
return False
def blink_try_bots(self):
"""Returns the collection of builders used for updating expectations."""
return self.host.builders.filter_builders(
is_try=True, exclude_specifiers={'android'})
def parse_args(self, argv):
parser = argparse.ArgumentParser()
parser.description = __doc__
parser.add_argument(
'-v',
'--verbose',
action='store_true',
help='log extra details that may be helpful when debugging')
parser.add_argument(
'--ignore-exportable-commits',
action='store_true',
help='do not check for exportable commits that would be clobbered')
parser.add_argument('-r', '--revision', help='target wpt revision')
parser.add_argument(
'--auto-upload',
action='store_true',
help='upload a CL, update expectations, but do NOT trigger CQ')
parser.add_argument(
'--auto-update',
action='store_true',
help='upload a CL, update expectations, and trigger CQ')
parser.add_argument(
'--auto-file-bugs',
action='store_true',
help='file new failures automatically to crbug.com')
parser.add_argument(
'--auth-refresh-token-json',
help='authentication refresh token JSON file used for try jobs, '
'generally not necessary on developer machines')
parser.add_argument('--credentials-json',
help='A JSON file with GitHub credentials, '
'generally not necessary on developer machines')
parser.add_argument(
'--monorail-auth-json',
help='A JSON file containing the private key of a service account '
'to access Monorail (crbug.com), only needed when '
'--auto-file-bugs is used')
return parser.parse_args(argv)
def checkout_is_okay(self):
if self.chromium_git.has_working_directory_changes():
_log.warning('Checkout is dirty; aborting.')
return False
# TODO(robertma): Add a method in Git to query a range of commits.
local_commits = self.chromium_git.run(
['log', '--oneline', 'origin/master..HEAD'])
if local_commits:
_log.warning('Checkout has local commits before import.')
return True
def apply_exportable_commits_locally(self, local_wpt):
"""Applies exportable Chromium changes to the local WPT repo.
The purpose of this is to avoid clobbering changes that were made in
Chromium but not yet merged upstream. By applying these changes to the
local copy of web-platform-tests before copying files over, we make
it so that the resulting change in Chromium doesn't undo the
previous Chromium change.
Args:
A LocalWPT instance for our local copy of WPT.
Returns:
A list of commits applied (could be empty), or None if any
of the patches could not be applied cleanly.
"""
commits = self.exportable_but_not_exported_commits(local_wpt)
for commit in commits:
_log.info('Applying exportable commit locally:')
_log.info(commit.url())
_log.info('Subject: %s', commit.subject().strip())
# Log a note about the corresponding PR.
# This might not be necessary, and could potentially be removed.
pull_request = self.wpt_github.pr_for_chromium_commit(commit)
if pull_request:
_log.info('PR: %spull/%d', WPT_GH_URL, pull_request.number)
else:
_log.warning('No pull request found.')
error = local_wpt.apply_patch(commit.format_patch())
if error:
_log.error('Commit cannot be applied cleanly:')
_log.error(error)
return None
self.wpt_git.commit_locally_with_message('Applying patch %s' %
commit.sha)
return commits
def exportable_but_not_exported_commits(self, local_wpt):
"""Returns a list of commits that would be clobbered by importer.
The list contains all exportable but not exported commits, not filtered
by whether they can apply cleanly.
"""
# The errors returned by exportable_commits_over_last_n_commits are
# irrelevant and ignored here, because it tests patches *individually*
# while the importer tries to reapply these patches *cumulatively*.
commits, _ = exportable_commits_over_last_n_commits(
self.host,
local_wpt,
self.wpt_github,
require_clean=False,
verify_merged_pr=True)
return commits
def _generate_manifest(self):
"""Generates MANIFEST.json for imported tests.
Runs the (newly-updated) manifest command if it's found, and then
stages the generated MANIFEST.json in the git index, ready to commit.
"""
_log.info('Generating MANIFEST.json')
WPTManifest.generate_manifest(self.host, self.dest_path)
manifest_path = self.fs.join(self.dest_path, 'MANIFEST.json')
assert self.fs.exists(manifest_path)
manifest_base_path = self.fs.normpath(
self.fs.join(self.dest_path, '..', BASE_MANIFEST_NAME))
self.copyfile(manifest_path, manifest_base_path)
self.chromium_git.add_list([manifest_base_path])
def _clear_out_dest_path(self):
"""Removes all files that are synced with upstream from Chromium WPT.
Instead of relying on TestCopier to overwrite these files, cleaning up
first ensures if upstream deletes some files, we also delete them.
"""
_log.info('Cleaning out tests from %s.', self.dest_path)
should_remove = lambda fs, dirname, basename: (is_file_exportable(
fs.relpath(fs.join(dirname, basename), self.finder.chromium_base())
))
files_to_delete = self.fs.files_under(self.dest_path,
file_filter=should_remove)
for subpath in files_to_delete:
self.remove(self.finder.path_from_web_tests('external', subpath))
def _commit_changes(self, commit_message):
_log.info('Committing changes.')
self.chromium_git.commit_locally_with_message(commit_message)
def _only_wpt_manifest_changed(self):
changed_files = self.chromium_git.changed_files()
wpt_base_manifest = self.fs.relpath(
self.fs.join(self.dest_path, '..', BASE_MANIFEST_NAME),
self.finder.chromium_base())
return changed_files == [wpt_base_manifest]
def _commit_message(self,
chromium_commit_sha,
import_commit_sha,
locally_applied_commits=None):
message = 'Import {}\n\nUsing wpt-import in Chromium {}.\n'.format(
import_commit_sha, chromium_commit_sha)
if locally_applied_commits:
message += 'With Chromium commits locally applied on WPT:\n'
message += '\n'.join(
str(commit) for commit in locally_applied_commits)
message += '\nNo-Export: true'
return message
def _delete_orphaned_baselines(self):
_log.info('Deleting any orphaned baselines.')
is_baseline_filter = lambda fs, dirname, basename: is_testharness_baseline(
basename)
baselines = self.fs.files_under(self.dest_path,
file_filter=is_baseline_filter)
# Note about possible refactoring:
# - the manifest path could be factored out to a common location, and
# - the logic for reading the manifest could be factored out from here
# and the Port class.
manifest_path = self.finder.path_from_web_tests(
'external', 'wpt', 'MANIFEST.json')
manifest = WPTManifest(self.fs.read_text_file(manifest_path))
wpt_urls = manifest.all_urls()
# Currently baselines for tests with query strings are merged,
# so that the tests foo.html?r=1 and foo.html?r=2 both have the same
# baseline, foo-expected.txt.
# TODO(qyearsley): Remove this when this behavior is fixed.
wpt_urls = [url.split('?')[0] for url in wpt_urls]
wpt_dir = self.finder.path_from_web_tests('external', 'wpt')
for full_path in baselines:
rel_path = self.fs.relpath(full_path, wpt_dir)
if not self._has_corresponding_test(rel_path, wpt_urls):
self.fs.remove(full_path)
def _has_corresponding_test(self, rel_path, wpt_urls):
# TODO(qyearsley): Ensure that this works with platform baselines and
# virtual baselines, and add unit tests.
base = '/' + rel_path.replace('-expected.txt', '')
return any(
(base + ext) in wpt_urls for ext in Port.supported_file_extensions)
def copyfile(self, source, destination):
_log.debug('cp %s %s', source, destination)
self.fs.copyfile(source, destination)
def remove(self, dest):
_log.debug('rm %s', dest)
self.fs.remove(dest)
def _upload_patchset(self, message):
self.git_cl.run(['upload', '-f', '-t', message])
def _upload_cl(self):
_log.info('Uploading change list.')
directory_owners = self.get_directory_owners()
description = self._cl_description(directory_owners)
sheriff_email = self.tbr_reviewer()
temp_file, temp_path = self.fs.open_text_tempfile()
temp_file.write(description)
temp_file.close()
self.git_cl.run([
'upload',
'-f',
'--message-file',
temp_path,
'--tbrs',
sheriff_email,
# Note: we used to CC all the directory owners, but have stopped
# in search of a better notification mechanism. (crbug.com/765334)
'--cc',
'*****@*****.**',
])
self.fs.remove(temp_path)
def get_directory_owners(self):
"""Returns a mapping of email addresses to owners of changed tests."""
_log.info('Gathering directory owners emails to CC.')
changed_files = self.chromium_git.changed_files()
extractor = DirectoryOwnersExtractor(self.fs)
return extractor.list_owners(changed_files)
def _cl_description(self, directory_owners):
"""Returns a CL description string.
Args:
directory_owners: A dict of tuples of owner names to lists of directories.
"""
# TODO(robertma): Add a method in Git for getting the commit body.
description = self.chromium_git.run(['log', '-1', '--format=%B'])
description += (
'Note to sheriffs: This CL imports external tests and adds\n'
'expectations for those tests; if this CL is large and causes\n'
'a few new failures, please fix the failures by adding new\n'
'lines to TestExpectations rather than reverting. See:\n'
'https://chromium.googlesource.com'
'/chromium/src/+/master/docs/testing/web_platform_tests.md\n\n')
if directory_owners:
description += self._format_directory_owners(
directory_owners) + '\n\n'
# Prevent FindIt from auto-reverting import CLs.
description += 'NOAUTOREVERT=true\n'
# Move any No-Export tag to the end of the description.
description = description.replace('No-Export: true', '')
description = description.replace('\n\n\n\n', '\n\n')
description += 'No-Export: true'
return description
@staticmethod
def _format_directory_owners(directory_owners):
message_lines = ['Directory owners for changes in this CL:']
for owner_tuple, directories in sorted(directory_owners.items()):
message_lines.append(', '.join(owner_tuple) + ':')
message_lines.extend(' ' + d for d in directories)
return '\n'.join(message_lines)
def tbr_reviewer(self):
"""Returns the user name or email address to use as the reviewer.
This tries to fetch the current ecosystem infra sheriff, but falls back
in case of error.
Either a user name (which is assumed to have a chromium.org email
address) or a full email address (for other cases) is returned.
"""
username = ''
try:
username = self._fetch_ecosystem_infra_sheriff_username()
except (IOError, KeyError, ValueError) as error:
_log.error('Exception while fetching current sheriff: %s', error)
if username in ['kyleju']:
_log.warning('Cannot TBR by %s: not a committer', username)
username = ''
return username or TBR_FALLBACK
def _fetch_ecosystem_infra_sheriff_username(self):
try:
content = self.host.web.get_binary(ROTATIONS_URL)
except NetworkTimeout:
_log.error('Cannot fetch %s', ROTATIONS_URL)
return ''
data = json.loads(content)
today = datetime.date.fromtimestamp(self.host.time()).isoformat()
index = data['rotations'].index('ecosystem_infra')
calendar = data['calendar']
for entry in calendar:
if entry['date'] == today:
if not entry['participants'][index]:
_log.info('No sheriff today.')
return ''
return entry['participants'][index][0]
_log.error('No entry found for date %s in rotations table.', today)
return ''
def fetch_new_expectations_and_baselines(self):
"""Modifies expectation lines and baselines based on try job results.
Assuming that there are some try job results available, this
adds new expectation lines to TestExpectations and downloads new
baselines based on the try job results.
This is the same as invoking the `wpt-update-expectations` script.
"""
_log.info('Adding test expectations lines to TestExpectations.')
expectation_updater = WPTExpectationsUpdater(self.host)
self.rebaselined_tests, self.new_test_expectations = expectation_updater.update_expectations(
)
def update_all_test_expectations_files(self, deleted_tests, renamed_tests):
"""Updates all test expectations files for tests that have been deleted or renamed.
This is only for deleted or renamed tests in the initial import,
not for tests that have failures in try jobs.
"""
port = self.host.port_factory.get()
for path, file_contents in port.all_expectations_dict().iteritems():
self._update_single_test_expectations_file(port, path,
file_contents,
deleted_tests,
renamed_tests)
def _update_single_test_expectations_file(self, port, path, file_contents,
deleted_tests, renamed_tests):
"""Updates a single test expectations file."""
test_expectations = TestExpectations(
port, expectations_dict={path: file_contents})
new_lines = []
for line in test_expectations.get_updated_lines(path):
# if a test is a glob type expectation or empty line or comment then add it to the updated
# expectations file without modifications
if not line.test or line.is_glob:
new_lines.append(line.to_string())
continue
test_name = line.test
if self.finder.is_webdriver_test_path(test_name):
root_test_file, subtest_suffix = port.split_webdriver_test_name(
test_name)
else:
root_test_file = test_name
if root_test_file in deleted_tests:
continue
if root_test_file in renamed_tests:
if self.finder.is_webdriver_test_path(root_test_file):
renamed_test = renamed_tests[root_test_file]
test_name = port.add_webdriver_subtest_suffix(
renamed_test, subtest_suffix)
else:
test_name = renamed_tests[root_test_file]
line.test = test_name
new_lines.append(line.to_string())
self.host.filesystem.write_text_file(path, '\n'.join(new_lines) + '\n')
def _list_deleted_tests(self):
"""List of web tests that have been deleted."""
# TODO(robertma): Improve Git.changed_files so that we can use it here.
out = self.chromium_git.run([
'diff', 'origin/master', '-M100%', '--diff-filter=D', '--name-only'
])
deleted_tests = []
for path in out.splitlines():
test = self._relative_to_web_test_dir(path)
if test:
deleted_tests.append(test)
return deleted_tests
def _list_renamed_tests(self):
"""Lists tests that have been renamed.
Returns a dict mapping source name to destination name.
"""
out = self.chromium_git.run([
'diff', 'origin/master', '-M100%', '--diff-filter=R',
'--name-status'
])
renamed_tests = {}
for line in out.splitlines():
_, source_path, dest_path = line.split()
source_test = self._relative_to_web_test_dir(source_path)
dest_test = self._relative_to_web_test_dir(dest_path)
if source_test and dest_test:
renamed_tests[source_test] = dest_test
return renamed_tests
def _relative_to_web_test_dir(self, path_relative_to_repo_root):
"""Returns a path that's relative to the web tests directory."""
abs_path = self.finder.path_from_chromium_base(
path_relative_to_repo_root)
if not abs_path.startswith(self.finder.web_tests_dir()):
return None
return self.fs.relpath(abs_path, self.finder.web_tests_dir())
def _get_last_imported_wpt_revision(self):
"""Finds the last imported WPT revision."""
# TODO(robertma): Only match commit subjects.
output = self.chromium_git.most_recent_log_matching(
'^Import wpt@', self.finder.chromium_base())
# No line-start anchor (^) below because of the formatting of output.
result = re.search(r'Import wpt@(\w+)', output)
if result:
return result.group(1)
else:
_log.error('Cannot find last WPT import.')
return None
def send_notifications(self, local_wpt, auto_file_bugs,
monorail_auth_json):
issue = self.git_cl.run(['status', '--field=id']).strip()
patchset = self.git_cl.run(['status', '--field=patch']).strip()
# Construct the notifier here so that any errors won't affect the import.
notifier = ImportNotifier(self.host, self.chromium_git, local_wpt)
notifier.main(self.last_wpt_revision,
self.wpt_revision,
self.rebaselined_tests,
self.new_test_expectations,
issue,
patchset,
dry_run=not auto_file_bugs,
service_account_key_json=monorail_auth_json)
return True
class ImportNotifier(object):
def __init__(self, host, chromium_git, local_wpt):
self.host = host
self.git = chromium_git
self.local_wpt = local_wpt
self._monorail_api = MonorailAPI
self.default_port = host.port_factory.get()
self.finder = PathFinder(host.filesystem)
self.owners_extractor = DirectoryOwnersExtractor(host.filesystem)
self.new_failures_by_directory = defaultdict(list)
def main(self,
wpt_revision_start,
wpt_revision_end,
rebaselined_tests,
test_expectations,
issue,
patchset,
dry_run=True,
service_account_key_json=None):
"""Files bug reports for new failures.
Args:
wpt_revision_start: The start of the imported WPT revision range
(exclusive), i.e. the last imported revision.
wpt_revision_end: The end of the imported WPT revision range
(inclusive), i.e. the current imported revision.
rebaselined_tests: A list of test names that have been rebaselined.
test_expectations: A dictionary mapping names of tests that cannot
be rebaselined to a list of new test expectation lines.
issue: The issue number of the import CL (a string).
patchset: The patchset number of the import CL (a string).
dry_run: If True, no bugs will be actually filed to crbug.com.
service_account_key_json: The path to a JSON private key of a
service account for accessing Monorail. If None, try to get an
access token from luci-auth.
Note: "test names" are paths of the tests relative to web_tests.
"""
gerrit_url = SHORT_GERRIT_PREFIX + issue
gerrit_url_with_ps = gerrit_url + '/' + patchset + '/'
changed_test_baselines = self.find_changed_baselines_of_tests(
rebaselined_tests)
self.examine_baseline_changes(changed_test_baselines,
gerrit_url_with_ps)
self.examine_new_test_expectations(test_expectations)
bugs = self.create_bugs_from_new_failures(wpt_revision_start,
wpt_revision_end, gerrit_url)
self.file_bugs(bugs, dry_run, service_account_key_json)
def find_changed_baselines_of_tests(self, rebaselined_tests):
"""Finds the corresponding changed baselines of each test.
Args:
rebaselined_tests: A list of test names that have been rebaselined.
Returns:
A dictionary mapping test names to paths of their baselines changed
in this import CL (paths relative to the root of Chromium repo).
"""
test_baselines = {}
changed_files = self.git.changed_files()
for test_name in rebaselined_tests:
test_without_ext, _ = self.host.filesystem.splitext(test_name)
changed_baselines = []
# TODO(robertma): Refactor this into web_tests.port.base.
baseline_name = test_without_ext + '-expected.txt'
for changed_file in changed_files:
if changed_file.endswith(baseline_name):
changed_baselines.append(changed_file)
if changed_baselines:
test_baselines[test_name] = changed_baselines
return test_baselines
def examine_baseline_changes(self, changed_test_baselines,
gerrit_url_with_ps):
"""Examines all changed baselines to find new failures.
Args:
changed_test_baselines: A dictionary mapping test names to paths of
changed baselines.
gerrit_url_with_ps: Gerrit URL of this CL with the patchset number.
"""
for test_name, changed_baselines in changed_test_baselines.iteritems():
directory = self.find_owned_directory(test_name)
if not directory:
_log.warning('Cannot find OWNERS of %s', test_name)
continue
for baseline in changed_baselines:
if self.more_failures_in_baseline(baseline):
self.new_failures_by_directory[directory].append(
TestFailure(
TestFailure.BASELINE_CHANGE,
test_name,
baseline_path=baseline,
gerrit_url_with_ps=gerrit_url_with_ps))
def more_failures_in_baseline(self, baseline):
diff = self.git.run(['diff', '-U0', 'origin/master', '--', baseline])
delta_failures = 0
for line in diff.splitlines():
if line.startswith('+FAIL'):
delta_failures += 1
if line.startswith('-FAIL'):
delta_failures -= 1
return delta_failures > 0
def examine_new_test_expectations(self, test_expectations):
"""Examines new test expectations to find new failures.
Args:
test_expectations: A dictionary mapping names of tests that cannot
be rebaselined to a list of new test expectation lines.
"""
for test_name, expectation_lines in test_expectations.iteritems():
directory = self.find_owned_directory(test_name)
if not directory:
_log.warning('Cannot find OWNERS of %s', test_name)
continue
for expectation_line in expectation_lines:
self.new_failures_by_directory[directory].append(
TestFailure(
TestFailure.NEW_EXPECTATION,
test_name,
expectation_line=expectation_line))
def create_bugs_from_new_failures(self, wpt_revision_start,
wpt_revision_end, gerrit_url):
"""Files bug reports for new failures.
Args:
wpt_revision_start: The start of the imported WPT revision range
(exclusive), i.e. the last imported revision.
wpt_revision_end: The end of the imported WPT revision range
(inclusive), i.e. the current imported revision.
gerrit_url: Gerrit URL of the CL.
Return:
A list of MonorailIssue objects that should be filed.
"""
imported_commits = self.local_wpt.commits_in_range(
wpt_revision_start, wpt_revision_end)
bugs = []
for directory, failures in self.new_failures_by_directory.iteritems():
summary = '[WPT] New failures introduced in {} by import {}'.format(
directory, gerrit_url)
full_directory = self.host.filesystem.join(
self.finder.web_tests_dir(), directory)
owners_file = self.host.filesystem.join(full_directory, 'OWNERS')
is_wpt_notify_enabled = self.owners_extractor.is_wpt_notify_enabled(
owners_file)
owners = self.owners_extractor.extract_owners(owners_file)
# owners may be empty but not None.
cc = owners
component = self.owners_extractor.extract_component(owners_file)
# component could be None.
components = [component] if component else None
prologue = ('WPT import {} introduced new failures in {}:\n\n'
'List of new failures:\n'.format(
gerrit_url, directory))
failure_list = ''
for failure in failures:
failure_list += str(failure) + '\n'
epilogue = '\nThis import contains upstream changes from {} to {}:\n'.format(
wpt_revision_start, wpt_revision_end)
commit_list = self.format_commit_list(imported_commits,
full_directory)
description = prologue + failure_list + epilogue + commit_list
bug = MonorailIssue.new_chromium_issue(summary, description, cc,
components)
_log.info(unicode(bug))
if is_wpt_notify_enabled:
_log.info(
"WPT-NOTIFY enabled in this directory; adding the bug to the pending list."
)
bugs.append(bug)
else:
_log.info(
"WPT-NOTIFY disabled in this directory; discarding the bug."
)
return bugs
def format_commit_list(self, imported_commits, directory):
"""Formats the list of imported WPT commits.
Imports affecting the given directory will be highlighted.
Args:
imported_commits: A list of (SHA, commit subject) pairs.
directory: An absolute path of a directory in the Chromium repo, for
which the list is formatted.
Returns:
A multi-line string.
"""
path_from_wpt = self.host.filesystem.relpath(
directory, self.finder.path_from_web_tests('external', 'wpt'))
commit_list = ''
for sha, subject in imported_commits:
# subject is a Unicode string and can contain non-ASCII characters.
line = u'{}: {}'.format(subject, GITHUB_COMMIT_PREFIX + sha)
if self.local_wpt.is_commit_affecting_directory(
sha, path_from_wpt):
line += ' [affecting this directory]'
commit_list += line + '\n'
return commit_list
def find_owned_directory(self, test_name):
"""Finds the lowest directory that contains the test and has OWNERS.
Args:
The name of the test (a path relative to web_tests).
Returns:
The path of the found directory relative to web_tests.
"""
# Always use non-virtual test names when looking up OWNERS.
if self.default_port.lookup_virtual_test_base(test_name):
test_name = self.default_port.lookup_virtual_test_base(test_name)
# find_owners_file takes either a relative path from the *root* of the
# repository, or an absolute path.
abs_test_path = self.finder.path_from_web_tests(test_name)
owners_file = self.owners_extractor.find_owners_file(
self.host.filesystem.dirname(abs_test_path))
if not owners_file:
return None
owned_directory = self.host.filesystem.dirname(owners_file)
short_directory = self.host.filesystem.relpath(
owned_directory, self.finder.web_tests_dir())
return short_directory
def file_bugs(self, bugs, dry_run, service_account_key_json=None):
"""Files a list of bugs to Monorail.
Args:
bugs: A list of MonorailIssue objects.
dry_run: A boolean, whether we are in dry run mode.
service_account_key_json: Optional, see docs for main().
"""
# TODO(robertma): Better error handling in this method.
if dry_run:
_log.info(
'[dry_run] Would have filed the %d bugs in the pending list.',
len(bugs))
return
_log.info('Filing %d bugs in the pending list to Monorail', len(bugs))
api = self._get_monorail_api(service_account_key_json)
for index, bug in enumerate(bugs, start=1):
response = api.insert_issue(bug)
_log.info('[%d] Filed bug: %s', index,
MonorailIssue.crbug_link(response['id']))
def _get_monorail_api(self, service_account_key_json):
if service_account_key_json:
return self._monorail_api(
service_account_key_json=service_account_key_json)
token = LuciAuth(self.host).get_access_token()
return self._monorail_api(access_token=token)
def _test_base_path(self):
"""Returns the relative path from the repo root to the web tests."""
finder = PathFinder(self._tool.filesystem)
return self._tool.filesystem.relpath(
finder.web_tests_dir(), finder.path_from_chromium_base()) + '/'
class TestCopier(object):
def __init__(self, host, source_repo_path):
"""Initializes variables to prepare for copying and converting files.
Args:
host: An instance of Host.
source_repo_path: Path to the local checkout of web-platform-tests.
"""
self.host = host
assert self.host.filesystem.exists(source_repo_path)
self.source_repo_path = source_repo_path
self.filesystem = self.host.filesystem
self.path_finder = PathFinder(self.filesystem)
self.web_tests_dir = self.path_finder.web_tests_dir()
self.destination_directory = self.filesystem.normpath(
self.filesystem.join(
self.web_tests_dir, DEST_DIR_NAME,
self.filesystem.basename(self.source_repo_path)))
self.import_in_place = (
self.source_repo_path == self.destination_directory)
self.dir_above_repo = self.filesystem.dirname(self.source_repo_path)
self.import_list = []
# This is just a FYI list of CSS properties that still need to be prefixed,
# which may be output after importing.
self._prefixed_properties = {}
def do_import(self):
_log.info('Importing %s into %s', self.source_repo_path,
self.destination_directory)
self.find_importable_tests()
self.import_tests()
def find_importable_tests(self):
"""Walks through the source directory to find what tests should be imported.
This function sets self.import_list, which contains information about how many
tests are being imported, and their source and destination paths.
"""
paths_to_skip = self.find_paths_to_skip()
for root, dirs, files in self.filesystem.walk(self.source_repo_path):
cur_dir = root.replace(self.dir_above_repo + '/', '') + '/'
_log.debug('Scanning %s...', cur_dir)
dirs_to_skip = ('.git', )
if dirs:
for name in dirs_to_skip:
if name in dirs:
dirs.remove(name)
for path in paths_to_skip:
path_base = path.replace(DEST_DIR_NAME + '/', '')
path_base = path_base.replace(cur_dir, '')
path_full = self.filesystem.join(root, path_base)
if path_base in dirs:
_log.info('Skipping: %s', path_full)
dirs.remove(path_base)
if self.import_in_place:
self.filesystem.rmtree(path_full)
copy_list = []
for filename in files:
path_full = self.filesystem.join(root, filename)
path_base = path_full.replace(self.source_repo_path + '/', '')
path_base = self.destination_directory.replace(
self.web_tests_dir + '/', '') + '/' + path_base
if path_base in paths_to_skip:
if self.import_in_place:
_log.debug('Pruning: %s', path_base)
self.filesystem.remove(path_full)
continue
else:
continue
# FIXME: This block should really be a separate function, but the early-continues make that difficult.
if is_basename_skipped(filename):
_log.debug('Skipping: %s', path_full)
_log.debug(
' Reason: This file may cause Chromium presubmit to fail.'
)
continue
copy_list.append({'src': path_full, 'dest': filename})
if copy_list:
# Only add this directory to the list if there's something to import
self.import_list.append({
'dirname': root,
'copy_list': copy_list
})
def find_paths_to_skip(self):
paths_to_skip = set()
port = self.host.port_factory.get()
w3c_import_expectations_path = self.path_finder.path_from_web_tests(
'W3CImportExpectations')
w3c_import_expectations = self.filesystem.read_text_file(
w3c_import_expectations_path)
parser = TestExpectationParser(port, all_tests=(), is_lint_mode=False)
expectation_lines = parser.parse(w3c_import_expectations_path,
w3c_import_expectations)
for line in expectation_lines:
if 'SKIP' in line.expectations:
if line.specifiers:
_log.warning(
'W3CImportExpectations:%s should not have any specifiers',
line.line_numbers)
continue
paths_to_skip.add(line.name)
return paths_to_skip
def import_tests(self):
"""Reads |self.import_list|, and converts and copies files to their destination."""
for dir_to_copy in self.import_list:
if not dir_to_copy['copy_list']:
continue
orig_path = dir_to_copy['dirname']
relative_dir = self.filesystem.relpath(orig_path,
self.source_repo_path)
dest_dir = self.filesystem.join(self.destination_directory,
relative_dir)
if not self.filesystem.exists(dest_dir):
self.filesystem.maybe_make_directory(dest_dir)
for file_to_copy in dir_to_copy['copy_list']:
self.copy_file(file_to_copy, dest_dir)
_log.info('')
_log.info('Import complete')
_log.info('')
if self._prefixed_properties:
_log.info('Properties needing prefixes (by count):')
for prefixed_property in sorted(
self._prefixed_properties,
key=lambda p: self._prefixed_properties[p]):
_log.info(' %s: %s', prefixed_property,
self._prefixed_properties[prefixed_property])
def copy_file(self, file_to_copy, dest_dir):
"""Converts and copies a file, if it should be copied.
Args:
file_to_copy: A dict in a file copy list constructed by
find_importable_tests, which represents one file to copy, including
the keys:
"src": Absolute path to the source location of the file.
"destination": File name of the destination file.
And possibly also the keys "reference_support_info" or "is_jstest".
dest_dir: Path to the directory where the file should be copied.
"""
source_path = self.filesystem.normpath(file_to_copy['src'])
dest_path = self.filesystem.join(dest_dir, file_to_copy['dest'])
if self.filesystem.isdir(source_path):
_log.error('%s refers to a directory', source_path)
return
if not self.filesystem.exists(source_path):
_log.error('%s not found. Possible error in the test.',
source_path)
return
if not self.filesystem.exists(self.filesystem.dirname(dest_path)):
if not self.import_in_place:
self.filesystem.maybe_make_directory(
self.filesystem.dirname(dest_path))
relpath = self.filesystem.relpath(dest_path, self.web_tests_dir)
# FIXME: Maybe doing a file diff is in order here for existing files?
# In other words, there's no sense in overwriting identical files, but
# there's no harm in copying the identical thing.
_log.debug(' copying %s', relpath)
if not self.import_in_place:
self.filesystem.copyfile(source_path, dest_path)
if self.filesystem.read_binary_file(source_path)[:2] == '#!':
self.filesystem.make_executable(dest_path)