class WPTExpectationsUpdater(object):
def __init__(self, host):
self.host = host
self.port = self.host.port_factory.get()
self.git_cl = GitCL(host)
self.finder = PathFinder(self.host.filesystem)
self.ports_with_no_results = set()
self.ports_with_all_pass = set()
self.patchset = None
def run(self, args=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
'--patchset',
default=None,
help='Patchset number to fetch new baselines from.')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='More verbose logging.')
args = parser.parse_args(args)
log_level = logging.DEBUG if args.verbose else logging.INFO
configure_logging(logging_level=log_level, include_time=True)
self.patchset = args.patchset
self.update_expectations()
return 0
def update_expectations(self):
"""Downloads text new baselines and adds test expectations lines.
Returns:
A pair: A set of tests that are rebaselined, and a dictionary
mapping tests that couldn't be rebaselined to lists of expectation
lines written to TestExpectations.
"""
issue_number = self.get_issue_number()
if issue_number == 'None':
raise ScriptError('No issue on current branch.')
build_to_status = self.get_latest_try_jobs()
_log.debug('Latest try jobs: %r', build_to_status)
if not build_to_status:
raise ScriptError('No try job information was collected.')
# Here we build up a dict of failing test results for all platforms.
test_expectations = {}
for build, job_status in build_to_status.iteritems():
if job_status.result == 'SUCCESS':
self.ports_with_all_pass.add(self.port_name(build))
port_results = self.get_failing_results_dict(build)
test_expectations = self.merge_dicts(test_expectations,
port_results)
# And then we merge results for different platforms that had the same results.
for test_name, platform_result in test_expectations.iteritems():
# platform_result is a dict mapping platforms to results.
test_expectations[test_name] = self.merge_same_valued_keys(
platform_result)
# At this point, test_expectations looks like: {
# 'test-with-failing-result': {
# ('port-name1', 'port-name2'): SimpleTestResult,
# 'port-name3': AnotherSimpleTestResult
# }
# }
rebaselined_tests, test_expectations = self.download_text_baselines(
test_expectations)
test_expectation_lines = self.create_line_dict(test_expectations)
self.write_to_test_expectations(test_expectation_lines)
return rebaselined_tests, test_expectation_lines
def get_issue_number(self):
"""Returns current CL number. Can be replaced in unit tests."""
return self.git_cl.get_issue_number()
def get_latest_try_jobs(self):
"""Returns the latest finished try jobs as Build objects."""
return self.git_cl.latest_try_jobs(builder_names=self._get_try_bots(),
patchset=self.patchset)
def get_failing_results_dict(self, build):
"""Returns a nested dict of failing test results.
Retrieves a full list of web test results from a builder result URL.
Collects the builder name, platform and a list of tests that did not
run as expected.
Args:
build: A Build object.
Returns:
A dictionary with the structure: {
'test-with-failing-result': {
'full-port-name': SimpleTestResult
}
}
If results could be fetched but none are failing,
this will return an empty dictionary.
"""
port_name = self.port_name(build)
if port_name in self.ports_with_all_pass:
# All tests passed, so there should be no failing results.
return {}
test_result_list = [self.host.results_fetcher.fetch_results(build)]
has_webdriver_tests = self.host.builders.has_webdriver_tests_for_builder(
build.builder_name)
if has_webdriver_tests:
master = self.host.builders.master_for_builder(build.builder_name)
test_result_list.append(
self.host.results_fetcher.fetch_webdriver_test_results(
build, master))
test_result_list = filter(None, test_result_list)
if not test_result_list:
_log.warning('No results for build %s', build)
self.ports_with_no_results.add(self.port_name(build))
return {}
failing_test_results = []
for test_result in test_result_list:
failing_test_results += [
result
for result in test_result.didnt_run_as_expected_results()
if not result.did_pass()
]
return self.generate_results_dict(self.port_name(build),
failing_test_results)
@memoized
def port_name(self, build):
return self.host.builders.port_name_for_builder_name(
build.builder_name)
def generate_results_dict(self, full_port_name, web_test_results):
"""Makes a dict with results for one platform.
Args:
full_port_name: The fully-qualified port name, e.g. "win-win10".
web_test_results: A list of WebTestResult objects.
Returns:
A dictionary with the structure: {
'test-name': {
'full-port-name': SimpleTestResult
}
}
"""
test_dict = {}
for result in web_test_results:
test_name = result.test_name()
if not self.port.is_wpt_test(test_name):
continue
test_dict[test_name] = {
full_port_name:
SimpleTestResult(expected=result.expected_results(),
actual=result.actual_results(),
bug=UMBRELLA_BUG)
}
return test_dict
def merge_dicts(self, target, source, path=None):
"""Recursively merges nested dictionaries.
Args:
target: First dictionary, which is updated based on source.
source: Second dictionary, not modified.
path: A list of keys, only used for making error messages.
Returns:
The updated target dictionary.
"""
path = path or []
for key in source:
if key in target:
if (isinstance(target[key], dict)) and isinstance(
source[key], dict):
self.merge_dicts(target[key], source[key],
path + [str(key)])
elif target[key] == source[key]:
pass
else:
raise ValueError(
'The key: %s already exist in the target dictionary.' %
'.'.join(path))
else:
target[key] = source[key]
return target
def merge_same_valued_keys(self, dictionary):
"""Merges keys in dictionary with same value.
Traverses through a dict and compares the values of keys to one another.
If the values match, the keys are combined to a tuple and the previous
keys are removed from the dict.
Args:
dictionary: A dictionary with a dictionary as the value.
Returns:
A new dictionary with updated keys to reflect matching values of keys.
Example: {
'one': {'foo': 'bar'},
'two': {'foo': 'bar'},
'three': {'foo': 'bar'}
}
is converted to a new dictionary with that contains
{('one', 'two', 'three'): {'foo': 'bar'}}
"""
merged_dict = {}
matching_value_keys = set()
keys = sorted(dictionary.keys())
while keys:
current_key = keys[0]
found_match = False
if current_key == keys[-1]:
merged_dict[current_key] = dictionary[current_key]
keys.remove(current_key)
break
for next_item in keys[1:]:
if dictionary[current_key] == dictionary[next_item]:
found_match = True
matching_value_keys.update([current_key, next_item])
if next_item == keys[-1]:
if found_match:
merged_dict[tuple(
matching_value_keys)] = dictionary[current_key]
keys = [
k for k in keys if k not in matching_value_keys
]
else:
merged_dict[current_key] = dictionary[current_key]
keys.remove(current_key)
matching_value_keys = set()
return merged_dict
def get_expectations(self, result, test_name=''):
"""Returns a set of test expectations based on the result of a test.
Returns a set of one or more test expectations based on the expected
and actual results of a given test name. This function is to decide
expectations for tests that could not be rebaselined.
Args:
result: A SimpleTestResult.
test_name: The test name string (optional).
Returns:
A set of one or more test expectation strings with the first letter
capitalized. Example: {'Failure', 'Timeout'}.
"""
actual_results = set(result.actual.split())
# If the result is MISSING, this implies that the test was not
# rebaselined and has an actual result but no baseline. We can't
# add a Missing expectation (this is not allowed), but no other
# expectation is correct.
# We also want to skip any new manual tests that are not automated;
# see crbug.com/708241 for context.
if 'MISSING' in actual_results:
return {'Skip'}
if '-manual.' in test_name and 'TIMEOUT' in actual_results:
return {'Skip'}
expectations = set()
failure_types = {'TEXT', 'IMAGE+TEXT', 'IMAGE', 'AUDIO', 'FAIL'}
other_types = {'TIMEOUT', 'CRASH', 'PASS'}
for actual in actual_results:
if actual in failure_types:
expectations.add('Failure')
if actual in other_types:
expectations.add(actual.capitalize())
return expectations
def create_line_dict(self, merged_results):
"""Creates list of test expectations lines.
Traverses through the given |merged_results| dictionary and parses the
value to create one test expectations line per key.
Test expectation lines have the following format:
['BUG_URL [PLATFORM(S)] TEST_NAME [EXPECTATION(S)]']
Args:
merged_results: A dictionary with the format:
{
'test-with-failing-result': {
('port-name1', 'port-name2'): SimpleTestResult,
'port-name3': SimpleTestResult
}
}
Returns:
A dictionary from test names to a list of test expectation lines
(each SimpleTestResult turns into a line).
"""
line_dict = defaultdict(list)
for test_name, port_results in sorted(merged_results.iteritems()):
if not self.port.is_wpt_test(test_name):
_log.warning(
'Non-WPT test "%s" unexpectedly passed to create_line_dict.',
test_name)
continue
for port_names, result in sorted(port_results.iteritems()):
line_dict[test_name].extend(
self._create_lines(test_name, port_names, result))
return line_dict
def _create_lines(self, test_name, port_names, result):
"""Constructs test expectation line strings.
Args:
test_name: The test name string.
port_names: A list of full port names that the line should apply to.
result: A SimpleTestResult.
Returns:
A list of strings which each is a line of test expectation for given
|test_name|.
"""
lines = []
port_names = self.tuple_or_value_to_list(port_names)
# The set of ports with no results is assumed to have have no
# overlap with the set of port names passed in here.
assert (set(port_names) & self.ports_with_no_results) == set()
# The ports with no results are generally ports of builders that
# failed, maybe for unrelated reasons. At this point, we add ports
# with no results to the list of platforms because we're guessing
# that this new expectation might be cross-platform and should
# also apply to any ports that we weren't able to get results for.
port_names.extend(self.ports_with_no_results)
expectations = '[ %s ]' % ' '.join(
self.get_expectations(result, test_name))
for specifier in self.normalized_specifiers(test_name, port_names):
line_parts = []
if specifier:
line_parts.append('[ %s ]' % specifier)
# Escape literal asterisks for typ (https://crbug.com/1036130).
line_parts.append(test_name.replace('*', '\\*'))
line_parts.append(expectations)
# Only add the bug link if the expectations do not include WontFix.
if 'WontFix' not in expectations:
line_parts.insert(0, result.bug)
lines.append(' '.join(line_parts))
return lines
def normalized_specifiers(self, test_name, port_names):
"""Converts and simplifies ports into platform specifiers.
Args:
test_name: The test name string.
port_names: A list of full port names that the line should apply to.
Returns:
A list of specifier string, e.g. ["Mac", "Win"].
[''] will be returned if the line should apply to all platforms.
"""
specifiers = []
for name in sorted(port_names):
specifiers.append(
self.host.builders.version_specifier_for_port_name(name))
if self.specifiers_can_extend_to_all_platforms(specifiers, test_name):
return ['']
specifiers = self.simplify_specifiers(
specifiers, self.port.configuration_specifier_macros())
if not specifiers:
return ['']
return specifiers
@staticmethod
def tuple_or_value_to_list(tuple_or_value):
"""Converts a tuple to a list, and a string value to a one-item list."""
if isinstance(tuple_or_value, tuple):
return list(tuple_or_value)
return [tuple_or_value]
def specifiers_can_extend_to_all_platforms(self, specifiers, test_name):
"""Tests whether a list of specifiers can be extended to all platforms.
Tries to add skipped platform specifiers to the list and tests if the
extended list covers all platforms.
"""
extended_specifiers = specifiers + self.skipped_specifiers(test_name)
# If the list is simplified to empty, then all platforms are covered.
return not self.simplify_specifiers(
extended_specifiers, self.port.configuration_specifier_macros())
def skipped_specifiers(self, test_name):
"""Returns a list of platform specifiers for which the test is skipped."""
specifiers = []
for port in self.all_try_builder_ports():
if port.skips_test(test_name):
specifiers.append(
self.host.builders.version_specifier_for_port_name(
port.name()))
return specifiers
@memoized
def all_try_builder_ports(self):
"""Returns a list of Port objects for all try builders."""
return [
self.host.port_factory.get_from_builder_name(name)
for name in self._get_try_bots()
]
def simplify_specifiers(self, specifiers, specifier_macros):
"""Simplifies the specifier part of an expectation line if possible.
"Simplifying" means finding the shortest list of platform specifiers
that is equivalent to the given list of specifiers. This can be done
because there are "macro specifiers" that stand in for multiple version
specifiers, and an empty list stands in for "all platforms".
Args:
specifiers: A collection of specifiers (case insensitive).
specifier_macros: A dict mapping "macros" for groups of specifiers
to lists of version specifiers. e.g. {"win": ["win7", "win10"]}.
If there are versions in this dict for that have no corresponding
try bots, they are ignored.
Returns:
A shortened list of specifiers (capitalized). For example, ["win7",
"win10"] would be converted to ["Win"]. If the given list covers
all supported platforms, then an empty list is returned.
"""
specifiers = {s.lower() for s in specifiers}
covered_by_try_bots = self._platform_specifiers_covered_by_try_bots()
for macro, versions in specifier_macros.iteritems():
macro = macro.lower()
# Only consider version specifiers that have corresponding try bots.
versions = {
s.lower()
for s in versions if s.lower() in covered_by_try_bots
}
if len(versions) == 0:
continue
if versions <= specifiers:
specifiers -= versions
specifiers.add(macro)
if specifiers == {macro.lower() for macro in specifier_macros}:
return []
return sorted(specifier.capitalize() for specifier in specifiers)
def _platform_specifiers_covered_by_try_bots(self):
all_platform_specifiers = set()
for builder_name in self._get_try_bots():
all_platform_specifiers.add(
self.host.builders.platform_specifier_for_builder(
builder_name).lower())
return frozenset(all_platform_specifiers)
def write_to_test_expectations(self, line_dict):
"""Writes the given lines to the TestExpectations file.
The place in the file where the new lines are inserted is after a marker
comment line. If this marker comment line is not found, then everything
including the marker line is appended to the end of the file.
All WontFix tests are inserted to NeverFixTests file instead of TextExpectations
file.
Args:
line_dict: A dictionary from test names to a list of test expectation lines.
"""
if not line_dict:
_log.info(
'No lines to write to TestExpectations, WebdriverExpectations or NeverFixTests.'
)
return
line_list = []
wont_fix_list = []
webdriver_list = []
for lines in line_dict.itervalues():
for line in lines:
if 'Skip' in line and '-manual.' in line:
wont_fix_list.append(line)
elif self.finder.webdriver_prefix() in line:
webdriver_list.append(line)
else:
line_list.append(line)
list_to_expectation = {
self.port.path_to_generic_test_expectations_file(): line_list,
self.port.path_to_webdriver_expectations_file(): webdriver_list
}
for expectations_file_path, lines in list_to_expectation.iteritems():
if not lines:
continue
_log.info('Lines to write to %s:\n %s', expectations_file_path,
'\n'.join(lines))
# Writes to TestExpectations file.
file_contents = self.host.filesystem.read_text_file(
expectations_file_path)
marker_comment_index = file_contents.find(MARKER_COMMENT)
if marker_comment_index == -1:
file_contents += '\n%s\n' % MARKER_COMMENT
file_contents += '\n'.join(lines)
else:
end_of_marker_line = (file_contents[marker_comment_index:].
find('\n')) + marker_comment_index
file_contents = file_contents[:end_of_marker_line +
1] + '\n'.join(
lines) + file_contents[
end_of_marker_line:]
self.host.filesystem.write_text_file(expectations_file_path,
file_contents)
if wont_fix_list:
_log.info('Lines to write to NeverFixTests:\n %s',
'\n'.join(wont_fix_list))
# Writes to NeverFixTests file.
wont_fix_path = self.port.path_to_never_fix_tests_file()
wont_fix_file_content = self.host.filesystem.read_text_file(
wont_fix_path)
if not wont_fix_file_content.endswith('\n'):
wont_fix_file_content += '\n'
wont_fix_file_content += '\n'.join(wont_fix_list)
wont_fix_file_content += '\n'
self.host.filesystem.write_text_file(wont_fix_path,
wont_fix_file_content)
# TODO(robertma): Unit test this method.
def download_text_baselines(self, test_results):
"""Fetches new baseline files for tests that should be rebaselined.
Invokes `blink_tool.py rebaseline-cl` in order to download new baselines
(-expected.txt files) for testharness.js tests that did not crash or
time out. Then, the platform-specific test is removed from the overall
failure test dictionary and the resulting dictionary is returned.
Args:
test_results: A dictionary of failing test results, mapping test
names to lists of platforms to SimpleTestResult.
Returns:
A pair: A set of tests that are rebaselined, and a modified copy of
the test_results dictionary containing only tests that couldn't be
rebaselined.
"""
tests_to_rebaseline, test_results = self.get_tests_to_rebaseline(
test_results)
if not tests_to_rebaseline:
_log.info('No tests to rebaseline.')
return tests_to_rebaseline, test_results
_log.info('Tests to rebaseline:')
for test in tests_to_rebaseline:
_log.info(' %s', test)
blink_tool = self.finder.path_from_blink_tools('blink_tool.py')
command = [
'python',
blink_tool,
'rebaseline-cl',
'--verbose',
'--no-trigger-jobs',
'--fill-missing',
]
if self.patchset:
command.append('--patchset=' + str(self.patchset))
command += tests_to_rebaseline
self.host.executive.run_command(command)
return tests_to_rebaseline, test_results
def get_tests_to_rebaseline(self, test_results):
"""Filters failing tests that can be rebaselined.
Creates a list of tests to rebaseline depending on the tests' platform-
specific results. In general, this will be non-ref tests that failed
due to a baseline mismatch (rather than crash or timeout).
Args:
test_results: A dictionary of failing test results, mapping test
names to lists of platforms to SimpleTestResult.
Returns:
A pair: A set of tests to be rebaselined, and a modified copy of
the test_results dictionary. The tests to be rebaselined should
include testharness.js tests that failed due to a baseline mismatch.
"""
new_test_results = copy.deepcopy(test_results)
tests_to_rebaseline = set()
for test_name in test_results:
for platforms, result in test_results[test_name].iteritems():
if self.can_rebaseline(test_name, result):
del new_test_results[test_name][platforms]
tests_to_rebaseline.add(test_name)
return sorted(tests_to_rebaseline), new_test_results
def can_rebaseline(self, test_name, result):
"""Checks if a test can be rebaselined.
Args:
test_name: The test name string.
result: A SimpleTestResult.
"""
if self.is_reference_test(test_name):
return False
if any(x in result.actual for x in ('CRASH', 'TIMEOUT', 'MISSING')):
return False
if self.is_webdriver_test(test_name):
return False
return True
def is_reference_test(self, test_name):
"""Checks whether a given test is a reference test."""
return bool(self.port.reference_files(test_name))
def is_webdriver_test(self, test_name):
"""Checks whether a given test is a WebDriver test."""
return self.finder.is_webdriver_test_path(test_name)
def _get_try_bots(self):
return self.host.builders.all_try_builder_names()
class TestImporter(object):
def __init__(self, host, wpt_github=None):
self.host = host
self.wpt_github = wpt_github
self.executive = host.executive
self.fs = host.filesystem
self.finder = PathFinder(self.fs)
self.chromium_git = self.host.git(self.finder.chromium_base())
self.dest_path = self.finder.path_from_web_tests('external', 'wpt')
# A common.net.git_cl.GitCL instance.
self.git_cl = None
# Another Git instance with local WPT as CWD, which can only be
# instantiated after the working directory is created.
self.wpt_git = None
# The WPT revision we are importing and the one imported last time.
self.wpt_revision = None
self.last_wpt_revision = None
# A set of rebaselined tests and a dictionary of new test expectations
# mapping failing tests to platforms to
# wpt_expectations_updater.SimpleTestResult.
self.rebaselined_tests = set()
self.new_test_expectations = {}
self.verbose = False
def main(self, argv=None):
# TODO(robertma): Test this method! Split it to make it easier to test
# if necessary.
options = self.parse_args(argv)
self.verbose = options.verbose
log_level = logging.DEBUG if self.verbose else logging.INFO
configure_logging(logging_level=log_level, include_time=True)
if options.verbose:
# Print out the full output when executive.run_command fails.
self.host.executive.error_output_limit = None
if options.auto_update and options.auto_upload:
_log.error(
'--auto-upload and --auto-update cannot be used together.')
return 1
if not self.checkout_is_okay():
return 1
credentials = read_credentials(self.host, options.credentials_json)
gh_user = credentials.get('GH_USER')
gh_token = credentials.get('GH_TOKEN')
if not gh_user or not gh_token:
_log.warning('You have not set your GitHub credentials. This '
'script may fail with a network error when making '
'an API request to GitHub.')
_log.warning('See https://chromium.googlesource.com/chromium/src'
'/+/master/docs/testing/web_platform_tests.md'
'#GitHub-credentials for instructions on how to set '
'your credentials up.')
self.wpt_github = self.wpt_github or WPTGitHub(self.host, gh_user,
gh_token)
self.git_cl = GitCL(
self.host, auth_refresh_token_json=options.auth_refresh_token_json)
_log.debug('Noting the current Chromium revision.')
chromium_revision = self.chromium_git.latest_git_commit()
# Instantiate Git after local_wpt.fetch() to make sure the path exists.
local_wpt = LocalWPT(self.host, gh_token=gh_token)
local_wpt.fetch()
self.wpt_git = self.host.git(local_wpt.path)
if options.revision is not None:
_log.info('Checking out %s', options.revision)
self.wpt_git.run(['checkout', options.revision])
_log.debug('Noting the revision we are importing.')
self.wpt_revision = self.wpt_git.latest_git_commit()
self.last_wpt_revision = self._get_last_imported_wpt_revision()
import_commit = 'wpt@%s' % self.wpt_revision
_log.info('Importing %s to Chromium %s', import_commit,
chromium_revision)
if options.ignore_exportable_commits:
commit_message = self._commit_message(chromium_revision,
import_commit)
else:
commits = self.apply_exportable_commits_locally(local_wpt)
if commits is None:
_log.error('Could not apply some exportable commits cleanly.')
_log.error('Aborting import to prevent clobbering commits.')
return 1
commit_message = self._commit_message(
chromium_revision,
import_commit,
locally_applied_commits=commits)
self._clear_out_dest_path()
_log.info('Copying the tests from the temp repo to the destination.')
test_copier = TestCopier(self.host, local_wpt.path)
test_copier.do_import()
# TODO(robertma): Implement `add --all` in Git (it is different from `commit --all`).
self.chromium_git.run(['add', '--all', self.dest_path])
self._generate_manifest()
# TODO(crbug.com/800570 robertma): Re-enable it once we fix the bug.
# self._delete_orphaned_baselines()
_log.info(
'Updating TestExpectations for any removed or renamed tests.')
self.update_all_test_expectations_files(self._list_deleted_tests(),
self._list_renamed_tests())
if not self.chromium_git.has_working_directory_changes():
_log.info('Done: no changes to import.')
return 0
if self._only_wpt_manifest_changed():
_log.info('Only manifest was updated; skipping the import.')
return 0
self._commit_changes(commit_message)
_log.info('Changes imported and committed.')
if not options.auto_upload and not options.auto_update:
return 0
self._upload_cl()
_log.info('Issue: %s', self.git_cl.run(['issue']).strip())
if not self.update_expectations_for_cl():
return 1
if not options.auto_update:
return 0
if not self.run_commit_queue_for_cl():
return 1
if not self.send_notifications(local_wpt, options.auto_file_bugs,
options.monorail_auth_json):
return 1
return 0
def update_expectations_for_cl(self):
"""Performs the expectation-updating part of an auto-import job.
This includes triggering try jobs and waiting; then, if applicable,
writing new baselines and TestExpectation lines, committing, and
uploading a new patchset.
This assumes that there is CL associated with the current branch.
Returns True if everything is OK to continue, or False on failure.
"""
_log.info('Triggering try jobs for updating expectations.')
self.git_cl.trigger_try_jobs(self.blink_try_bots())
cl_status = self.git_cl.wait_for_try_jobs(
poll_delay_seconds=POLL_DELAY_SECONDS,
timeout_seconds=TIMEOUT_SECONDS)
if not cl_status:
_log.error('No initial try job results, aborting.')
self.git_cl.run(['set-close'])
return False
if cl_status.status == 'closed':
_log.error('The CL was closed, aborting.')
return False
_log.info('All jobs finished.')
try_results = cl_status.try_job_results
if try_results and self.git_cl.some_failed(try_results):
self.fetch_new_expectations_and_baselines()
if self.chromium_git.has_working_directory_changes():
self._generate_manifest()
message = 'Update test expectations and baselines.'
self._commit_changes(message)
self._upload_patchset(message)
return True
def run_commit_queue_for_cl(self):
"""Triggers CQ and either commits or aborts; returns True on success."""
_log.info('Triggering CQ try jobs.')
self.git_cl.run(['try'])
cl_status = self.git_cl.wait_for_try_jobs(
poll_delay_seconds=POLL_DELAY_SECONDS,
timeout_seconds=TIMEOUT_SECONDS,
cq_only=True)
if not cl_status:
self.git_cl.run(['set-close'])
_log.error('Timed out waiting for CQ; aborting.')
return False
if cl_status.status == 'closed':
_log.error('The CL was closed; aborting.')
return False
_log.info('All jobs finished.')
cq_try_results = cl_status.try_job_results
if not cq_try_results:
_log.error('No CQ try results found in try results')
self.git_cl.run(['set-close'])
return False
if not self.git_cl.all_success(cq_try_results):
_log.error('CQ appears to have failed; aborting.')
self.git_cl.run(['set-close'])
return False
_log.info('CQ appears to have passed; trying to commit.')
self.git_cl.run(['upload', '-f', '--send-mail']) # Turn off WIP mode.
self.git_cl.run(['set-commit'])
if self.git_cl.wait_for_closed_status():
_log.info('Update completed.')
return True
_log.error('Cannot submit CL; aborting.')
try:
self.git_cl.run(['set-close'])
except ScriptError as e:
if e.output and 'Conflict: change is merged' in e.output:
_log.error('CL is already merged; treating as success.')
return True
else:
raise e
return False
def blink_try_bots(self):
"""Returns the collection of builders used for updating expectations."""
return self.host.builders.filter_builders(
is_try=True, exclude_specifiers={'android'})
def parse_args(self, argv):
parser = argparse.ArgumentParser()
parser.description = __doc__
parser.add_argument(
'-v',
'--verbose',
action='store_true',
help='log extra details that may be helpful when debugging')
parser.add_argument(
'--ignore-exportable-commits',
action='store_true',
help='do not check for exportable commits that would be clobbered')
parser.add_argument('-r', '--revision', help='target wpt revision')
parser.add_argument(
'--auto-upload',
action='store_true',
help='upload a CL, update expectations, but do NOT trigger CQ')
parser.add_argument(
'--auto-update',
action='store_true',
help='upload a CL, update expectations, and trigger CQ')
parser.add_argument(
'--auto-file-bugs',
action='store_true',
help='file new failures automatically to crbug.com')
parser.add_argument(
'--auth-refresh-token-json',
help='authentication refresh token JSON file used for try jobs, '
'generally not necessary on developer machines')
parser.add_argument('--credentials-json',
help='A JSON file with GitHub credentials, '
'generally not necessary on developer machines')
parser.add_argument(
'--monorail-auth-json',
help='A JSON file containing the private key of a service account '
'to access Monorail (crbug.com), only needed when '
'--auto-file-bugs is used')
return parser.parse_args(argv)
def checkout_is_okay(self):
if self.chromium_git.has_working_directory_changes():
_log.warning('Checkout is dirty; aborting.')
return False
# TODO(robertma): Add a method in Git to query a range of commits.
local_commits = self.chromium_git.run(
['log', '--oneline', 'origin/master..HEAD'])
if local_commits:
_log.warning('Checkout has local commits before import.')
return True
def apply_exportable_commits_locally(self, local_wpt):
"""Applies exportable Chromium changes to the local WPT repo.
The purpose of this is to avoid clobbering changes that were made in
Chromium but not yet merged upstream. By applying these changes to the
local copy of web-platform-tests before copying files over, we make
it so that the resulting change in Chromium doesn't undo the
previous Chromium change.
Args:
A LocalWPT instance for our local copy of WPT.
Returns:
A list of commits applied (could be empty), or None if any
of the patches could not be applied cleanly.
"""
commits = self.exportable_but_not_exported_commits(local_wpt)
for commit in commits:
_log.info('Applying exportable commit locally:')
_log.info(commit.url())
_log.info('Subject: %s', commit.subject().strip())
# Log a note about the corresponding PR.
# This might not be necessary, and could potentially be removed.
pull_request = self.wpt_github.pr_for_chromium_commit(commit)
if pull_request:
_log.info('PR: %spull/%d', WPT_GH_URL, pull_request.number)
else:
_log.warning('No pull request found.')
error = local_wpt.apply_patch(commit.format_patch())
if error:
_log.error('Commit cannot be applied cleanly:')
_log.error(error)
return None
self.wpt_git.commit_locally_with_message('Applying patch %s' %
commit.sha)
return commits
def exportable_but_not_exported_commits(self, local_wpt):
"""Returns a list of commits that would be clobbered by importer.
The list contains all exportable but not exported commits, not filtered
by whether they can apply cleanly.
"""
# The errors returned by exportable_commits_over_last_n_commits are
# irrelevant and ignored here, because it tests patches *individually*
# while the importer tries to reapply these patches *cumulatively*.
commits, _ = exportable_commits_over_last_n_commits(
self.host,
local_wpt,
self.wpt_github,
require_clean=False,
verify_merged_pr=True)
return commits
def _generate_manifest(self):
"""Generates MANIFEST.json for imported tests.
Runs the (newly-updated) manifest command if it's found, and then
stages the generated MANIFEST.json in the git index, ready to commit.
"""
_log.info('Generating MANIFEST.json')
WPTManifest.generate_manifest(self.host, self.dest_path)
manifest_path = self.fs.join(self.dest_path, 'MANIFEST.json')
assert self.fs.exists(manifest_path)
manifest_base_path = self.fs.normpath(
self.fs.join(self.dest_path, '..', BASE_MANIFEST_NAME))
self.copyfile(manifest_path, manifest_base_path)
self.chromium_git.add_list([manifest_base_path])
def _clear_out_dest_path(self):
"""Removes all files that are synced with upstream from Chromium WPT.
Instead of relying on TestCopier to overwrite these files, cleaning up
first ensures if upstream deletes some files, we also delete them.
"""
_log.info('Cleaning out tests from %s.', self.dest_path)
should_remove = lambda fs, dirname, basename: (is_file_exportable(
fs.relpath(fs.join(dirname, basename), self.finder.chromium_base())
))
files_to_delete = self.fs.files_under(self.dest_path,
file_filter=should_remove)
for subpath in files_to_delete:
self.remove(self.finder.path_from_web_tests('external', subpath))
def _commit_changes(self, commit_message):
_log.info('Committing changes.')
self.chromium_git.commit_locally_with_message(commit_message)
def _only_wpt_manifest_changed(self):
changed_files = self.chromium_git.changed_files()
wpt_base_manifest = self.fs.relpath(
self.fs.join(self.dest_path, '..', BASE_MANIFEST_NAME),
self.finder.chromium_base())
return changed_files == [wpt_base_manifest]
def _commit_message(self,
chromium_commit_sha,
import_commit_sha,
locally_applied_commits=None):
message = 'Import {}\n\nUsing wpt-import in Chromium {}.\n'.format(
import_commit_sha, chromium_commit_sha)
if locally_applied_commits:
message += 'With Chromium commits locally applied on WPT:\n'
message += '\n'.join(
str(commit) for commit in locally_applied_commits)
message += '\nNo-Export: true'
return message
def _delete_orphaned_baselines(self):
_log.info('Deleting any orphaned baselines.')
is_baseline_filter = lambda fs, dirname, basename: is_testharness_baseline(
basename)
baselines = self.fs.files_under(self.dest_path,
file_filter=is_baseline_filter)
# Note about possible refactoring:
# - the manifest path could be factored out to a common location, and
# - the logic for reading the manifest could be factored out from here
# and the Port class.
manifest_path = self.finder.path_from_web_tests(
'external', 'wpt', 'MANIFEST.json')
manifest = WPTManifest(self.fs.read_text_file(manifest_path))
wpt_urls = manifest.all_urls()
# Currently baselines for tests with query strings are merged,
# so that the tests foo.html?r=1 and foo.html?r=2 both have the same
# baseline, foo-expected.txt.
# TODO(qyearsley): Remove this when this behavior is fixed.
wpt_urls = [url.split('?')[0] for url in wpt_urls]
wpt_dir = self.finder.path_from_web_tests('external', 'wpt')
for full_path in baselines:
rel_path = self.fs.relpath(full_path, wpt_dir)
if not self._has_corresponding_test(rel_path, wpt_urls):
self.fs.remove(full_path)
def _has_corresponding_test(self, rel_path, wpt_urls):
# TODO(qyearsley): Ensure that this works with platform baselines and
# virtual baselines, and add unit tests.
base = '/' + rel_path.replace('-expected.txt', '')
return any(
(base + ext) in wpt_urls for ext in Port.supported_file_extensions)
def copyfile(self, source, destination):
_log.debug('cp %s %s', source, destination)
self.fs.copyfile(source, destination)
def remove(self, dest):
_log.debug('rm %s', dest)
self.fs.remove(dest)
def _upload_patchset(self, message):
self.git_cl.run(['upload', '-f', '-t', message])
def _upload_cl(self):
_log.info('Uploading change list.')
directory_owners = self.get_directory_owners()
description = self._cl_description(directory_owners)
sheriff_email = self.tbr_reviewer()
temp_file, temp_path = self.fs.open_text_tempfile()
temp_file.write(description)
temp_file.close()
self.git_cl.run([
'upload',
'-f',
'--message-file',
temp_path,
'--tbrs',
sheriff_email,
# Note: we used to CC all the directory owners, but have stopped
# in search of a better notification mechanism. (crbug.com/765334)
'--cc',
'*****@*****.**',
])
self.fs.remove(temp_path)
def get_directory_owners(self):
"""Returns a mapping of email addresses to owners of changed tests."""
_log.info('Gathering directory owners emails to CC.')
changed_files = self.chromium_git.changed_files()
extractor = DirectoryOwnersExtractor(self.fs)
return extractor.list_owners(changed_files)
def _cl_description(self, directory_owners):
"""Returns a CL description string.
Args:
directory_owners: A dict of tuples of owner names to lists of directories.
"""
# TODO(robertma): Add a method in Git for getting the commit body.
description = self.chromium_git.run(['log', '-1', '--format=%B'])
description += (
'Note to sheriffs: This CL imports external tests and adds\n'
'expectations for those tests; if this CL is large and causes\n'
'a few new failures, please fix the failures by adding new\n'
'lines to TestExpectations rather than reverting. See:\n'
'https://chromium.googlesource.com'
'/chromium/src/+/master/docs/testing/web_platform_tests.md\n\n')
if directory_owners:
description += self._format_directory_owners(
directory_owners) + '\n\n'
# Prevent FindIt from auto-reverting import CLs.
description += 'NOAUTOREVERT=true\n'
# Move any No-Export tag to the end of the description.
description = description.replace('No-Export: true', '')
description = description.replace('\n\n\n\n', '\n\n')
description += 'No-Export: true'
return description
@staticmethod
def _format_directory_owners(directory_owners):
message_lines = ['Directory owners for changes in this CL:']
for owner_tuple, directories in sorted(directory_owners.items()):
message_lines.append(', '.join(owner_tuple) + ':')
message_lines.extend(' ' + d for d in directories)
return '\n'.join(message_lines)
def tbr_reviewer(self):
"""Returns the user name or email address to use as the reviewer.
This tries to fetch the current ecosystem infra sheriff, but falls back
in case of error.
Either a user name (which is assumed to have a chromium.org email
address) or a full email address (for other cases) is returned.
"""
username = ''
try:
username = self._fetch_ecosystem_infra_sheriff_username()
except (IOError, KeyError, ValueError) as error:
_log.error('Exception while fetching current sheriff: %s', error)
if username in ['kyleju']:
_log.warning('Cannot TBR by %s: not a committer', username)
username = ''
return username or TBR_FALLBACK
def _fetch_ecosystem_infra_sheriff_username(self):
try:
content = self.host.web.get_binary(ROTATIONS_URL)
except NetworkTimeout:
_log.error('Cannot fetch %s', ROTATIONS_URL)
return ''
data = json.loads(content)
today = datetime.date.fromtimestamp(self.host.time()).isoformat()
index = data['rotations'].index('ecosystem_infra')
calendar = data['calendar']
for entry in calendar:
if entry['date'] == today:
if not entry['participants'][index]:
_log.info('No sheriff today.')
return ''
return entry['participants'][index][0]
_log.error('No entry found for date %s in rotations table.', today)
return ''
def fetch_new_expectations_and_baselines(self):
"""Modifies expectation lines and baselines based on try job results.
Assuming that there are some try job results available, this
adds new expectation lines to TestExpectations and downloads new
baselines based on the try job results.
This is the same as invoking the `wpt-update-expectations` script.
"""
_log.info('Adding test expectations lines to TestExpectations.')
expectation_updater = WPTExpectationsUpdater(self.host)
self.rebaselined_tests, self.new_test_expectations = expectation_updater.update_expectations(
)
def update_all_test_expectations_files(self, deleted_tests, renamed_tests):
"""Updates all test expectations files for tests that have been deleted or renamed.
This is only for deleted or renamed tests in the initial import,
not for tests that have failures in try jobs.
"""
port = self.host.port_factory.get()
for path, file_contents in port.all_expectations_dict().iteritems():
self._update_single_test_expectations_file(port, path,
file_contents,
deleted_tests,
renamed_tests)
def _update_single_test_expectations_file(self, port, path, file_contents,
deleted_tests, renamed_tests):
"""Updates a single test expectations file."""
test_expectations = TestExpectations(
port, expectations_dict={path: file_contents})
new_lines = []
for line in test_expectations.get_updated_lines(path):
# if a test is a glob type expectation or empty line or comment then add it to the updated
# expectations file without modifications
if not line.test or line.is_glob:
new_lines.append(line.to_string())
continue
test_name = line.test
if self.finder.is_webdriver_test_path(test_name):
root_test_file, subtest_suffix = port.split_webdriver_test_name(
test_name)
else:
root_test_file = test_name
if root_test_file in deleted_tests:
continue
if root_test_file in renamed_tests:
if self.finder.is_webdriver_test_path(root_test_file):
renamed_test = renamed_tests[root_test_file]
test_name = port.add_webdriver_subtest_suffix(
renamed_test, subtest_suffix)
else:
test_name = renamed_tests[root_test_file]
line.test = test_name
new_lines.append(line.to_string())
self.host.filesystem.write_text_file(path, '\n'.join(new_lines) + '\n')
def _list_deleted_tests(self):
"""List of web tests that have been deleted."""
# TODO(robertma): Improve Git.changed_files so that we can use it here.
out = self.chromium_git.run([
'diff', 'origin/master', '-M100%', '--diff-filter=D', '--name-only'
])
deleted_tests = []
for path in out.splitlines():
test = self._relative_to_web_test_dir(path)
if test:
deleted_tests.append(test)
return deleted_tests
def _list_renamed_tests(self):
"""Lists tests that have been renamed.
Returns a dict mapping source name to destination name.
"""
out = self.chromium_git.run([
'diff', 'origin/master', '-M100%', '--diff-filter=R',
'--name-status'
])
renamed_tests = {}
for line in out.splitlines():
_, source_path, dest_path = line.split()
source_test = self._relative_to_web_test_dir(source_path)
dest_test = self._relative_to_web_test_dir(dest_path)
if source_test and dest_test:
renamed_tests[source_test] = dest_test
return renamed_tests
def _relative_to_web_test_dir(self, path_relative_to_repo_root):
"""Returns a path that's relative to the web tests directory."""
abs_path = self.finder.path_from_chromium_base(
path_relative_to_repo_root)
if not abs_path.startswith(self.finder.web_tests_dir()):
return None
return self.fs.relpath(abs_path, self.finder.web_tests_dir())
def _get_last_imported_wpt_revision(self):
"""Finds the last imported WPT revision."""
# TODO(robertma): Only match commit subjects.
output = self.chromium_git.most_recent_log_matching(
'^Import wpt@', self.finder.chromium_base())
# No line-start anchor (^) below because of the formatting of output.
result = re.search(r'Import wpt@(\w+)', output)
if result:
return result.group(1)
else:
_log.error('Cannot find last WPT import.')
return None
def send_notifications(self, local_wpt, auto_file_bugs,
monorail_auth_json):
issue = self.git_cl.run(['status', '--field=id']).strip()
patchset = self.git_cl.run(['status', '--field=patch']).strip()
# Construct the notifier here so that any errors won't affect the import.
notifier = ImportNotifier(self.host, self.chromium_git, local_wpt)
notifier.main(self.last_wpt_revision,
self.wpt_revision,
self.rebaselined_tests,
self.new_test_expectations,
issue,
patchset,
dry_run=not auto_file_bugs,
service_account_key_json=monorail_auth_json)
return True
class WPTExpectationsUpdater(object):
MARKER_COMMENT = '# ====== New tests from wpt-importer added here ======'
UMBRELLA_BUG = 'crbug.com/626703'
def __init__(self, host, args=None):
self.host = host
self.port = self.host.port_factory.get()
self.finder = PathFinder(self.host.filesystem)
self.git_cl = GitCL(host)
self.git = self.host.git(self.finder.chromium_base())
self.configs_with_no_results = []
self.configs_with_all_pass = []
self.patchset = None
# Get options from command line arguments.
parser = argparse.ArgumentParser(description=__doc__)
self.add_arguments(parser)
self.options = parser.parse_args(args or [])
# Set up TestExpectations instance which contains all
# expectations files associated with the platform.
expectations_dict = {p: self.host.filesystem.read_text_file(p)
for p in self.expectations_files()}
self._test_expectations = TestExpectations(
self.port, expectations_dict=expectations_dict)
def expectations_files(self):
"""Returns list of expectations files.
Each expectation file in the list will be cleaned of expectations
for tests that were removed and will also have test names renamed
for tests that were renamed. Also the files may have their expectations
updated using builder results.
"""
return (self.port.all_expectations_dict().keys() +
PRODUCTS_TO_EXPECTATION_FILE_PATHS.values())
def run(self):
"""Does required setup before calling update_expectations().
Do not override this function!
"""
log_level = logging.DEBUG if self.options.verbose else logging.INFO
configure_logging(logging_level=log_level, include_time=True)
self.patchset = self.options.patchset
# Remove expectations for deleted tests and rename tests in expectations
# for renamed tests.
self.cleanup_test_expectations_files()
if not self.options.cleanup_test_expectations_only:
# Use try job results to update expectations and baselines
self.update_expectations()
return 0
def add_arguments(self, parser):
parser.add_argument(
'--patchset',
default=None,
help='Patchset number to fetch new baselines from.')
parser.add_argument(
'-v',
'--verbose',
action='store_true',
help='More verbose logging.')
parser.add_argument(
'--clean-up-affected-tests-only',
action='store_true',
help='Only cleanup expectations deleted or renamed in current CL. '
'If flag is not used then a full cleanup of deleted or '
'renamed tests will be done in expectations.')
parser.add_argument(
'--cleanup-test-expectations-only',
action='store_true',
help='Cleanup test expectations files and then exit script.')
# TODO(rmhasan): Move this argument to the
# AndroidWPTExpectationsUpdater add_arguments implementation.
# Also look into using sub parsers to separate android and
# desktop specific arguments.
parser.add_argument(
'--android-product', action='append', default=[],
help='Android products whose baselines will be updated.',
choices=PRODUCTS)
def update_expectations(self):
"""Downloads text new baselines and adds test expectations lines.
Returns:
A pair: A set of tests that are rebaselined, and a dictionary
mapping tests that couldn't be rebaselined to lists of expectation
lines written to TestExpectations.
"""
issue_number = self.get_issue_number()
if issue_number == 'None':
raise ScriptError('No issue on current branch.')
build_to_status = self.get_latest_try_jobs()
_log.debug('Latest try jobs: %r', build_to_status)
if not build_to_status:
raise ScriptError('No try job information was collected.')
# Here we build up a dict of failing test results for all platforms.
test_expectations = {}
for build, job_status in build_to_status.iteritems():
if job_status.result == 'SUCCESS':
self.configs_with_all_pass.extend(
self.get_builder_configs(build))
continue
result_dicts = self.get_failing_results_dicts(build)
for result_dict in result_dicts:
test_expectations = self.merge_dicts(
test_expectations, result_dict)
# At this point, test_expectations looks like: {
# 'test-with-failing-result': {
# config1: SimpleTestResult,
# config2: SimpleTestResult,
# config3: AnotherSimpleTestResult
# }
# }
# And then we merge results for different platforms that had the same results.
for test_name, platform_result in test_expectations.iteritems():
# platform_result is a dict mapping platforms to results.
test_expectations[test_name] = self.merge_same_valued_keys(
platform_result)
# At this point, test_expectations looks like: {
# 'test-with-failing-result': {
# (config1, config2): SimpleTestResult,
# (config3,): AnotherSimpleTestResult
# }
# }
rebaselined_tests, test_expectations = self.download_text_baselines(
test_expectations)
exp_lines_dict = self.write_to_test_expectations(test_expectations)
return rebaselined_tests, exp_lines_dict
def get_issue_number(self):
"""Returns current CL number. Can be replaced in unit tests."""
return self.git_cl.get_issue_number()
def get_latest_try_jobs(self):
"""Returns the latest finished try jobs as Build objects."""
return self.git_cl.latest_try_jobs(
builder_names=self._get_try_bots(), patchset=self.patchset)
def get_failing_results_dicts(self, build):
"""Returns a list of nested dicts of failing test results.
Retrieves a full list of web test results from a builder result URL.
Collects the builder name, platform and a list of tests that did not
run as expected.
Args:
build: A Build object.
Returns:
A list of dictionaries that have the following structure.
{
'test-with-failing-result': {
config: SimpleTestResult
}
}
If results could be fetched but none are failing,
this will return an empty list.
"""
test_results_list = self._get_web_test_results(build)
has_webdriver_tests = self.host.builders.has_webdriver_tests_for_builder(
build.builder_name)
if has_webdriver_tests:
master = self.host.builders.master_for_builder(build.builder_name)
test_results_list.append(
self.host.results_fetcher.fetch_webdriver_test_results(
build, master))
test_results_list = filter(None, test_results_list)
if not test_results_list:
_log.warning('No results for build %s', build)
self.configs_with_no_results.extend(self.get_builder_configs(build))
return []
failing_test_results = []
for results_set in test_results_list:
results_dict = self.generate_failing_results_dict(
build, results_set)
if results_dict:
failing_test_results.append(results_dict)
return failing_test_results
def _get_web_test_results(self, build):
"""Gets web tests results for a builder.
Args:
build: Named tuple containing builder name and number
Returns:
List of web tests results for each web test step
in build.
"""
return [self.host.results_fetcher.fetch_results(build)]
def get_builder_configs(self, build, *_):
return [DesktopConfig(port_name=self.port_name(build))]
@memoized
def port_name(self, build):
return self.host.builders.port_name_for_builder_name(
build.builder_name)
def generate_failing_results_dict(self, build, web_test_results):
"""Makes a dict with results for one platform.
Args:
builder: Builder instance containing builder information..
web_test_results: A list of WebTestResult objects.
Returns:
A dictionary with the structure: {
'test-name': {
('full-port-name',): SimpleTestResult
}
}
"""
test_dict = {}
configs = self.get_builder_configs(build, web_test_results)
_log.debug('Getting failing results dictionary'
' for %s step in latest %s build' % (
web_test_results.step_name(), build.builder_name))
if len(configs) > 1:
raise ScriptError('More than one configs were produced for'
' builder and web tests step combination')
if not configs:
raise ScriptError('No configuration was found for builder and web test'
' step combination ')
config = configs[0]
if config in self.configs_with_all_pass:
return {}
for result in web_test_results.didnt_run_as_expected_results():
# TODO(rmhasan) If a test fails unexpectedly then it runs multiple
# times until, it passes or a retry limit is reached. Even though
# it passed we there are still flaky failures that we are not
# creating test expectations for. Maybe we should add a mode
# which creates expectations for tests that are flaky but still
# pass in a web test step.
if result.did_pass():
continue
test_name = result.test_name()
if not self._is_wpt_test(test_name):
continue
test_dict[test_name] = {
config:
SimpleTestResult(
expected=result.expected_results(),
actual=result.actual_results(),
bug=self.UMBRELLA_BUG)
}
return test_dict
def _is_wpt_test(self, test_name):
"""Check if a web test is a WPT tests.
In blink web tests results, each test name is relative to
the web_tests directory instead of the wpt directory. We
need to use the port.is_wpt_test() function to find out if a test
is from the WPT suite.
Returns: True if a test is in the external/wpt subdirectory of
the web_tests directory."""
return self.port.is_wpt_test(test_name)
def merge_dicts(self, target, source, path=None):
"""Recursively merges nested dictionaries.
Args:
target: First dictionary, which is updated based on source.
source: Second dictionary, not modified.
path: A list of keys, only used for making error messages.
Returns:
The updated target dictionary.
"""
path = path or []
for key in source:
if key in target:
if (isinstance(target[key], dict)) and isinstance(
source[key], dict):
self.merge_dicts(target[key], source[key],
path + [str(key)])
elif target[key] == source[key]:
pass
else:
raise ValueError(
'The key: %s already exist in the target dictionary.' %
'.'.join(path))
else:
target[key] = source[key]
return target
def merge_same_valued_keys(self, dictionary):
"""Merges keys in dictionary with same value.
Traverses through a dict and compares the values of keys to one another.
If the values match, the keys are combined to a tuple and the previous
keys are removed from the dict.
Args:
dictionary: A dictionary with a dictionary as the value.
Returns:
A new dictionary with updated keys to reflect matching values of keys.
Example: {
'one': {'foo': 'bar'},
'two': {'foo': 'bar'},
'three': {'foo': 'bar'}
}
is converted to a new dictionary with that contains
{('one', 'two', 'three'): {'foo': 'bar'}}
"""
merged_dict = {}
matching_value_keys = set()
keys = sorted(dictionary.keys())
while keys:
current_key = keys[0]
found_match = False
if current_key == keys[-1]:
merged_dict[tuple([current_key])] = dictionary[current_key]
keys.remove(current_key)
break
for next_item in keys[1:]:
if dictionary[current_key] == dictionary[next_item]:
found_match = True
matching_value_keys.update([current_key, next_item])
if next_item == keys[-1]:
if found_match:
merged_dict[
tuple(matching_value_keys)] = dictionary[current_key]
keys = [
k for k in keys if k not in matching_value_keys
]
else:
merged_dict[tuple([current_key])] = dictionary[current_key]
keys.remove(current_key)
matching_value_keys = set()
return merged_dict
def get_expectations(self, result, test_name=''):
"""Returns a set of test expectations based on the result of a test.
Returns a set of one or more test expectations based on the expected
and actual results of a given test name. This function is to decide
expectations for tests that could not be rebaselined.
Args:
result: A SimpleTestResult.
test_name: The test name string (optional).
Returns:
A set of one or more test expectation strings with the first letter
capitalized. Example: {'Failure', 'Timeout'}.
"""
actual_results = set(result.actual.split())
# If the result is MISSING, this implies that the test was not
# rebaselined and has an actual result but no baseline. We can't
# add a Missing expectation (this is not allowed), but no other
# expectation is correct.
# We also want to skip any new manual tests that are not automated;
# see crbug.com/708241 for context.
if 'MISSING' in actual_results:
return {'Skip'}
if '-manual.' in test_name and 'TIMEOUT' in actual_results:
return {'Skip'}
expectations = set()
failure_types = {'TEXT', 'IMAGE+TEXT', 'IMAGE', 'AUDIO', 'FAIL'}
other_types = {'TIMEOUT', 'CRASH', 'PASS'}
for actual in actual_results:
if actual in failure_types:
expectations.add('Failure')
if actual in other_types:
expectations.add(actual.capitalize())
return expectations
def create_line_dict(self, merged_results):
"""Creates list of test expectations lines.
Traverses through the given |merged_results| dictionary and parses the
value to create one test expectations line per key.
Test expectation lines have the following format:
['BUG_URL [PLATFORM(S)] TEST_NAME [EXPECTATION(S)]']
Args:
merged_results: A dictionary with the format:
{
'test-with-failing-result': {
(config1, config2): SimpleTestResult,
(config3,): SimpleTestResult
}
}
Returns:
A dictionary from test names to a list of test expectation lines
(each SimpleTestResult turns into a line).
"""
line_dict = defaultdict(list)
for test_name, test_results in sorted(merged_results.iteritems()):
if not self._is_wpt_test(test_name):
_log.warning(
'Non-WPT test "%s" unexpectedly passed to create_line_dict.',
test_name)
continue
for configs, result in sorted(test_results.iteritems()):
line_dict[test_name].extend(
self._create_lines(test_name, configs, result))
return line_dict
def _create_lines(self, test_name, configs, result):
"""Constructs test expectation line strings.
Args:
test_name: The test name string.
configs: A list of full configs that the line should apply to.
result: A SimpleTestResult.
Returns:
A list of strings which each is a line of test expectation for given
|test_name|.
"""
lines = []
# The set of ports with no results is assumed to have have no
# overlap with the set of port names passed in here.
assert set(configs) & set(self.configs_with_no_results) == set()
# The ports with no results are generally ports of builders that
# failed, maybe for unrelated reasons. At this point, we add ports
# with no results to the list of platforms because we're guessing
# that this new expectation might be cross-platform and should
# also apply to any ports that we weren't able to get results for.
configs = tuple(list(configs) + self.configs_with_no_results)
expectations = '[ %s ]' % \
' '.join(self.get_expectations(result, test_name))
for specifier in self.normalized_specifiers(test_name, configs):
line_parts = []
if specifier:
line_parts.append('[ %s ]' % specifier)
# Escape literal asterisks for typ (https://crbug.com/1036130).
line_parts.append(test_name.replace('*', '\\*'))
line_parts.append(expectations)
# Only add the bug link if the expectations do not include WontFix.
if 'WontFix' not in expectations and result.bug:
line_parts.insert(0, result.bug)
lines.append(' '.join(line_parts))
return lines
def normalized_specifiers(self, test_name, configs):
"""Converts and simplifies ports into platform specifiers.
Args:
test_name: The test name string.
configs: A list of full configs that the line should apply to.
Returns:
A list of specifier string, e.g. ["Mac", "Win"].
[''] will be returned if the line should apply to all platforms.
"""
specifiers = []
for config in configs:
specifiers.append(
self.host.builders.version_specifier_for_port_name(
config.port_name))
if self.specifiers_can_extend_to_all_platforms(specifiers, test_name):
return ['']
specifiers = self.simplify_specifiers(
specifiers, self.port.configuration_specifier_macros())
if not specifiers:
return ['']
return specifiers
def specifiers_can_extend_to_all_platforms(self, specifiers, test_name):
"""Tests whether a list of specifiers can be extended to all platforms.
Tries to add skipped platform specifiers to the list and tests if the
extended list covers all platforms.
"""
extended_specifiers = specifiers + self.skipped_specifiers(test_name)
# If the list is simplified to empty, then all platforms are covered.
return not self.simplify_specifiers(
extended_specifiers, self.port.configuration_specifier_macros())
def skipped_specifiers(self, test_name):
"""Returns a list of platform specifiers for which the test is skipped."""
specifiers = []
for port in self.all_try_builder_ports():
if port.skips_test(test_name):
specifiers.append(
self.host.builders.version_specifier_for_port_name(
port.name()))
return specifiers
@memoized
def all_try_builder_ports(self):
"""Returns a list of Port objects for all try builders."""
return [
self.host.port_factory.get_from_builder_name(name)
for name in self._get_try_bots()
]
def simplify_specifiers(self, specifiers, specifier_macros):
"""Simplifies the specifier part of an expectation line if possible.
"Simplifying" means finding the shortest list of platform specifiers
that is equivalent to the given list of specifiers. This can be done
because there are "macro specifiers" that stand in for multiple version
specifiers, and an empty list stands in for "all platforms".
Args:
specifiers: A collection of specifiers (case insensitive).
specifier_macros: A dict mapping "macros" for groups of specifiers
to lists of version specifiers. e.g. {"win": ["win7", "win10"]}.
If there are versions in this dict for that have no corresponding
try bots, they are ignored.
Returns:
A shortened list of specifiers (capitalized). For example, ["win7",
"win10"] would be converted to ["Win"]. If the given list covers
all supported platforms, then an empty list is returned.
"""
specifiers = {s.lower() for s in specifiers}
covered_by_try_bots = self._platform_specifiers_covered_by_try_bots()
for macro, versions in specifier_macros.iteritems():
macro = macro.lower()
# Only consider version specifiers that have corresponding try bots.
versions = {
s.lower()
for s in versions if s.lower() in covered_by_try_bots
}
if len(versions) == 0:
continue
if versions <= specifiers:
specifiers -= versions
specifiers.add(macro)
if specifiers == {macro.lower() for macro in specifier_macros}:
return []
return sorted(specifier.capitalize() for specifier in specifiers)
def _platform_specifiers_covered_by_try_bots(self):
all_platform_specifiers = set()
for builder_name in self._get_try_bots():
all_platform_specifiers.add(
self.host.builders.platform_specifier_for_builder(
builder_name).lower())
return frozenset(all_platform_specifiers)
def write_to_test_expectations(self, test_expectations):
"""Writes the given lines to the TestExpectations file.
The place in the file where the new lines are inserted is after a marker
comment line. If this marker comment line is not found, then everything
including the marker line is appended to the end of the file.
All WontFix tests are inserted to NeverFixTests file instead of TextExpectations
file.
Args:
test_expectations: A dictionary mapping test names to a dictionary
mapping platforms and test results.
Returns:
Dictionary mapping test names to lists of test expectation strings.
"""
line_dict = self.create_line_dict(test_expectations)
if not line_dict:
_log.info(
'No lines to write to TestExpectations,'
' WebdriverExpectations or NeverFixTests.'
)
return {}
line_list = []
wont_fix_list = []
webdriver_list = []
for lines in line_dict.itervalues():
for line in lines:
if 'Skip' in line and '-manual.' in line:
wont_fix_list.append(line)
elif self.finder.webdriver_prefix() in line:
webdriver_list.append(line)
else:
line_list.append(line)
list_to_expectation = {
self.port.path_to_generic_test_expectations_file(): line_list,
self.port.path_to_webdriver_expectations_file(): webdriver_list
}
for expectations_file_path, lines in list_to_expectation.iteritems():
if not lines:
continue
_log.info('Lines to write to %s:\n %s', expectations_file_path,
'\n'.join(lines))
# Writes to TestExpectations file.
file_contents = self.host.filesystem.read_text_file(
expectations_file_path)
marker_comment_index = file_contents.find(self.MARKER_COMMENT)
if marker_comment_index == -1:
file_contents += '\n%s\n' % self.MARKER_COMMENT
file_contents += '\n'.join(lines)
else:
end_of_marker_line = (file_contents[marker_comment_index:].
find('\n')) + marker_comment_index
file_contents = (
file_contents[:end_of_marker_line + 1] + '\n'.join(lines) +
file_contents[end_of_marker_line:])
self.host.filesystem.write_text_file(expectations_file_path,
file_contents)
if wont_fix_list:
_log.info('Lines to write to NeverFixTests:\n %s',
'\n'.join(wont_fix_list))
# Writes to NeverFixTests file.
wont_fix_path = self.port.path_to_never_fix_tests_file()
wont_fix_file_content = self.host.filesystem.read_text_file(
wont_fix_path)
if not wont_fix_file_content.endswith('\n'):
wont_fix_file_content += '\n'
wont_fix_file_content += '\n'.join(wont_fix_list)
wont_fix_file_content += '\n'
self.host.filesystem.write_text_file(wont_fix_path,
wont_fix_file_content)
return line_dict
def cleanup_test_expectations_files(self):
"""Removes deleted tests from expectations files.
Removes expectations for deleted tests or renames test names in
expectation files for tests that were renamed. If the
--clean-up-affected-tests-only command line argument is used then
only tests deleted in the CL will have their expectations removed
through this script. If that command line argument is not used then
expectations for test files that no longer exist will be deleted.
"""
deleted_test_files = self._list_deleted_test_files()
renamed_test_files = self._list_renamed_test_files()
for path in self._test_expectations.expectations_dict:
_log.info(
'Updating %s for any removed or renamed tests.' %
self.host.filesystem.basename(path))
self._clean_single_test_expectations_file(
path, deleted_test_files, renamed_test_files)
self._test_expectations.commit_changes()
def _clean_single_test_expectations_file(
self, path, deleted_files, renamed_files):
"""Cleans up a single test expectations file.
Args:
path: Path of expectations file that is being cleaned up.
deleted_files: List of test file paths relative to the web tests
directory which were deleted.
renamed_files: Dictionary mapping test file paths to their new file
name after renaming.
"""
for line in self._test_expectations.get_updated_lines(path):
# if a test is a glob type expectation or empty line or comment then
# add it to the updated expectations file without modifications
if not line.test or line.is_glob:
continue
root_test_file = self._get_root_file(line.test)
if root_test_file in renamed_files:
self._test_expectations.remove_expectations(path, [line])
new_file_name = renamed_files[root_test_file]
if self.finder.is_webdriver_test_path(root_test_file):
_, subtest_suffix = self.port.split_webdriver_test_name(
line.test)
line.test = self.port.add_webdriver_subtest_suffix(
new_file_name, subtest_suffix)
elif '?' in line.test:
line.test = (
new_file_name + line.test[line.test.find('?'):])
else:
line.test = new_file_name
self._test_expectations.add_expectations(
path, [line], lineno=line.lineno)
elif root_test_file in deleted_files:
self._test_expectations.remove_expectations(
path, [line])
@memoized
def _get_root_file(self, test_name):
"""Strips arguments from a web test name in order to get the file name.
It also removes the arguments for web driver tests. For instances for
the test test1/example.html?Hello this function will return
test1/example.html. For a webdriver test it would include arguments and
would have the following format, {test file}>>{argument}.
Args:
test_name: Test name which may include test arguments.
Returns:
Returns the test file which is the root of a test.
"""
if self.finder.is_webdriver_test_path(test_name):
root_test_file, _ = (
self.port.split_webdriver_test_name(test_name))
elif '?' in test_name:
root_test_file = test_name[:test_name.find('?')]
else:
root_test_file = test_name
return root_test_file
def _list_deleted_test_files(self):
"""Returns a list of web tests that have been deleted.
If --clean-up-affected-tests-only is true then only test files deleted
in the current CL may be removed from expectations. Otherwise, any test
file may be removed from expectations if it has been deleted.
Returns: A list of web test files that have been deleted.
"""
if self.options.clean_up_affected_tests_only:
# TODO(robertma): Improve Git.changed_files so that we can use
# it here.
paths = set(self.git.run(
['diff', 'origin/master', '-M100%', '--diff-filter=D',
'--name-only']).splitlines())
deleted_tests = set()
for path in paths:
test = self._relative_to_web_test_dir(path)
if test:
deleted_tests.add(test)
else:
# Remove expectations for all test which have files that
# were deleted. Paths are already relative to the web_tests
# directory
deleted_tests = self._deleted_test_files_in_expectations()
return deleted_tests
def _list_renamed_test_files(self):
"""Returns a dictionary mapping tests to their new name.
Regardless of the command line arguments used this test will only
return a dictionary for tests affected in the current CL.
Returns a dictionary mapping source name to destination name.
"""
out = self.git.run(
['diff', 'origin/master', '-M100%', '--diff-filter=R',
'--name-status'])
renamed_tests = {}
for line in out.splitlines():
_, source_path, dest_path = line.split()
source_test = self._relative_to_web_test_dir(source_path)
dest_test = self._relative_to_web_test_dir(dest_path)
if source_test and dest_test:
renamed_tests[source_test] = dest_test
return renamed_tests
def _relative_to_web_test_dir(self, path_relative_to_repo_root):
"""Returns a path that's relative to the web tests directory."""
abs_path = self.finder.path_from_chromium_base(
path_relative_to_repo_root)
if not abs_path.startswith(self.finder.web_tests_dir()):
return None
return self.host.filesystem.relpath(
abs_path, self.finder.web_tests_dir())
def _deleted_test_files_in_expectations(self):
"""Returns a list of test files that were deleted.
Returns a list of test file names that are still in the expectations
files but no longer exists in the web tests directory.
"""
deleted_files = set()
existing_files = {
self._get_root_file(p)
for p in self.port.tests()}
for path in self._test_expectations.expectations_dict:
for line in self._test_expectations.get_updated_lines(path):
if not line.test or line.is_glob:
continue
root_test_file = self._get_root_file(line.test)
if root_test_file not in existing_files:
deleted_files.add(root_test_file)
return deleted_files
# TODO(robertma): Unit test this method.
def download_text_baselines(self, test_results):
"""Fetches new baseline files for tests that should be rebaselined.
Invokes `blink_tool.py rebaseline-cl` in order to download new baselines
(-expected.txt files) for testharness.js tests that did not crash or
time out. Then, the platform-specific test is removed from the overall
failure test dictionary and the resulting dictionary is returned.
Args:
test_results: A dictionary of failing test results, mapping test
names to lists of platforms to SimpleTestResult.
Returns:
A pair: A set of tests that are rebaselined, and a modified copy of
the test_results dictionary containing only tests that couldn't be
rebaselined.
"""
tests_to_rebaseline, test_results = self.get_tests_to_rebaseline(
test_results)
if not tests_to_rebaseline:
_log.info('No tests to rebaseline.')
return tests_to_rebaseline, test_results
_log.info('Tests to rebaseline:')
for test in tests_to_rebaseline:
_log.info(' %s', test)
blink_tool = self.finder.path_from_blink_tools('blink_tool.py')
command = [
'python',
blink_tool,
'rebaseline-cl',
'--verbose',
'--no-trigger-jobs',
'--fill-missing',
]
if self.patchset:
command.append('--patchset=' + str(self.patchset))
command += tests_to_rebaseline
self.host.executive.run_command(command)
return tests_to_rebaseline, test_results
def get_tests_to_rebaseline(self, test_results):
"""Filters failing tests that can be rebaselined.
Creates a list of tests to rebaseline depending on the tests' platform-
specific results. In general, this will be non-ref tests that failed
due to a baseline mismatch (rather than crash or timeout).
Args:
test_results: A dictionary of failing test results, mapping test
names to lists of platforms to SimpleTestResult.
Returns:
A pair: A set of tests to be rebaselined, and a modified copy of
the test_results dictionary. The tests to be rebaselined should
include testharness.js tests that failed due to a baseline mismatch.
"""
new_test_results = copy.deepcopy(test_results)
tests_to_rebaseline = set()
for test_name in test_results:
for platforms, result in test_results[test_name].iteritems():
if self.can_rebaseline(test_name, result):
del new_test_results[test_name][platforms]
tests_to_rebaseline.add(test_name)
return sorted(tests_to_rebaseline), new_test_results
def can_rebaseline(self, test_name, result):
"""Checks if a test can be rebaselined.
Args:
test_name: The test name string.
result: A SimpleTestResult.
"""
if self.is_reference_test(test_name):
return False
if any(x in result.actual for x in ('CRASH', 'TIMEOUT', 'MISSING')):
return False
if self.is_webdriver_test(test_name):
return False
return True
def is_reference_test(self, test_name):
"""Checks whether a given test is a reference test."""
return bool(self.port.reference_files(test_name))
def is_webdriver_test(self, test_name):
"""Checks whether a given test is a WebDriver test."""
return self.finder.is_webdriver_test_path(test_name)
@memoized
def _get_try_bots(self):
return self.host.builders.filter_builders(
is_try=True, exclude_specifiers={'android'})
def test_is_webdriver_test_path(self):
finder = PathFinder(MockFileSystem())
path_with_webdriver_prefix = 'external/wpt/webdriver/' + 'foo/bar.py>>test'
self.assertTrue(
finder.is_webdriver_test_path(path_with_webdriver_prefix))