def __init__(self, port_obj, output_dir):
super(WPTServe, self).__init__(port_obj, output_dir)
# These ports must match wpt_tools/wpt.config.json
http_port = 8001
http_alt_port = 8081
http_private_port = 8082
http_public_port = 8083
https_port = 8444
https_alt_port = 8445
https_private_port = 8446
https_public_port = 8447
h2_port = 9000
ws_port = 9001
wss_port = 9444
webtransport_h3_port = 11000
self._name = 'wptserve'
self._log_prefixes = ('wptserve_stderr', )
self._mappings = [{
'port': http_port,
'scheme': 'http'
}, {
'port': http_alt_port,
'scheme': 'http'
}, {
'port': http_private_port,
'scheme': 'http'
}, {
'port': http_public_port,
'scheme': 'http'
}, {
'port': https_port,
'scheme': 'https',
'sslcert': True
}, {
'port': https_alt_port,
'scheme': 'https',
'sslcert': True
}, {
'port': https_private_port,
'scheme': 'https'
}, {
'port': https_public_port,
'scheme': 'https'
}, {
'port': h2_port,
'scheme': 'https',
'sslcert': True
}, {
'port': ws_port,
'scheme': 'ws'
}, {
'port': wss_port,
'scheme': 'wss',
'sslcert': True
}]
# TODO(burnik): We can probably avoid PID files for WPT in the future.
fs = self._filesystem
self._pid_file = fs.join(self._runtime_path, '%s.pid' % self._name)
self._config_file = fs.join(self._runtime_path, 'wpt.config.json')
finder = PathFinder(fs)
path_to_pywebsocket = finder.path_from_chromium_base(
'third_party', 'pywebsocket3', 'src')
self.path_to_wpt_support = finder.path_from_chromium_base(
'third_party', 'wpt_tools')
path_to_wpt_root = fs.join(self.path_to_wpt_support, 'wpt')
path_to_wpt_tests = fs.abspath(
fs.join(self._port_obj.web_tests_dir(), 'external', 'wpt'))
path_to_ws_handlers = fs.join(path_to_wpt_tests, 'websockets',
'handlers')
wpt_script = fs.join(path_to_wpt_root, 'wpt')
start_cmd = [
self._port_obj.python3_command(),
'-u',
wpt_script,
'serve',
'--config',
self._config_file,
'--doc_root',
path_to_wpt_tests,
]
# Some users (e.g. run_webdriver_tests.py) do not need WebSocket
# handlers, so we only add the flag if the directory exists.
if self._port_obj.host.filesystem.exists(path_to_ws_handlers):
start_cmd += ['--ws_doc_root', path_to_ws_handlers]
if six.PY3:
self._mappings.append({
'port': webtransport_h3_port,
'scheme': 'webtransport-h3'
})
start_cmd.append('--webtransport-h3')
# TODO(burnik): We should stop setting the CWD once WPT can be run without it.
self._cwd = path_to_wpt_root
self._env = port_obj.host.environ.copy()
self._env.update({'PYTHONPATH': path_to_pywebsocket})
self._start_cmd = start_cmd
self._error_log_path = self._filesystem.join(output_dir,
'wptserve_stderr.txt')
self._output_log_path = self._filesystem.join(output_dir,
'wptserve_stdout.txt')
expiration_date = datetime.date(2025, 1, 4)
if datetime.date.today() > expiration_date - datetime.timedelta(30):
_log.error(
'Pre-generated keys and certificates are going to be expired at %s.'
' Please re-generate them by following steps in %s/README.chromium.',
expiration_date.strftime('%b %d %Y'), self.path_to_wpt_support)
def __init__(self, port_obj, output_dir):
super(WPTServe, self).__init__(port_obj, output_dir)
# These ports must match wpt_support/wpt.config.json
http_port, http_alt_port, https_port = (8001, 8081, 8444)
ws_port, wss_port = (9001, 9444)
self._name = 'wptserve'
self._log_prefixes = ('access_log', 'error_log')
self._mappings = [{
'port': http_port,
'scheme': 'http'
}, {
'port': http_alt_port,
'scheme': 'http'
}, {
'port': https_port,
'scheme': 'https',
'sslcert': True
}, {
'port': ws_port,
'scheme': 'ws'
}, {
'port': wss_port,
'scheme': 'wss',
'sslcert': True
}]
# TODO(burnik): We can probably avoid PID files for WPT in the future.
fs = self._filesystem
self._pid_file = fs.join(self._runtime_path, '%s.pid' % self._name)
finder = PathFinder(fs)
path_to_pywebsocket = finder.path_from_chromium_base(
'third_party', 'pywebsocket', 'src')
path_to_wpt_support = finder.path_from_blink_tools(
'blinkpy', 'third_party', 'wpt')
path_to_wpt_root = fs.join(path_to_wpt_support, 'wpt')
path_to_wpt_tests = fs.abspath(
fs.join(self._port_obj.web_tests_dir(), 'external', 'wpt'))
path_to_ws_handlers = fs.join(path_to_wpt_tests, 'websockets',
'handlers')
self._config_file = self._prepare_wptserve_config(path_to_wpt_support)
wpt_script = fs.join(path_to_wpt_root, 'wpt')
start_cmd = [
self._port_obj.host.executable, '-u', wpt_script, 'serve',
'--config', self._config_file, '--doc_root', path_to_wpt_tests
]
# TODO(burnik): Merge with default start_cmd once we roll in websockets.
if self._port_obj.host.filesystem.exists(path_to_ws_handlers):
start_cmd += ['--ws_doc_root', path_to_ws_handlers]
# TODO(burnik): We should stop setting the CWD once WPT can be run without it.
self._cwd = path_to_wpt_root
self._env = port_obj.host.environ.copy()
self._env.update({'PYTHONPATH': path_to_pywebsocket})
self._start_cmd = start_cmd
expiration_date = datetime.date(2025, 1, 4)
if datetime.date.today() > expiration_date - datetime.timedelta(30):
_log.error(
'Pre-generated keys and certificates are going to be expired at %s.'
' Please re-generate them by following steps in %s/README.chromium.',
expiration_date.strftime('%b %d %Y'), path_to_wpt_support)
parser.add_argument('--test-path', help='Path to the WPT WebDriver tests')
parser.add_argument(
'-v',
'--verbose',
action='store_true',
help='log extra details that may be helpful when debugging')
options = parser.parse_args()
test_results = []
log_level = logging.DEBUG if options.verbose else logging.INFO
configure_logging(logging_level=log_level, include_time=True)
host = Host()
port = host.port_factory.get()
path_finder = PathFinder(host.filesystem)
# Starts WPT Serve to serve the WPT WebDriver test content.
port.start_wptserve()
# WebDriverExpectations stores skipped and failed WebDriver tests.
expectations = parse_webdriver_expectations(host, port)
skipped_tests = preprocess_skipped_tests(test_results, expectations,
path_finder)
options.chromedriver = util.GetAbsolutePathOfUserPath(options.chromedriver)
if (not os.path.exists(options.chromedriver)
and util.GetPlatformName() == 'win'
and not options.chromedriver.lower().endswith('.exe')):
options.chromedriver = options.chromedriver + '.exe'
def __init__(self, host):
self.filesystem = host.filesystem
self.finder = PathFinder(self.filesystem)
self.executive = host.executive
self.owner_map = None
class TestImporter(object):
def __init__(self, host, wpt_github=None):
self.host = host
self.wpt_github = wpt_github
self.executive = host.executive
self.fs = host.filesystem
self.finder = PathFinder(self.fs)
self.chromium_git = self.host.git(self.finder.chromium_base())
self.dest_path = self.finder.path_from_web_tests('external', 'wpt')
# A common.net.git_cl.GitCL instance.
self.git_cl = None
# Another Git instance with local WPT as CWD, which can only be
# instantiated after the working directory is created.
self.wpt_git = None
# The WPT revision we are importing and the one imported last time.
self.wpt_revision = None
self.last_wpt_revision = None
# A set of rebaselined tests and a dictionary of new test expectations
# mapping failing tests to platforms to
# wpt_expectations_updater.SimpleTestResult.
self.rebaselined_tests = set()
self.new_test_expectations = {}
self.verbose = False
args = [
'--clean-up-affected-tests-only', '--clean-up-test-expectations'
]
self._expectations_updater = WPTExpectationsUpdater(self.host, args)
def main(self, argv=None):
# TODO(robertma): Test this method! Split it to make it easier to test
# if necessary.
options = self.parse_args(argv)
self.verbose = options.verbose
log_level = logging.DEBUG if self.verbose else logging.INFO
configure_logging(logging_level=log_level, include_time=True)
# Having the full output when executive.run_command fails is useful when
# investigating a failed import, as all we have are logs.
self.executive.error_output_limit = None
if options.auto_update and options.auto_upload:
_log.error(
'--auto-upload and --auto-update cannot be used together.')
return 1
if not self.checkout_is_okay():
return 1
credentials = read_credentials(self.host, options.credentials_json)
gh_user = credentials.get('GH_USER')
gh_token = credentials.get('GH_TOKEN')
if not gh_user or not gh_token:
_log.warning('You have not set your GitHub credentials. This '
'script may fail with a network error when making '
'an API request to GitHub.')
_log.warning('See https://chromium.googlesource.com/chromium/src'
'/+/master/docs/testing/web_platform_tests.md'
'#GitHub-credentials for instructions on how to set '
'your credentials up.')
self.wpt_github = self.wpt_github or WPTGitHub(self.host, gh_user,
gh_token)
self.git_cl = GitCL(
self.host, auth_refresh_token_json=options.auth_refresh_token_json)
_log.debug('Noting the current Chromium revision.')
chromium_revision = self.chromium_git.latest_git_commit()
# Instantiate Git after local_wpt.fetch() to make sure the path exists.
local_wpt = LocalWPT(self.host, gh_token=gh_token)
local_wpt.fetch()
self.wpt_git = self.host.git(local_wpt.path)
if options.revision is not None:
_log.info('Checking out %s', options.revision)
self.wpt_git.run(['checkout', options.revision])
_log.debug('Noting the revision we are importing.')
self.wpt_revision = self.wpt_git.latest_git_commit()
self.last_wpt_revision = self._get_last_imported_wpt_revision()
import_commit = 'wpt@%s' % self.wpt_revision
_log.info('Importing %s to Chromium %s', import_commit,
chromium_revision)
if options.ignore_exportable_commits:
commit_message = self._commit_message(chromium_revision,
import_commit)
else:
commits = self.apply_exportable_commits_locally(local_wpt)
if commits is None:
_log.error('Could not apply some exportable commits cleanly.')
_log.error('Aborting import to prevent clobbering commits.')
return 1
commit_message = self._commit_message(
chromium_revision,
import_commit,
locally_applied_commits=commits)
self._clear_out_dest_path()
_log.info('Copying the tests from the temp repo to the destination.')
test_copier = TestCopier(self.host, local_wpt.path)
test_copier.do_import()
# TODO(robertma): Implement `add --all` in Git (it is different from `commit --all`).
self.chromium_git.run(['add', '--all', self.dest_path])
self._generate_manifest()
# TODO(crbug.com/800570 robertma): Re-enable it once we fix the bug.
# self._delete_orphaned_baselines()
# Remove expectations for tests that were deleted and rename tests
# in expectations for renamed tests.
self._expectations_updater.cleanup_test_expectations_files()
if not self.chromium_git.has_working_directory_changes():
_log.info('Done: no changes to import.')
return 0
if self._only_wpt_manifest_changed():
_log.info('Only manifest was updated; skipping the import.')
return 0
self._commit_changes(commit_message)
_log.info('Changes imported and committed.')
if not options.auto_upload and not options.auto_update:
return 0
self._upload_cl()
_log.info('Issue: %s', self.git_cl.run(['issue']).strip())
if not self.update_expectations_for_cl():
return 1
if not options.auto_update:
return 0
if not self.run_commit_queue_for_cl():
return 1
if not self.send_notifications(local_wpt, options.auto_file_bugs,
options.monorail_auth_json):
return 1
return 0
def update_expectations_for_cl(self):
"""Performs the expectation-updating part of an auto-import job.
This includes triggering try jobs and waiting; then, if applicable,
writing new baselines and TestExpectation lines, committing, and
uploading a new patchset.
This assumes that there is CL associated with the current branch.
Returns True if everything is OK to continue, or False on failure.
"""
_log.info('Triggering try jobs for updating expectations.')
self.git_cl.trigger_try_jobs(self.blink_try_bots())
cl_status = self.git_cl.wait_for_try_jobs(
poll_delay_seconds=POLL_DELAY_SECONDS,
timeout_seconds=TIMEOUT_SECONDS)
if not cl_status:
_log.error('No initial try job results, aborting.')
self.git_cl.run(['set-close'])
return False
if cl_status.status == 'closed':
_log.error('The CL was closed, aborting.')
return False
_log.info('All jobs finished.')
try_results = cl_status.try_job_results
if try_results and self.git_cl.some_failed(try_results):
self.fetch_new_expectations_and_baselines()
if self.chromium_git.has_working_directory_changes():
self._generate_manifest()
message = 'Update test expectations and baselines.'
self._commit_changes(message)
self._upload_patchset(message)
return True
def run_commit_queue_for_cl(self):
"""Triggers CQ and either commits or aborts; returns True on success."""
_log.info('Triggering CQ try jobs.')
self.git_cl.run(['try'])
cl_status = self.git_cl.wait_for_try_jobs(
poll_delay_seconds=POLL_DELAY_SECONDS,
timeout_seconds=TIMEOUT_SECONDS,
cq_only=True)
if not cl_status:
self.git_cl.run(['set-close'])
_log.error('Timed out waiting for CQ; aborting.')
return False
if cl_status.status == 'closed':
_log.error('The CL was closed; aborting.')
return False
_log.info('All jobs finished.')
cq_try_results = cl_status.try_job_results
if not cq_try_results:
_log.error('No CQ try results found in try results')
self.git_cl.run(['set-close'])
return False
if not self.git_cl.all_success(cq_try_results):
_log.error('CQ appears to have failed; aborting.')
self.git_cl.run(['set-close'])
return False
_log.info('CQ appears to have passed; trying to commit.')
self.git_cl.run(['upload', '-f', '--send-mail']) # Turn off WIP mode.
self.git_cl.run(['set-commit'])
if self.git_cl.wait_for_closed_status():
_log.info('Update completed.')
return True
_log.error('Cannot submit CL; aborting.')
try:
self.git_cl.run(['set-close'])
except ScriptError as e:
if e.output and 'Conflict: change is merged' in e.output:
_log.error('CL is already merged; treating as success.')
return True
else:
raise e
return False
def blink_try_bots(self):
"""Returns the collection of builders used for updating expectations."""
return self.host.builders.filter_builders(
is_try=True, exclude_specifiers={'android'})
def parse_args(self, argv):
parser = argparse.ArgumentParser()
parser.description = __doc__
parser.add_argument(
'-v',
'--verbose',
action='store_true',
help='log extra details that may be helpful when debugging')
parser.add_argument(
'--ignore-exportable-commits',
action='store_true',
help='do not check for exportable commits that would be clobbered')
parser.add_argument('-r', '--revision', help='target wpt revision')
parser.add_argument(
'--auto-upload',
action='store_true',
help='upload a CL, update expectations, but do NOT trigger CQ')
parser.add_argument(
'--auto-update',
action='store_true',
help='upload a CL, update expectations, and trigger CQ')
parser.add_argument(
'--auto-file-bugs',
action='store_true',
help='file new failures automatically to crbug.com')
parser.add_argument(
'--auth-refresh-token-json',
help='authentication refresh token JSON file used for try jobs, '
'generally not necessary on developer machines')
parser.add_argument('--credentials-json',
help='A JSON file with GitHub credentials, '
'generally not necessary on developer machines')
parser.add_argument(
'--monorail-auth-json',
help='A JSON file containing the private key of a service account '
'to access Monorail (crbug.com), only needed when '
'--auto-file-bugs is used')
return parser.parse_args(argv)
def checkout_is_okay(self):
if self.chromium_git.has_working_directory_changes():
_log.warning('Checkout is dirty; aborting.')
return False
# TODO(robertma): Add a method in Git to query a range of commits.
local_commits = self.chromium_git.run(
['log', '--oneline', 'origin/master..HEAD'])
if local_commits:
_log.warning('Checkout has local commits before import.')
return True
def apply_exportable_commits_locally(self, local_wpt):
"""Applies exportable Chromium changes to the local WPT repo.
The purpose of this is to avoid clobbering changes that were made in
Chromium but not yet merged upstream. By applying these changes to the
local copy of web-platform-tests before copying files over, we make
it so that the resulting change in Chromium doesn't undo the
previous Chromium change.
Args:
A LocalWPT instance for our local copy of WPT.
Returns:
A list of commits applied (could be empty), or None if any
of the patches could not be applied cleanly.
"""
commits = self.exportable_but_not_exported_commits(local_wpt)
for commit in commits:
_log.info('Applying exportable commit locally:')
_log.info(commit.url())
_log.info('Subject: %s', commit.subject().strip())
# Log a note about the corresponding PR.
# This might not be necessary, and could potentially be removed.
pull_request = self.wpt_github.pr_for_chromium_commit(commit)
if pull_request:
_log.info('PR: %spull/%d', WPT_GH_URL, pull_request.number)
else:
_log.warning('No pull request found.')
error = local_wpt.apply_patch(commit.format_patch())
if error:
_log.error('Commit cannot be applied cleanly:')
_log.error(error)
return None
self.wpt_git.commit_locally_with_message('Applying patch %s' %
commit.sha)
return commits
def exportable_but_not_exported_commits(self, local_wpt):
"""Returns a list of commits that would be clobbered by importer.
The list contains all exportable but not exported commits, not filtered
by whether they can apply cleanly.
"""
# The errors returned by exportable_commits_over_last_n_commits are
# irrelevant and ignored here, because it tests patches *individually*
# while the importer tries to reapply these patches *cumulatively*.
commits, _ = exportable_commits_over_last_n_commits(
self.host,
local_wpt,
self.wpt_github,
require_clean=False,
verify_merged_pr=True)
return commits
def _generate_manifest(self):
"""Generates MANIFEST.json for imported tests.
Runs the (newly-updated) manifest command if it's found, and then
stages the generated MANIFEST.json in the git index, ready to commit.
"""
_log.info('Generating MANIFEST.json')
WPTManifest.generate_manifest(self.host, self.dest_path)
manifest_path = self.fs.join(self.dest_path, 'MANIFEST.json')
assert self.fs.exists(manifest_path)
manifest_base_path = self.fs.normpath(
self.fs.join(self.dest_path, '..', BASE_MANIFEST_NAME))
self.copyfile(manifest_path, manifest_base_path)
self.chromium_git.add_list([manifest_base_path])
def _clear_out_dest_path(self):
"""Removes all files that are synced with upstream from Chromium WPT.
Instead of relying on TestCopier to overwrite these files, cleaning up
first ensures if upstream deletes some files, we also delete them.
"""
_log.info('Cleaning out tests from %s.', self.dest_path)
should_remove = lambda fs, dirname, basename: (is_file_exportable(
fs.relpath(fs.join(dirname, basename), self.finder.chromium_base())
))
files_to_delete = self.fs.files_under(self.dest_path,
file_filter=should_remove)
for subpath in files_to_delete:
self.remove(self.finder.path_from_web_tests('external', subpath))
def _commit_changes(self, commit_message):
_log.info('Committing changes.')
self.chromium_git.commit_locally_with_message(commit_message)
def _only_wpt_manifest_changed(self):
changed_files = self.chromium_git.changed_files()
wpt_base_manifest = self.fs.relpath(
self.fs.join(self.dest_path, '..', BASE_MANIFEST_NAME),
self.finder.chromium_base())
return changed_files == [wpt_base_manifest]
def _commit_message(self,
chromium_commit_sha,
import_commit_sha,
locally_applied_commits=None):
message = 'Import {}\n\nUsing wpt-import in Chromium {}.\n'.format(
import_commit_sha, chromium_commit_sha)
if locally_applied_commits:
message += 'With Chromium commits locally applied on WPT:\n'
message += '\n'.join(
str(commit) for commit in locally_applied_commits)
message += '\nNo-Export: true'
return message
def _delete_orphaned_baselines(self):
_log.info('Deleting any orphaned baselines.')
is_baseline_filter = lambda fs, dirname, basename: is_testharness_baseline(
basename)
baselines = self.fs.files_under(self.dest_path,
file_filter=is_baseline_filter)
# Note about possible refactoring:
# - the manifest path could be factored out to a common location, and
# - the logic for reading the manifest could be factored out from here
# and the Port class.
manifest_path = self.finder.path_from_web_tests(
'external', 'wpt', 'MANIFEST.json')
manifest = WPTManifest(self.fs.read_text_file(manifest_path))
wpt_urls = manifest.all_urls()
# Currently baselines for tests with query strings are merged,
# so that the tests foo.html?r=1 and foo.html?r=2 both have the same
# baseline, foo-expected.txt.
# TODO(qyearsley): Remove this when this behavior is fixed.
wpt_urls = [url.split('?')[0] for url in wpt_urls]
wpt_dir = self.finder.path_from_web_tests('external', 'wpt')
for full_path in baselines:
rel_path = self.fs.relpath(full_path, wpt_dir)
if not self._has_corresponding_test(rel_path, wpt_urls):
self.fs.remove(full_path)
def _has_corresponding_test(self, rel_path, wpt_urls):
# TODO(qyearsley): Ensure that this works with platform baselines and
# virtual baselines, and add unit tests.
base = '/' + rel_path.replace('-expected.txt', '')
return any(
(base + ext) in wpt_urls for ext in Port.supported_file_extensions)
def copyfile(self, source, destination):
_log.debug('cp %s %s', source, destination)
self.fs.copyfile(source, destination)
def remove(self, dest):
_log.debug('rm %s', dest)
self.fs.remove(dest)
def _upload_patchset(self, message):
self.git_cl.run(['upload', '-f', '-t', message])
def _upload_cl(self):
_log.info('Uploading change list.')
directory_owners = self.get_directory_owners()
description = self._cl_description(directory_owners)
sheriff_email = self.tbr_reviewer()
temp_file, temp_path = self.fs.open_text_tempfile()
temp_file.write(description)
temp_file.close()
self.git_cl.run([
'upload',
'-f',
'--message-file',
temp_path,
'--tbrs',
sheriff_email,
# Note: we used to CC all the directory owners, but have stopped
# in search of a better notification mechanism. (crbug.com/765334)
'--cc',
'*****@*****.**',
])
self.fs.remove(temp_path)
def get_directory_owners(self):
"""Returns a mapping of email addresses to owners of changed tests."""
_log.info('Gathering directory owners emails to CC.')
changed_files = self.chromium_git.changed_files()
extractor = DirectoryOwnersExtractor(self.host)
return extractor.list_owners(changed_files)
def _cl_description(self, directory_owners):
"""Returns a CL description string.
Args:
directory_owners: A dict of tuples of owner names to lists of directories.
"""
# TODO(robertma): Add a method in Git for getting the commit body.
description = self.chromium_git.run(['log', '-1', '--format=%B'])
description += (
'Note to sheriffs: This CL imports external tests and adds\n'
'expectations for those tests; if this CL is large and causes\n'
'a few new failures, please fix the failures by adding new\n'
'lines to TestExpectations rather than reverting. See:\n'
'https://chromium.googlesource.com'
'/chromium/src/+/master/docs/testing/web_platform_tests.md\n\n')
if directory_owners:
description += self._format_directory_owners(
directory_owners) + '\n\n'
# Prevent FindIt from auto-reverting import CLs.
description += 'NOAUTOREVERT=true\n'
# Move any No-Export tag to the end of the description.
description = description.replace('No-Export: true', '')
description = description.replace('\n\n\n\n', '\n\n')
description += 'No-Export: true'
return description
@staticmethod
def _format_directory_owners(directory_owners):
message_lines = ['Directory owners for changes in this CL:']
for owner_tuple, directories in sorted(directory_owners.items()):
message_lines.append(', '.join(owner_tuple) + ':')
message_lines.extend(' ' + d for d in directories)
return '\n'.join(message_lines)
def tbr_reviewer(self):
"""Returns the email address to use as the reviewer.
This tries to fetch the current ecosystem infra sheriff, but falls back
in case of error.
"""
email = ''
try:
email = self._fetch_ecosystem_infra_sheriff_email()
except (IOError, KeyError, ValueError) as error:
_log.error('Exception while fetching current sheriff: %s', error)
if email in ['*****@*****.**']:
_log.warning('Cannot TBR by %s: not a committer', email)
email = ''
return email or TBR_FALLBACK
def _fetch_ecosystem_infra_sheriff_email(self):
try:
content = self.host.web.get_binary(ROTATIONS_URL)
except NetworkTimeout:
_log.error('Cannot fetch %s', ROTATIONS_URL)
return ''
data = json.loads(content)
if not data.get('emails'):
_log.error(
'No email found for current sheriff. Retrieved content: %s',
content)
return ''
return data['emails'][0]
def fetch_new_expectations_and_baselines(self):
"""Modifies expectation lines and baselines based on try job results.
Assuming that there are some try job results available, this
adds new expectation lines to TestExpectations and downloads new
baselines based on the try job results.
This is the same as invoking the `wpt-update-expectations` script.
"""
_log.info('Adding test expectations lines to TestExpectations.')
self.rebaselined_tests, self.new_test_expectations = (
self._expectations_updater.update_expectations())
def _get_last_imported_wpt_revision(self):
"""Finds the last imported WPT revision."""
# TODO(robertma): Only match commit subjects.
output = self.chromium_git.most_recent_log_matching(
'^Import wpt@', self.finder.chromium_base())
# No line-start anchor (^) below because of the formatting of output.
result = re.search(r'Import wpt@(\w+)', output)
if result:
return result.group(1)
else:
_log.error('Cannot find last WPT import.')
return None
def send_notifications(self, local_wpt, auto_file_bugs,
monorail_auth_json):
issue = self.git_cl.run(['status', '--field=id']).strip()
patchset = self.git_cl.run(['status', '--field=patch']).strip()
# Construct the notifier here so that any errors won't affect the import.
notifier = ImportNotifier(self.host, self.chromium_git, local_wpt)
notifier.main(self.last_wpt_revision,
self.wpt_revision,
self.rebaselined_tests,
self.new_test_expectations,
issue,
patchset,
dry_run=not auto_file_bugs,
service_account_key_json=monorail_auth_json)
return True
class Manager(object):
"""A class for managing running a series of layout tests."""
HTTP_SUBDIR = 'http'
PERF_SUBDIR = 'perf'
WEBSOCKET_SUBDIR = 'websocket'
ARCHIVED_RESULTS_LIMIT = 25
def __init__(self, port, options, printer):
"""Initializes test runner data structures.
Args:
port: An object implementing platform-specific functionality.
options: An options argument which contains command line options.
printer: A Printer object to record updates to.
"""
self._port = port
self._filesystem = port.host.filesystem
self._options = options
self._printer = printer
self._expectations = None
self._http_server_started = False
self._wptserve_started = False
self._websockets_server_started = False
self._results_directory = self._port.results_directory()
self._finder = LayoutTestFinder(self._port, self._options)
self._path_finder = PathFinder(port.host.filesystem)
self._runner = LayoutTestRunner(self._options, self._port,
self._printer, self._results_directory,
self._test_is_slow)
def run(self, args):
"""Runs the tests and return a RunDetails object with the results."""
start_time = time.time()
self._printer.write_update('Collecting tests ...')
running_all_tests = False
if self._options.manifest_update and (not args
or any('external' in path
for path in args)):
self._printer.write_update(
'Generating MANIFEST.json for web-platform-tests ...')
WPTManifest.ensure_manifest(self._port.host)
self._printer.write_update('Completed generating manifest.')
self._printer.write_update('Collecting tests ...')
try:
paths, all_test_names, running_all_tests = self._collect_tests(
args)
except IOError:
# This is raised if --test-list doesn't exist
return test_run_results.RunDetails(
exit_code=exit_codes.NO_TESTS_EXIT_STATUS)
test_names, tests_in_other_chunks = self._finder.split_into_chunks(
all_test_names)
if self._options.order == 'natural':
test_names.sort(key=self._port.test_key)
elif self._options.order == 'random':
test_names.sort()
random.Random(self._options.seed).shuffle(test_names)
self._printer.write_update('Parsing expectations ...')
self._expectations = test_expectations.TestExpectations(
self._port, test_names)
tests_to_run, tests_to_skip = self._prepare_lists(paths, test_names)
self._expectations.remove_tests_from_expectations(
tests_in_other_chunks)
self._printer.print_found(len(all_test_names), len(test_names),
len(tests_to_run), self._options.repeat_each,
self._options.iterations)
# Check to make sure we're not skipping every test.
if not tests_to_run:
msg = 'No tests to run.'
if self._options.zero_tests_executed_ok:
_log.info(msg)
# Keep executing to produce valid (but empty) results.
else:
_log.critical(msg)
code = exit_codes.NO_TESTS_EXIT_STATUS
return test_run_results.RunDetails(exit_code=code)
exit_code = self._set_up_run(tests_to_run)
if exit_code:
return test_run_results.RunDetails(exit_code=exit_code)
if self._options.num_retries is None:
# If --test-list is passed, or if no test narrowing is specified,
# default to 3 retries. Otherwise [e.g. if tests are being passed by
# name], default to 0 retries.
if self._options.test_list or len(paths) < len(test_names):
self._options.num_retries = 3
else:
self._options.num_retries = 0
should_retry_failures = self._options.num_retries > 0
try:
self._start_servers(tests_to_run)
if self._options.watch:
run_results = self._run_test_loop(tests_to_run, tests_to_skip)
else:
run_results = self._run_test_once(tests_to_run, tests_to_skip,
should_retry_failures)
initial_results, all_retry_results = run_results
finally:
self._stop_servers()
self._clean_up_run()
# Some crash logs can take a long time to be written out so look
# for new logs after the test run finishes.
self._printer.write_update('Looking for new crash logs ...')
self._look_for_new_crash_logs(initial_results, start_time)
for retry_attempt_results in all_retry_results:
self._look_for_new_crash_logs(retry_attempt_results, start_time)
self._printer.write_update('Summarizing results ...')
summarized_full_results = test_run_results.summarize_results(
self._port, self._expectations, initial_results, all_retry_results)
summarized_failing_results = test_run_results.summarize_results(
self._port,
self._expectations,
initial_results,
all_retry_results,
only_include_failing=True)
exit_code = summarized_failing_results['num_regressions']
if exit_code > exit_codes.MAX_FAILURES_EXIT_STATUS:
_log.warning('num regressions (%d) exceeds max exit status (%d)',
exit_code, exit_codes.MAX_FAILURES_EXIT_STATUS)
exit_code = exit_codes.MAX_FAILURES_EXIT_STATUS
if not self._options.dry_run:
self._write_json_files(summarized_full_results,
summarized_failing_results, initial_results,
running_all_tests)
self._upload_json_files()
self._copy_results_html_file(self._results_directory,
'results.html')
if initial_results.keyboard_interrupted:
exit_code = exit_codes.INTERRUPTED_EXIT_STATUS
else:
if initial_results.interrupted:
exit_code = exit_codes.EARLY_EXIT_STATUS
if self._options.show_results and (
exit_code or initial_results.total_failures):
self._port.show_results_html_file(
self._filesystem.join(self._results_directory,
'results.html'))
self._printer.print_results(time.time() - start_time,
initial_results)
return test_run_results.RunDetails(exit_code, summarized_full_results,
summarized_failing_results,
initial_results, all_retry_results)
def _run_test_loop(self, tests_to_run, tests_to_skip):
# Don't show results in a new browser window because we're already
# printing the link to diffs in the loop
self._options.show_results = False
while True:
initial_results, all_retry_results = self._run_test_once(
tests_to_run, tests_to_skip, should_retry_failures=False)
for name in initial_results.failures_by_name:
failure = initial_results.failures_by_name[name][0]
if isinstance(failure, test_failures.FailureTextMismatch):
full_test_path = self._filesystem.join(
self._results_directory, name)
filename, _ = self._filesystem.splitext(full_test_path)
pretty_diff_path = 'file://' + filename + '-pretty-diff.html'
self._printer.writeln('Link to pretty diff:')
self._printer.writeln(pretty_diff_path + '\n')
self._printer.writeln('Finished running tests')
user_input = self._port.host.user.prompt(
'Interactive watch mode: (q)uit (r)etry\n').lower()
if user_input == 'q' or user_input == 'quit':
return (initial_results, all_retry_results)
def _run_test_once(self, tests_to_run, tests_to_skip,
should_retry_failures):
num_workers = self._port.num_workers(int(
self._options.child_processes))
initial_results = self._run_tests(tests_to_run, tests_to_skip,
self._options.repeat_each,
self._options.iterations,
num_workers)
# Don't retry failures when interrupted by user or failures limit exception.
should_retry_failures = should_retry_failures and not (
initial_results.interrupted
or initial_results.keyboard_interrupted)
tests_to_retry = self._tests_to_retry(initial_results)
all_retry_results = []
if should_retry_failures and tests_to_retry:
for retry_attempt in xrange(1, self._options.num_retries + 1):
if not tests_to_retry:
break
_log.info('')
_log.info(
'Retrying %s, attempt %d of %d...',
grammar.pluralize('unexpected failure',
len(tests_to_retry)), retry_attempt,
self._options.num_retries)
retry_results = self._run_tests(tests_to_retry,
tests_to_skip=set(),
repeat_each=1,
iterations=1,
num_workers=num_workers,
retry_attempt=retry_attempt)
all_retry_results.append(retry_results)
tests_to_retry = self._tests_to_retry(retry_results)
return (initial_results, all_retry_results)
def _collect_tests(self, args):
return self._finder.find_tests(
args,
test_list=self._options.test_list,
fastest_percentile=self._options.fastest)
def _is_http_test(self, test):
return (test.startswith(self.HTTP_SUBDIR +
self._port.TEST_PATH_SEPARATOR)
or self._is_websocket_test(test)
or self._port.TEST_PATH_SEPARATOR + self.HTTP_SUBDIR +
self._port.TEST_PATH_SEPARATOR in test)
def _is_websocket_test(self, test):
if self._port.should_use_wptserve(test):
return False
return self.WEBSOCKET_SUBDIR + self._port.TEST_PATH_SEPARATOR in test
def _http_tests(self, test_names):
return set(test for test in test_names if self._is_http_test(test))
def _is_perf_test(self, test):
return self.PERF_SUBDIR == test or (
self.PERF_SUBDIR + self._port.TEST_PATH_SEPARATOR) in test
def _prepare_lists(self, paths, test_names):
tests_to_skip = self._finder.skip_tests(paths, test_names,
self._expectations,
self._http_tests(test_names))
tests_to_run = [
test for test in test_names if test not in tests_to_skip
]
return tests_to_run, tests_to_skip
def _test_input_for_file(self, test_file):
return TestInput(
test_file, self._options.slow_time_out_ms
if self._test_is_slow(test_file) else self._options.time_out_ms,
self._test_requires_lock(test_file))
def _test_requires_lock(self, test_file):
"""Returns True if the test needs to be locked when running multiple
instances of this test runner.
Perf tests are locked because heavy load caused by running other
tests in parallel might cause some of them to time out.
"""
return self._is_http_test(test_file) or self._is_perf_test(test_file)
def _test_is_slow(self, test_file):
expectations = self._expectations.model().get_expectations(test_file)
return (test_expectations.SLOW in expectations
or self._port.is_slow_wpt_test(test_file))
def _needs_servers(self, test_names):
return any(
self._test_requires_lock(test_name) for test_name in test_names)
def _rename_results_folder(self):
try:
timestamp = time.strftime(
"%Y-%m-%d-%H-%M-%S",
time.localtime(
self._filesystem.mtime(
self._filesystem.join(self._results_directory,
'results.html'))))
except (IOError, OSError) as error:
# It might be possible that results.html was not generated in previous run, because the test
# run was interrupted even before testing started. In those cases, don't archive the folder.
# Simply override the current folder contents with new results.
import errno
if error.errno in (errno.EEXIST, errno.ENOENT):
self._printer.write_update(
'No results.html file found in previous run, skipping it.')
return None
archived_name = ''.join(
(self._filesystem.basename(self._results_directory), '_',
timestamp))
archived_path = self._filesystem.join(
self._filesystem.dirname(self._results_directory), archived_name)
self._filesystem.move(self._results_directory, archived_path)
def _delete_dirs(self, dir_list):
for dir_path in dir_list:
self._filesystem.rmtree(dir_path)
def _limit_archived_results_count(self):
results_directory_path = self._filesystem.dirname(
self._results_directory)
file_list = self._filesystem.listdir(results_directory_path)
results_directories = []
for name in file_list:
file_path = self._filesystem.join(results_directory_path, name)
if self._filesystem.isdir(
file_path) and self._results_directory in file_path:
results_directories.append(file_path)
results_directories.sort(key=self._filesystem.mtime)
self._printer.write_update('Clobbering excess archived results in %s' %
results_directory_path)
self._delete_dirs(results_directories[:-self.ARCHIVED_RESULTS_LIMIT])
def _set_up_run(self, test_names):
self._printer.write_update('Checking build ...')
if self._options.build:
exit_code = self._port.check_build(self._needs_servers(test_names),
self._printer)
if exit_code:
_log.error('Build check failed')
return exit_code
if self._options.clobber_old_results:
self._clobber_old_results()
elif self._filesystem.exists(self._results_directory):
self._limit_archived_results_count()
# Rename the existing results folder for archiving.
self._rename_results_folder()
# Create the output directory if it doesn't already exist.
self._port.host.filesystem.maybe_make_directory(
self._results_directory)
exit_code = self._port.setup_test_run()
if exit_code:
_log.error('Build setup failed')
return exit_code
# Check that the system dependencies (themes, fonts, ...) are correct.
if not self._options.nocheck_sys_deps:
self._printer.write_update('Checking system dependencies ...')
exit_code = self._port.check_sys_deps()
if exit_code:
return exit_code
return exit_codes.OK_EXIT_STATUS
def _run_tests(self,
tests_to_run,
tests_to_skip,
repeat_each,
iterations,
num_workers,
retry_attempt=0):
test_inputs = []
for _ in xrange(iterations):
for test in tests_to_run:
for _ in xrange(repeat_each):
test_inputs.append(self._test_input_for_file(test))
return self._runner.run_tests(self._expectations, test_inputs,
tests_to_skip, num_workers,
retry_attempt)
def _start_servers(self, tests_to_run):
if any(self._port.is_wpt_test(test) for test in tests_to_run):
self._printer.write_update('Starting WPTServe ...')
self._port.start_wptserve()
self._wptserve_started = True
if self._port.requires_http_server() or any(
self._is_http_test(test) for test in tests_to_run):
self._printer.write_update('Starting HTTP server ...')
self._port.start_http_server(
additional_dirs={},
number_of_drivers=self._options.max_locked_shards)
self._http_server_started = True
if any(self._is_websocket_test(test) for test in tests_to_run):
self._printer.write_update('Starting WebSocket server ...')
self._port.start_websocket_server()
self._websockets_server_started = True
def _stop_servers(self):
if self._wptserve_started:
self._printer.write_update('Stopping WPTServe ...')
self._wptserve_started = False
self._port.stop_wptserve()
if self._http_server_started:
self._printer.write_update('Stopping HTTP server ...')
self._http_server_started = False
self._port.stop_http_server()
if self._websockets_server_started:
self._printer.write_update('Stopping WebSocket server ...')
self._websockets_server_started = False
self._port.stop_websocket_server()
def _clean_up_run(self):
_log.debug('Flushing stdout')
sys.stdout.flush()
_log.debug('Flushing stderr')
sys.stderr.flush()
_log.debug('Cleaning up port')
self._port.clean_up_test_run()
def _look_for_new_crash_logs(self, run_results, start_time):
"""Looks for and writes new crash logs, at the end of the test run.
Since crash logs can take a long time to be written out if the system is
under stress, do a second pass at the end of the test run.
Args:
run_results: The results of the test run.
start_time: Time the tests started at. We're looking for crash
logs after that time.
"""
crashed_processes = []
for test, result in run_results.unexpected_results_by_name.iteritems():
if result.type != test_expectations.CRASH:
continue
for failure in result.failures:
if not isinstance(failure, test_failures.FailureCrash):
continue
if failure.has_log:
continue
crashed_processes.append(
[test, failure.process_name, failure.pid])
sample_files = self._port.look_for_new_samples(crashed_processes,
start_time)
if sample_files:
for test, sample_file in sample_files.iteritems():
writer = TestResultWriter(self._filesystem, self._port,
self._port.results_directory(), test)
writer.copy_sample_file(sample_file)
crash_logs = self._port.look_for_new_crash_logs(
crashed_processes, start_time)
if crash_logs:
for test, (crash_log, crash_site) in crash_logs.iteritems():
writer = TestResultWriter(self._filesystem, self._port,
self._port.results_directory(), test)
writer.write_crash_log(crash_log)
run_results.unexpected_results_by_name[
test].crash_site = crash_site
def _clobber_old_results(self):
dir_above_results_path = self._filesystem.dirname(
self._results_directory)
self._printer.write_update('Clobbering old results in %s.' %
dir_above_results_path)
if not self._filesystem.exists(dir_above_results_path):
return
file_list = self._filesystem.listdir(dir_above_results_path)
results_directories = []
for name in file_list:
file_path = self._filesystem.join(dir_above_results_path, name)
if self._filesystem.isdir(
file_path) and self._results_directory in file_path:
results_directories.append(file_path)
self._delete_dirs(results_directories)
# Port specific clean-up.
self._port.clobber_old_port_specific_results()
def _tests_to_retry(self, run_results):
# TODO(ojan): This should also check that result.type != test_expectations.MISSING
# since retrying missing expectations is silly. But that's a bit tricky since we
# only consider the last retry attempt for the count of unexpected regressions.
return [
result.test_name
for result in run_results.unexpected_results_by_name.values()
if result.type != test_expectations.PASS
]
def _write_json_files(self, summarized_full_results,
summarized_failing_results, initial_results,
running_all_tests):
_log.debug("Writing JSON files in %s.", self._results_directory)
# FIXME: Upload stats.json to the server and delete times_ms.
times_trie = json_results_generator.test_timings_trie(
initial_results.results_by_name.values())
times_json_path = self._filesystem.join(self._results_directory,
'times_ms.json')
json_results_generator.write_json(self._filesystem, times_trie,
times_json_path)
# Save out the times data so we can use it for --fastest in the future.
if running_all_tests:
bot_test_times_path = self._port.bot_test_times_path()
self._filesystem.maybe_make_directory(
self._filesystem.dirname(bot_test_times_path))
json_results_generator.write_json(self._filesystem, times_trie,
bot_test_times_path)
stats_trie = self._stats_trie(initial_results)
stats_path = self._filesystem.join(self._results_directory,
'stats.json')
self._filesystem.write_text_file(stats_path, json.dumps(stats_trie))
full_results_path = self._filesystem.join(self._results_directory,
'full_results.json')
json_results_generator.write_json(self._filesystem,
summarized_full_results,
full_results_path)
full_results_jsonp_path = self._filesystem.join(
self._results_directory, 'full_results_jsonp.js')
json_results_generator.write_json(self._filesystem,
summarized_full_results,
full_results_jsonp_path,
callback='ADD_FULL_RESULTS')
full_results_path = self._filesystem.join(self._results_directory,
'failing_results.json')
# We write failing_results.json out as jsonp because we need to load it
# from a file url for results.html and Chromium doesn't allow that.
json_results_generator.write_json(self._filesystem,
summarized_failing_results,
full_results_path,
callback='ADD_RESULTS')
# Write out the JSON files suitable for other tools to process.
# As the output can be quite large (as there are 60k+ tests) we also
# support only outputting the failing results.
if self._options.json_failing_test_results:
# FIXME(tansell): Make sure this includes an *unexpected* results
# (IE Passing when expected to be failing.)
json_results_generator.write_json(
self._filesystem, summarized_failing_results,
self._options.json_failing_test_results)
if self._options.json_test_results:
json_results_generator.write_json(self._filesystem,
summarized_full_results,
self._options.json_test_results)
_log.debug('Finished writing JSON files.')
def _upload_json_files(self):
if not self._options.test_results_server:
return
if not self._options.master_name:
_log.error(
'--test-results-server was set, but --master-name was not. Not uploading JSON files.'
)
return
_log.debug('Uploading JSON files for builder: %s',
self._options.builder_name)
attrs = [('builder', self._options.builder_name),
('testtype', self._options.step_name),
('master', self._options.master_name)]
files = [
(name, self._filesystem.join(self._results_directory, name))
for name in
['failing_results.json', 'full_results.json', 'times_ms.json']
]
url = 'https://%s/testfile/upload' % self._options.test_results_server
# Set uploading timeout in case appengine server is having problems.
# 120 seconds are more than enough to upload test results.
uploader = FileUploader(url, 120)
try:
response = uploader.upload_as_multipart_form_data(
self._filesystem, files, attrs)
if response:
if response.code == 200:
_log.debug('JSON uploaded.')
else:
_log.debug('JSON upload failed, %d: "%s"', response.code,
response.read())
else:
_log.error('JSON upload failed; no response returned')
except IOError as err:
_log.error('Upload failed: %s', err)
def _copy_results_html_file(self, destination_dir, filename):
"""Copies a file from the template directory to the results directory."""
template_dir = self._path_finder.path_from_layout_tests(
'fast', 'harness')
source_path = self._filesystem.join(template_dir, filename)
destination_path = self._filesystem.join(destination_dir, filename)
# Note that the results.html template file won't exist when
# we're using a MockFileSystem during unit tests, so make sure
# it exists before we try to copy it.
if self._filesystem.exists(source_path):
self._filesystem.copyfile(source_path, destination_path)
def _stats_trie(self, initial_results):
def _worker_number(worker_name):
return int(worker_name.split('/')[1]) if worker_name else -1
stats = {}
for result in initial_results.results_by_name.values():
if result.type != test_expectations.SKIP:
stats[result.test_name] = {
'results':
(_worker_number(result.worker_name), result.test_number,
result.pid, int(result.test_run_time * 1000),
int(result.total_run_time * 1000))
}
stats_trie = {}
for name, value in stats.iteritems():
json_results_generator.add_path_to_trie(name, value, stats_trie)
return stats_trie
def test_path_from_layout_tests(self):
finder = PathFinder(MockFileSystem())
self.assertEqual(
finder.path_from_layout_tests('external', 'wpt'),
'/mock-checkout/' + RELATIVE_WEB_TESTS + 'external/wpt')
def load_default_builder_list(filesystem):
"""Loads the set of builders from a JSON file and returns the BuilderList."""
path = PathFinder(filesystem).path_from_blink_tools(
'blinkpy', 'common', 'config', 'builders.json')
contents = filesystem.read_text_file(path)
return BuilderList(json.loads(contents))
def test_layout_tests_dir(self):
finder = PathFinder(MockFileSystem())
self.assertEqual(finder.layout_tests_dir(),
'/mock-checkout/' + RELATIVE_WEB_TESTS[:-1])
def test_perf_tests_dir(self):
finder = PathFinder(MockFileSystem())
self.assertEqual(finder.perf_tests_dir(),
'/mock-checkout/third_party/blink/perf_tests')
def test_path_from_chromium_base(self):
finder = PathFinder(MockFileSystem())
self.assertEqual(finder.path_from_chromium_base('foo', 'bar.baz'),
'/mock-checkout/foo/bar.baz')
def __init__(self, tests, is_debug):
self.executive = Executive()
self.tests = tests
self.expected_failure = tests[-1]
self.is_debug = is_debug
self.path_finder = PathFinder(FileSystem())
class Bisector(object):
def __init__(self, tests, is_debug):
self.executive = Executive()
self.tests = tests
self.expected_failure = tests[-1]
self.is_debug = is_debug
self.path_finder = PathFinder(FileSystem())
def bisect(self):
if self.test_fails_in_isolation():
self.buckets = [Bucket([self.expected_failure])]
print '%s fails when run in isolation.' % self.expected_failure
self.print_result()
return 0
if not self.test_fails(self.tests):
_log.error('%s does not fail', self.expected_failure)
return 1
# Split the list of test into buckets. Each bucket has at least one test required to cause
# the expected failure at the end. Split buckets in half until there are only buckets left
# with one item in them.
self.buckets = [
Bucket(self.tests[:-1]),
Bucket([self.expected_failure])
]
while not self.is_done():
self.print_progress()
self.split_largest_bucket()
self.print_result()
self.verify_non_flaky()
return 0
def test_fails_in_isolation(self):
return self.test_bucket_list_fails([Bucket([self.expected_failure])])
def verify_non_flaky(self):
print 'Verifying the failure is not flaky by running 10 times.'
count_failures = 0
for _ in range(0, 10):
if self.test_bucket_list_fails(self.buckets):
count_failures += 1
print 'Failed %d/10 times' % count_failures
def print_progress(self):
count = 0
for bucket in self.buckets:
count += len(bucket.tests)
print '%d tests left, %d buckets' % (count, len(self.buckets))
def print_result(self):
tests = []
for bucket in self.buckets:
tests += bucket.tests
extra_args = ' --debug' if self.is_debug else ''
print 'run_web_tests.py%s --jobs=1 --order=none %s' % (extra_args,
' '.join(tests))
def is_done(self):
for bucket in self.buckets:
if bucket.size() > 1:
return False
return True
def split_largest_bucket(self):
index = 0
largest_index = 0
largest_size = 0
for bucket in self.buckets:
if bucket.size() > largest_size:
largest_index = index
largest_size = bucket.size()
index += 1
bucket_to_split = self.buckets[largest_index]
halfway_point = int(largest_size / 2)
first_half = Bucket(bucket_to_split.tests[:halfway_point])
second_half = Bucket(bucket_to_split.tests[halfway_point:])
buckets_before = self.buckets[:largest_index]
buckets_after = self.buckets[largest_index + 1:]
# Do the second half first because it tends to be faster because the http tests are front-loaded and slow.
new_buckets = buckets_before + [second_half] + buckets_after
if self.test_bucket_list_fails(new_buckets):
self.buckets = new_buckets
return
new_buckets = buckets_before + [first_half] + buckets_after
if self.test_bucket_list_fails(new_buckets):
self.buckets = new_buckets
return
self.buckets = buckets_before + [first_half, second_half
] + buckets_after
def test_bucket_list_fails(self, buckets):
tests = []
for bucket in buckets:
tests += bucket.tests
return self.test_fails(tests)
def test_fails(self, tests):
extra_args = ['--debug'] if self.is_debug else []
path_to_run_webkit_tests = self.path_finder.path_from_tools_scripts(
'run_web_tests.py')
output = self.executive.popen([
path_to_run_webkit_tests, '--jobs', '1', '--order', 'none',
'--no-retry', '--no-show-results', '--verbose'
] + extra_args + tests,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
failure_string = self.expected_failure + ' failed'
if failure_string in output.stderr.read():
return True
return False
class WPTExpectationsUpdater(object):
def __init__(self, host):
self.host = host
self.port = self.host.port_factory.get()
self.git_cl = GitCL(host)
self.finder = PathFinder(self.host.filesystem)
self.ports_with_no_results = set()
self.ports_with_all_pass = set()
self.patchset = None
def run(self, args=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
'--patchset',
default=None,
help='Patchset number to fetch new baselines from.')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='More verbose logging.')
args = parser.parse_args(args)
log_level = logging.DEBUG if args.verbose else logging.INFO
configure_logging(logging_level=log_level, include_time=True)
self.patchset = args.patchset
self.update_expectations()
return 0
def update_expectations(self):
"""Downloads text new baselines and adds test expectations lines.
Returns:
A pair: A set of tests that are rebaselined, and a dictionary
mapping tests that couldn't be rebaselined to lists of expectation
lines written to TestExpectations.
"""
issue_number = self.get_issue_number()
if issue_number == 'None':
raise ScriptError('No issue on current branch.')
build_to_status = self.get_latest_try_jobs()
_log.debug('Latest try jobs: %r', build_to_status)
if not build_to_status:
raise ScriptError('No try job information was collected.')
# Here we build up a dict of failing test results for all platforms.
test_expectations = {}
for build, job_status in build_to_status.iteritems():
if job_status.result == 'SUCCESS':
self.ports_with_all_pass.add(self.port_name(build))
port_results = self.get_failing_results_dict(build)
test_expectations = self.merge_dicts(test_expectations,
port_results)
# And then we merge results for different platforms that had the same results.
for test_name, platform_result in test_expectations.iteritems():
# platform_result is a dict mapping platforms to results.
test_expectations[test_name] = self.merge_same_valued_keys(
platform_result)
# At this point, test_expectations looks like: {
# 'test-with-failing-result': {
# ('port-name1', 'port-name2'): SimpleTestResult,
# 'port-name3': AnotherSimpleTestResult
# }
# }
rebaselined_tests, test_expectations = self.download_text_baselines(
test_expectations)
test_expectation_lines = self.create_line_dict(test_expectations)
self.write_to_test_expectations(test_expectation_lines)
return rebaselined_tests, test_expectation_lines
def get_issue_number(self):
"""Returns current CL number. Can be replaced in unit tests."""
return self.git_cl.get_issue_number()
def get_latest_try_jobs(self):
"""Returns the latest finished try jobs as Build objects."""
return self.git_cl.latest_try_jobs(builder_names=self._get_try_bots(),
patchset=self.patchset)
def get_failing_results_dict(self, build):
"""Returns a nested dict of failing test results.
Retrieves a full list of web test results from a builder result URL.
Collects the builder name, platform and a list of tests that did not
run as expected.
Args:
build: A Build object.
Returns:
A dictionary with the structure: {
'test-with-failing-result': {
'full-port-name': SimpleTestResult
}
}
If results could be fetched but none are failing,
this will return an empty dictionary.
"""
port_name = self.port_name(build)
if port_name in self.ports_with_all_pass:
# All tests passed, so there should be no failing results.
return {}
test_result_list = [self.host.buildbot.fetch_results(build)]
has_webdriver_tests = self.host.builders.has_webdriver_tests_for_builder(
build.builder_name)
if has_webdriver_tests:
master = self.host.builders.master_for_builder(build.builder_name)
test_result_list.append(
self.host.buildbot.fetch_webdriver_test_results(build, master))
test_result_list = filter(None, test_result_list)
if not test_result_list:
_log.warning('No results for build %s', build)
self.ports_with_no_results.add(self.port_name(build))
return {}
failing_test_results = []
for test_result in test_result_list:
failing_test_results += [
result
for result in test_result.didnt_run_as_expected_results()
if not result.did_pass()
]
return self.generate_results_dict(self.port_name(build),
failing_test_results)
@memoized
def port_name(self, build):
return self.host.builders.port_name_for_builder_name(
build.builder_name)
def generate_results_dict(self, full_port_name, web_test_results):
"""Makes a dict with results for one platform.
Args:
full_port_name: The fully-qualified port name, e.g. "win-win10".
web_test_results: A list of WebTestResult objects.
Returns:
A dictionary with the structure: {
'test-name': {
'full-port-name': SimpleTestResult
}
}
"""
test_dict = {}
for result in web_test_results:
test_name = result.test_name()
if not self.port.is_wpt_test(test_name):
continue
test_dict[test_name] = {
full_port_name:
SimpleTestResult(expected=result.expected_results(),
actual=result.actual_results(),
bug=UMBRELLA_BUG)
}
return test_dict
def merge_dicts(self, target, source, path=None):
"""Recursively merges nested dictionaries.
Args:
target: First dictionary, which is updated based on source.
source: Second dictionary, not modified.
path: A list of keys, only used for making error messages.
Returns:
The updated target dictionary.
"""
path = path or []
for key in source:
if key in target:
if (isinstance(target[key], dict)) and isinstance(
source[key], dict):
self.merge_dicts(target[key], source[key],
path + [str(key)])
elif target[key] == source[key]:
pass
else:
raise ValueError(
'The key: %s already exist in the target dictionary.' %
'.'.join(path))
else:
target[key] = source[key]
return target
def merge_same_valued_keys(self, dictionary):
"""Merges keys in dictionary with same value.
Traverses through a dict and compares the values of keys to one another.
If the values match, the keys are combined to a tuple and the previous
keys are removed from the dict.
Args:
dictionary: A dictionary with a dictionary as the value.
Returns:
A new dictionary with updated keys to reflect matching values of keys.
Example: {
'one': {'foo': 'bar'},
'two': {'foo': 'bar'},
'three': {'foo': 'bar'}
}
is converted to a new dictionary with that contains
{('one', 'two', 'three'): {'foo': 'bar'}}
"""
merged_dict = {}
matching_value_keys = set()
keys = sorted(dictionary.keys())
while keys:
current_key = keys[0]
found_match = False
if current_key == keys[-1]:
merged_dict[current_key] = dictionary[current_key]
keys.remove(current_key)
break
for next_item in keys[1:]:
if dictionary[current_key] == dictionary[next_item]:
found_match = True
matching_value_keys.update([current_key, next_item])
if next_item == keys[-1]:
if found_match:
merged_dict[tuple(
matching_value_keys)] = dictionary[current_key]
keys = [
k for k in keys if k not in matching_value_keys
]
else:
merged_dict[current_key] = dictionary[current_key]
keys.remove(current_key)
matching_value_keys = set()
return merged_dict
def get_expectations(self, result, test_name=''):
"""Returns a set of test expectations based on the result of a test.
Returns a set of one or more test expectations based on the expected
and actual results of a given test name. This function is to decide
expectations for tests that could not be rebaselined.
Args:
result: A SimpleTestResult.
test_name: The test name string (optional).
Returns:
A set of one or more test expectation strings with the first letter
capitalized. Example: {'Failure', 'Timeout'}.
"""
actual_results = set(result.actual.split())
# If the result is MISSING, this implies that the test was not
# rebaselined and has an actual result but no baseline. We can't
# add a Missing expectation (this is not allowed), but no other
# expectation is correct.
# We also want to skip any new manual tests that are not automated;
# see crbug.com/708241 for context.
if 'MISSING' in actual_results:
return {'Skip'}
if '-manual.' in test_name and 'TIMEOUT' in actual_results:
return {'WontFix'}
expectations = set()
failure_types = {'TEXT', 'IMAGE+TEXT', 'IMAGE', 'AUDIO', 'FAIL'}
other_types = {'TIMEOUT', 'CRASH', 'PASS'}
for actual in actual_results:
if actual in failure_types:
expectations.add('Failure')
if actual in other_types:
expectations.add(actual.capitalize())
return expectations
def create_line_dict(self, merged_results):
"""Creates list of test expectations lines.
Traverses through the given |merged_results| dictionary and parses the
value to create one test expectations line per key.
Test expectation lines have the following format:
['BUG_URL [PLATFORM(S)] TEST_NAME [EXPECTATION(S)]']
Args:
merged_results: A dictionary with the format:
{
'test-with-failing-result': {
('port-name1', 'port-name2'): SimpleTestResult,
'port-name3': SimpleTestResult
}
}
Returns:
A dictionary from test names to a list of test expectation lines
(each SimpleTestResult turns into a line).
"""
line_dict = defaultdict(list)
for test_name, port_results in sorted(merged_results.iteritems()):
if not self.port.is_wpt_test(test_name):
_log.warning(
'Non-WPT test "%s" unexpectedly passed to create_line_dict.',
test_name)
continue
for port_names, result in sorted(port_results.iteritems()):
line_dict[test_name].extend(
self._create_lines(test_name, port_names, result))
return line_dict
def _create_lines(self, test_name, port_names, result):
"""Constructs test expectation line strings.
Args:
test_name: The test name string.
port_names: A list of full port names that the line should apply to.
result: A SimpleTestResult.
Returns:
A list of strings which each is a line of test expectation for given
|test_name|.
"""
lines = []
port_names = self.tuple_or_value_to_list(port_names)
# The set of ports with no results is assumed to have have no
# overlap with the set of port names passed in here.
assert (set(port_names) & self.ports_with_no_results) == set()
# The ports with no results are generally ports of builders that
# failed, maybe for unrelated reasons. At this point, we add ports
# with no results to the list of platforms because we're guessing
# that this new expectation might be cross-platform and should
# also apply to any ports that we weren't able to get results for.
port_names.extend(self.ports_with_no_results)
expectations = '[ %s ]' % ' '.join(
self.get_expectations(result, test_name))
for specifier in self.normalized_specifiers(test_name, port_names):
line_parts = []
if specifier:
line_parts.append('[ %s ]' % specifier)
line_parts.append(test_name)
line_parts.append(expectations)
# Only add the bug link if the expectations do not include WontFix.
if 'WontFix' not in expectations:
line_parts.insert(0, result.bug)
lines.append(' '.join(line_parts))
return lines
def normalized_specifiers(self, test_name, port_names):
"""Converts and simplifies ports into platform specifiers.
Args:
test_name: The test name string.
port_names: A list of full port names that the line should apply to.
Returns:
A list of specifier string, e.g. ["Mac", "Win"].
[''] will be returned if the line should apply to all platforms.
"""
specifiers = []
for name in sorted(port_names):
specifiers.append(
self.host.builders.version_specifier_for_port_name(name))
if self.specifiers_can_extend_to_all_platforms(specifiers, test_name):
return ['']
specifiers = self.simplify_specifiers(
specifiers, self.port.configuration_specifier_macros())
if not specifiers:
return ['']
return specifiers
@staticmethod
def tuple_or_value_to_list(tuple_or_value):
"""Converts a tuple to a list, and a string value to a one-item list."""
if isinstance(tuple_or_value, tuple):
return list(tuple_or_value)
return [tuple_or_value]
def specifiers_can_extend_to_all_platforms(self, specifiers, test_name):
"""Tests whether a list of specifiers can be extended to all platforms.
Tries to add skipped platform specifiers to the list and tests if the
extended list covers all platforms.
"""
extended_specifiers = specifiers + self.skipped_specifiers(test_name)
# If the list is simplified to empty, then all platforms are covered.
return not self.simplify_specifiers(
extended_specifiers, self.port.configuration_specifier_macros())
def skipped_specifiers(self, test_name):
"""Returns a list of platform specifiers for which the test is skipped."""
specifiers = []
for port in self.all_try_builder_ports():
if port.skips_test(test_name):
specifiers.append(
self.host.builders.version_specifier_for_port_name(
port.name()))
return specifiers
@memoized
def all_try_builder_ports(self):
"""Returns a list of Port objects for all try builders."""
return [
self.host.port_factory.get_from_builder_name(name)
for name in self._get_try_bots()
]
def simplify_specifiers(self, specifiers, specifier_macros):
"""Simplifies the specifier part of an expectation line if possible.
"Simplifying" means finding the shortest list of platform specifiers
that is equivalent to the given list of specifiers. This can be done
because there are "macro specifiers" that stand in for multiple version
specifiers, and an empty list stands in for "all platforms".
Args:
specifiers: A collection of specifiers (case insensitive).
specifier_macros: A dict mapping "macros" for groups of specifiers
to lists of version specifiers. e.g. {"win": ["win7", "win10"]}.
If there are versions in this dict for that have no corresponding
try bots, they are ignored.
Returns:
A shortened list of specifiers (capitalized). For example, ["win7",
"win10"] would be converted to ["Win"]. If the given list covers
all supported platforms, then an empty list is returned.
"""
specifiers = {s.lower() for s in specifiers}
covered_by_try_bots = self._platform_specifiers_covered_by_try_bots()
for macro, versions in specifier_macros.iteritems():
macro = macro.lower()
# Only consider version specifiers that have corresponding try bots.
versions = {
s.lower()
for s in versions if s.lower() in covered_by_try_bots
}
if len(versions) == 0:
continue
if versions <= specifiers:
specifiers -= versions
specifiers.add(macro)
if specifiers == {macro.lower() for macro in specifier_macros}:
return []
return sorted(specifier.capitalize() for specifier in specifiers)
def _platform_specifiers_covered_by_try_bots(self):
all_platform_specifiers = set()
for builder_name in self._get_try_bots():
all_platform_specifiers.add(
self.host.builders.platform_specifier_for_builder(
builder_name).lower())
return frozenset(all_platform_specifiers)
def write_to_test_expectations(self, line_dict):
"""Writes the given lines to the TestExpectations file.
The place in the file where the new lines are inserted is after a marker
comment line. If this marker comment line is not found, then everything
including the marker line is appended to the end of the file.
All WontFix tests are inserted to NeverFixTests file instead of TextExpectations
file.
Args:
line_dict: A dictionary from test names to a list of test expectation lines.
"""
if not line_dict:
_log.info(
'No lines to write to TestExpectations, WebdriverExpectations or NeverFixTests.'
)
return
line_list = []
wont_fix_list = []
webdriver_list = []
for lines in line_dict.itervalues():
for line in lines:
if 'WontFix' in line:
wont_fix_list.append(line)
elif self.finder.webdriver_prefix() in line:
webdriver_list.append(line)
else:
line_list.append(line)
list_to_expectation = {
self.port.path_to_generic_test_expectations_file(): line_list,
self.port.path_to_webdriver_expectations_file(): webdriver_list
}
for expectations_file_path, lines in list_to_expectation.iteritems():
if not lines:
continue
_log.info('Lines to write to %s:\n %s', expectations_file_path,
'\n'.join(lines))
# Writes to TestExpectations file.
file_contents = self.host.filesystem.read_text_file(
expectations_file_path)
marker_comment_index = file_contents.find(MARKER_COMMENT)
if marker_comment_index == -1:
file_contents += '\n%s\n' % MARKER_COMMENT
file_contents += '\n'.join(lines)
else:
end_of_marker_line = (file_contents[marker_comment_index:].
find('\n')) + marker_comment_index
file_contents = file_contents[:end_of_marker_line +
1] + '\n'.join(
lines) + file_contents[
end_of_marker_line:]
self.host.filesystem.write_text_file(expectations_file_path,
file_contents)
if wont_fix_list:
_log.info('Lines to write to NeverFixTests:\n %s',
'\n'.join(wont_fix_list))
# Writes to NeverFixTests file.
wont_fix_path = self.port.path_to_never_fix_tests_file()
wont_fix_file_content = self.host.filesystem.read_text_file(
wont_fix_path)
if not wont_fix_file_content.endswith('\n'):
wont_fix_file_content += '\n'
wont_fix_file_content += '\n'.join(wont_fix_list)
wont_fix_file_content += '\n'
self.host.filesystem.write_text_file(wont_fix_path,
wont_fix_file_content)
# TODO(robertma): Unit test this method.
def download_text_baselines(self, test_results):
"""Fetches new baseline files for tests that should be rebaselined.
Invokes `blink_tool.py rebaseline-cl` in order to download new baselines
(-expected.txt files) for testharness.js tests that did not crash or
time out. Then, the platform-specific test is removed from the overall
failure test dictionary and the resulting dictionary is returned.
Args:
test_results: A dictionary of failing test results, mapping test
names to lists of platforms to SimpleTestResult.
Returns:
A pair: A set of tests that are rebaselined, and a modified copy of
the test_results dictionary containing only tests that couldn't be
rebaselined.
"""
tests_to_rebaseline, test_results = self.get_tests_to_rebaseline(
test_results)
if not tests_to_rebaseline:
_log.info('No tests to rebaseline.')
return tests_to_rebaseline, test_results
_log.info('Tests to rebaseline:')
for test in tests_to_rebaseline:
_log.info(' %s', test)
blink_tool = self.finder.path_from_blink_tools('blink_tool.py')
command = [
'python',
blink_tool,
'rebaseline-cl',
'--verbose',
'--no-trigger-jobs',
'--fill-missing',
]
if self.patchset:
command.append('--patchset=' + str(self.patchset))
command += tests_to_rebaseline
self.host.executive.run_command(command)
return tests_to_rebaseline, test_results
def get_tests_to_rebaseline(self, test_results):
"""Filters failing tests that can be rebaselined.
Creates a list of tests to rebaseline depending on the tests' platform-
specific results. In general, this will be non-ref tests that failed
due to a baseline mismatch (rather than crash or timeout).
Args:
test_results: A dictionary of failing test results, mapping test
names to lists of platforms to SimpleTestResult.
Returns:
A pair: A set of tests to be rebaselined, and a modified copy of
the test_results dictionary. The tests to be rebaselined should
include testharness.js tests that failed due to a baseline mismatch.
"""
new_test_results = copy.deepcopy(test_results)
tests_to_rebaseline = set()
for test_name in test_results:
for platforms, result in test_results[test_name].iteritems():
if self.can_rebaseline(test_name, result):
del new_test_results[test_name][platforms]
tests_to_rebaseline.add(test_name)
return sorted(tests_to_rebaseline), new_test_results
def can_rebaseline(self, test_name, result):
"""Checks if a test can be rebaselined.
Args:
test_name: The test name string.
result: A SimpleTestResult.
"""
if self.is_reference_test(test_name):
return False
if any(x in result.actual for x in ('CRASH', 'TIMEOUT', 'MISSING')):
return False
if self.is_webdriver_test(test_name):
return False
return True
def is_reference_test(self, test_name):
"""Checks whether a given test is a reference test."""
return bool(self.port.reference_files(test_name))
def is_webdriver_test(self, test_name):
"""Checks whether a given test is a WebDriver test."""
return self.finder.is_webdriver_test_path(test_name)
def _get_try_bots(self):
return self.host.builders.all_try_builder_names()
class DumpReaderMultipart(DumpReader):
"""Base class for Linux and Android breakpad dump reader."""
def __init__(self, host, build_dir):
super(DumpReaderMultipart, self).__init__(host, build_dir)
self._path_finder = PathFinder(host.filesystem)
self._breakpad_tools_available = None
self._generated_symbols = False
def check_is_functional(self):
return self._check_breakpad_tools_available()
def _get_pid_from_dump(self, dump_file):
dump = self._read_dump(dump_file)
if not dump:
return None
if 'pid' in dump:
return dump['pid'][0]
return None
def _get_stack_from_dump(self, dump_file):
dump = self._read_dump(dump_file)
if not dump:
return None
if not 'upload_file_minidump' in dump:
return None
self._generate_breakpad_symbols_if_necessary()
f, temp_name = self._host.filesystem.open_binary_tempfile('dmp')
f.write('\r\n'.join(dump['upload_file_minidump']))
f.close()
cmd = [
self._path_to_minidump_stackwalk(), temp_name,
self._symbols_dir()
]
try:
stack = self._host.executive.run_command(cmd, return_stderr=False)
except:
_log.warning('Failed to execute "%s"', ' '.join(cmd))
stack = None
finally:
self._host.filesystem.remove(temp_name)
return stack
def _read_dump(self, dump_file):
with self._host.filesystem.open_binary_file_for_reading(
dump_file) as f:
boundary = f.readline().strip()[2:]
f.seek(0)
try:
data = cgi.parse_multipart(f, {'boundary': boundary})
return data
except:
pass
return None
def _check_breakpad_tools_available(self):
if self._breakpad_tools_available is not None:
return self._breakpad_tools_available
REQUIRED_BREAKPAD_TOOLS = [
'dump_syms',
'minidump_stackwalk',
]
result = True
for binary in REQUIRED_BREAKPAD_TOOLS:
full_path = self._host.filesystem.join(self._build_dir, binary)
if not self._host.filesystem.exists(full_path):
result = False
_log.error('Unable to find %s', binary)
_log.error(' at %s', full_path)
if not result:
_log.error(
" Could not find breakpad tools, unexpected crashes won't be symbolized"
)
_log.error(' Did you build the target blink_tests?')
_log.error('')
self._breakpad_tools_available = result
return self._breakpad_tools_available
def _path_to_minidump_stackwalk(self):
return self._host.filesystem.join(self._build_dir,
'minidump_stackwalk')
def _path_to_generate_breakpad_symbols(self):
return self._path_finder.path_from_chromium_base(
'components', 'crash', 'content', 'tools',
'generate_breakpad_symbols.py')
def _symbols_dir(self):
return self._host.filesystem.join(self._build_dir,
'content_shell.syms')
def _generate_breakpad_symbols_if_necessary(self):
if self._generated_symbols:
return
self._generated_symbols = True
_log.debug('Generating breakpad symbols')
queue = Queue.Queue()
thread = threading.Thread(target=_symbolize_keepalive, args=(queue, ))
thread.start()
try:
for binary in self._binaries_to_symbolize():
_log.debug(' Symbolizing %s', binary)
full_path = self._host.filesystem.join(self._build_dir, binary)
cmd = [
self._path_to_generate_breakpad_symbols(),
'--binary=%s' % full_path,
'--symbols-dir=%s' % self._symbols_dir(),
'--build-dir=%s' % self._build_dir,
]
try:
self._host.executive.run_command(cmd)
except:
_log.error('Failed to execute "%s"', ' '.join(cmd))
finally:
queue.put(None)
thread.join()
_log.debug('Done generating breakpad symbols')
def _binaries_to_symbolize(self):
"""This routine must be implemented by subclasses.
Returns an array of binaries that need to be symbolized.
"""
raise NotImplementedError()
def _shorten_filename(self, filename):
finder = PathFinder(self._port.host.filesystem)
if filename.startswith(finder.path_from_chromium_base()):
return self._port.host.filesystem.relpath(filename, finder.path_from_chromium_base())
return filename
def __init__(self, host, build_dir):
super(DumpReaderMultipart, self).__init__(host, build_dir)
self._path_finder = PathFinder(host.filesystem)
self._breakpad_tools_available = None
self._generated_symbols = False
class WPTExpectationsUpdater(object):
MARKER_COMMENT = '# ====== New tests from wpt-importer added here ======'
UMBRELLA_BUG = 'crbug.com/626703'
def __init__(self, host, args=None, wpt_manifests=None):
self.host = host
self.port = self.host.port_factory.get()
self.finder = PathFinder(self.host.filesystem)
self.git_cl = GitCL(host)
self.git = self.host.git(self.finder.chromium_base())
self.configs_with_no_results = []
self.patchset = None
self.wpt_manifests = (
wpt_manifests
or [self.port.wpt_manifest(d) for d in self.port.WPT_DIRS])
# Get options from command line arguments.
parser = argparse.ArgumentParser(description=__doc__)
self.add_arguments(parser)
self.options = parser.parse_args(args or [])
if not (self.options.clean_up_test_expectations
or self.options.clean_up_test_expectations_only):
assert not self.options.clean_up_affected_tests_only, (
'Cannot use --clean-up-affected-tests-only without using '
'--clean-up-test-expectations or '
'--clean-up-test-expectations-only')
# Set up TestExpectations instance which contains all
# expectations files associated with the platform.
expectations_dict = {
p: self.host.filesystem.read_text_file(p)
for p in self.expectations_files()
}
self._test_expectations = TestExpectations(
self.port, expectations_dict=expectations_dict)
def expectations_files(self):
"""Returns list of expectations files.
Each expectation file in the list will be cleaned of expectations
for tests that were removed and will also have test names renamed
for tests that were renamed. Also the files may have their expectations
updated using builder results.
"""
return (self.port.all_expectations_dict().keys() +
PRODUCTS_TO_EXPECTATION_FILE_PATHS.values())
def run(self):
"""Does required setup before calling update_expectations().
Do not override this function!
"""
log_level = logging.DEBUG if self.options.verbose else logging.INFO
configure_logging(logging_level=log_level, include_time=True)
if not (self.options.android_product
or self.options.update_android_expectations_only):
assert not self.options.include_unexpected_pass, (
'Command line argument --include-unexpected-pass is not '
'supported in desktop mode.')
self.patchset = self.options.patchset
if (self.options.clean_up_test_expectations
or self.options.clean_up_test_expectations_only):
# Remove expectations for deleted tests and rename tests in
# expectations for renamed tests.
self.cleanup_test_expectations_files()
if not self.options.clean_up_test_expectations_only:
# Use try job results to update expectations and baselines
self.update_expectations()
return 0
def add_arguments(self, parser):
parser.add_argument(
'--patchset',
default=None,
help='Patchset number to fetch new baselines from.')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='More verbose logging.')
parser.add_argument('--clean-up-test-expectations',
action='store_true',
help='Cleanup test expectations files.')
parser.add_argument('--clean-up-test-expectations-only',
action='store_true',
help='Clean up expectations and then exit script.')
parser.add_argument(
'--clean-up-affected-tests-only',
action='store_true',
help='Only cleanup expectations deleted or renamed in current CL. '
'If flag is not used then a full cleanup of deleted or '
'renamed tests will be done in expectations.')
# TODO(rmhasan): Move below arguments to the
# AndroidWPTExpectationsUpdater add_arguments implementation.
# Also look into using sub parsers to separate android and
# desktop specific arguments.
parser.add_argument(
'--update-android-expectations-only',
action='store_true',
help='Update and clean up only Android test expectations.')
parser.add_argument(
'--android-product',
action='append',
default=[],
help='Android products whose baselines will be updated.',
choices=PRODUCTS)
parser.add_argument(
'--include-unexpected-pass',
action='store_true',
help='Adds Pass to tests with failure expectations. '
'This command line argument can be used to mark tests '
'as flaky.')
def update_expectations(self):
"""Downloads text new baselines and adds test expectations lines.
Returns:
A pair: A set of tests that are rebaselined, and a dictionary
mapping tests that couldn't be rebaselined to lists of expectation
lines written to TestExpectations.
"""
# The wpt_manifest function in Port is cached by default, but may be out
# of date if this code is called during test import. An out of date
# manifest will cause us to mistreat newly added tests, as they will not
# exist in the cached manifest. To avoid this, we invalidate the cache
# here. See https://crbug.com/1154650 .
self.port.wpt_manifest.cache_clear()
issue_number = self.get_issue_number()
if issue_number == 'None':
raise ScriptError('No issue on current branch.')
build_to_status = self.get_latest_try_jobs()
_log.debug('Latest try jobs: %r', build_to_status)
if not build_to_status:
raise ScriptError('No try job information was collected.')
# Here we build up a dict of failing test results for all platforms.
test_expectations = {}
for build, job_status in build_to_status.iteritems():
if (job_status.result == 'SUCCESS'
and not self.options.include_unexpected_pass):
continue
# Temporary logging for https://crbug.com/1154650
result_dicts = self.get_failing_results_dicts(build)
_log.info('Merging failing results dicts for %s', build)
for result_dict in result_dicts:
test_expectations = self.merge_dicts(test_expectations,
result_dict)
# At this point, test_expectations looks like: {
# 'test-with-failing-result': {
# config1: SimpleTestResult,
# config2: SimpleTestResult,
# config3: AnotherSimpleTestResult
# }
# }
# And then we merge results for different platforms that had the same results.
for test_name, platform_result in test_expectations.iteritems():
# platform_result is a dict mapping platforms to results.
test_expectations[test_name] = self.merge_same_valued_keys(
platform_result)
# At this point, test_expectations looks like: {
# 'test-with-failing-result': {
# (config1, config2): SimpleTestResult,
# (config3,): AnotherSimpleTestResult
# }
# }
rebaselined_tests, test_expectations = self.download_text_baselines(
test_expectations)
exp_lines_dict = self.write_to_test_expectations(test_expectations)
return rebaselined_tests, exp_lines_dict
def get_issue_number(self):
"""Returns current CL number. Can be replaced in unit tests."""
return self.git_cl.get_issue_number()
def get_latest_try_jobs(self):
"""Returns the latest finished try jobs as Build objects."""
return self.git_cl.latest_try_jobs(builder_names=self._get_try_bots(),
patchset=self.patchset)
def get_failing_results_dicts(self, build):
"""Returns a list of nested dicts of failing test results.
Retrieves a full list of web test results from a builder result URL.
Collects the builder name, platform and a list of tests that did not
run as expected.
Args:
build: A Build object.
Returns:
A list of dictionaries that have the following structure.
{
'test-with-failing-result': {
config: SimpleTestResult
}
}
If results could be fetched but none are failing,
this will return an empty list.
"""
test_results_list = self._get_web_test_results(build)
has_webdriver_tests = self.host.builders.has_webdriver_tests_for_builder(
build.builder_name)
if has_webdriver_tests:
master = self.host.builders.master_for_builder(build.builder_name)
test_results_list.append(
self.host.results_fetcher.fetch_webdriver_test_results(
build, master))
test_results_list = filter(None, test_results_list)
if not test_results_list:
_log.warning('No results for build %s', build)
self.configs_with_no_results.extend(
self.get_builder_configs(build))
return []
unexpected_test_results = []
for results_set in test_results_list:
results_dict = self.generate_failing_results_dict(
build, results_set)
if results_dict:
unexpected_test_results.append(results_dict)
return unexpected_test_results
def _get_web_test_results(self, build):
"""Gets web tests results for a builder.
Args:
build: Named tuple containing builder name and number
Returns:
List of web tests results for each web test step
in build.
"""
return [self.host.results_fetcher.fetch_results(build)]
def get_builder_configs(self, build, *_):
return [DesktopConfig(port_name=self.port_name(build))]
@memoized
def port_name(self, build):
return self.host.builders.port_name_for_builder_name(
build.builder_name)
def generate_failing_results_dict(self, build, web_test_results):
"""Makes a dict with results for one platform.
Args:
builder: Builder instance containing builder information..
web_test_results: A list of WebTestResult objects.
Returns:
A dictionary with the structure: {
'test-name': {
('full-port-name',): SimpleTestResult
}
}
"""
test_dict = {}
configs = self.get_builder_configs(build, web_test_results)
_log.debug(
'Getting failing results dictionary for %s step in latest %s build',
web_test_results.step_name(), build.builder_name)
if len(configs) > 1:
raise ScriptError('More than one configs were produced for'
' builder and web tests step combination')
if not configs:
raise ScriptError(
'No configuration was found for builder and web test'
' step combination ')
config = configs[0]
for result in web_test_results.didnt_run_as_expected_results():
# TODO(rmhasan) If a test fails unexpectedly then it runs multiple
# times until, it passes or a retry limit is reached. Even though
# it passed we there are still flaky failures that we are not
# creating test expectations for. Maybe we should add a mode
# which creates expectations for tests that are flaky but still
# pass in a web test step.
# Create flaky expectations for flaky tests on Android. In order to
# do this we should add 'Pass' to all tests with failing
# expectations that pass in the patchset's try job.
if result.did_pass() and not self.options.include_unexpected_pass:
continue
test_name = result.test_name()
if not self._is_wpt_test(test_name):
continue
test_dict[test_name] = {
config:
SimpleTestResult(expected=result.expected_results(),
actual=result.actual_results(),
bug=self.UMBRELLA_BUG)
}
return test_dict
def _is_wpt_test(self, test_name):
"""Check if a web test is a WPT tests.
In blink web tests results, each test name is relative to
the web_tests directory instead of the wpt directory. We
need to use the port.is_wpt_test() function to find out if a test
is from the WPT suite.
Returns: True if a test is in the external/wpt subdirectory of
the web_tests directory."""
return self.port.is_wpt_test(test_name)
def merge_dicts(self, target, source, path=None):
"""Recursively merges nested dictionaries.
Args:
target: First dictionary, which is updated based on source.
source: Second dictionary, not modified.
path: A list of keys, only used for making error messages.
Returns:
The updated target dictionary.
"""
path = path or []
for key in source:
if key in target:
if (isinstance(target[key], dict)) and isinstance(
source[key], dict):
self.merge_dicts(target[key], source[key],
path + [str(key)])
elif target[key] == source[key]:
pass
else:
# We have two different SimpleTestResults for the same test
# from two different builders. This can happen when a CQ bot
# and a blink-rel bot run on the same platform. We union the
# actual statuses from both builders.
_log.info(
"Joining differing results for path %s, key %s\n target:%s\nsource:%s"
% (path, key, target[key], source[key]))
target[key] = SimpleTestResult(
expected=target[key].expected,
actual='%s %s' %
(target[key].actual, source[key].actual),
bug=target[key].bug)
else:
target[key] = source[key]
return target
def merge_same_valued_keys(self, dictionary):
"""Merges keys in dictionary with same value.
Traverses through a dict and compares the values of keys to one another.
If the values match, the keys are combined to a tuple and the previous
keys are removed from the dict.
Args:
dictionary: A dictionary with a dictionary as the value.
Returns:
A new dictionary with updated keys to reflect matching values of keys.
Example: {
'one': {'foo': 'bar'},
'two': {'foo': 'bar'},
'three': {'foo': 'bar'}
}
is converted to a new dictionary with that contains
{('one', 'two', 'three'): {'foo': 'bar'}}
"""
merged_dict = {}
matching_value_keys = set()
keys = sorted(dictionary.keys())
while keys:
current_key = keys[0]
found_match = False
if current_key == keys[-1]:
merged_dict[tuple([current_key])] = dictionary[current_key]
keys.remove(current_key)
break
for next_item in keys[1:]:
if dictionary[current_key] == dictionary[next_item]:
found_match = True
matching_value_keys.update([current_key, next_item])
if next_item == keys[-1]:
if found_match:
merged_dict[tuple(
matching_value_keys)] = dictionary[current_key]
keys = [
k for k in keys if k not in matching_value_keys
]
else:
merged_dict[tuple([current_key
])] = dictionary[current_key]
keys.remove(current_key)
matching_value_keys = set()
return merged_dict
def get_expectations(self, result, test_name=''):
"""Returns a set of test expectations based on the result of a test.
Returns a set of one or more test expectations based on the expected
and actual results of a given test name. This function is to decide
expectations for tests that could not be rebaselined.
Args:
result: A SimpleTestResult.
test_name: The test name string (optional).
Returns:
A set of one or more test expectation strings with the first letter
capitalized. Example: {'Failure', 'Timeout'}.
"""
actual_results = set(result.actual.split())
# If the result is MISSING, this implies that the test was not
# rebaselined and has an actual result but no baseline. We can't
# add a Missing expectation (this is not allowed), but no other
# expectation is correct.
# We also want to skip any new manual tests that are not automated;
# see crbug.com/708241 for context.
if 'MISSING' in actual_results:
return {'Skip'}
if '-manual.' in test_name and 'TIMEOUT' in actual_results:
return {'Skip'}
expectations = set()
failure_types = {'TEXT', 'IMAGE+TEXT', 'IMAGE', 'AUDIO', 'FAIL'}
other_types = {'TIMEOUT', 'CRASH', 'PASS'}
for actual in actual_results:
if actual in failure_types:
expectations.add('Failure')
if actual in other_types:
expectations.add(actual.capitalize())
return expectations
def create_line_dict(self, merged_results):
"""Creates list of test expectations lines.
Traverses through the given |merged_results| dictionary and parses the
value to create one test expectations line per key.
Test expectation lines have the following format:
['BUG_URL [PLATFORM(S)] TEST_NAME [EXPECTATION(S)]']
Args:
merged_results: A dictionary with the format:
{
'test-with-failing-result': {
(config1, config2): SimpleTestResult,
(config3,): SimpleTestResult
}
}
Returns:
A dictionary from test names to a list of test expectation lines
(each SimpleTestResult turns into a line).
"""
line_dict = defaultdict(list)
for test_name, test_results in sorted(merged_results.iteritems()):
if not self._is_wpt_test(test_name):
_log.warning(
'Non-WPT test "%s" unexpectedly passed to create_line_dict.',
test_name)
continue
for configs, result in sorted(test_results.iteritems()):
line_dict[test_name].extend(
self._create_lines(test_name, configs, result))
return line_dict
def _create_lines(self, test_name, configs, result):
"""Constructs test expectation line strings.
Args:
test_name: The test name string.
configs: A list of full configs that the line should apply to.
result: A SimpleTestResult.
Returns:
A list of strings which each is a line of test expectation for given
|test_name|.
"""
lines = []
# The set of ports with no results is assumed to have have no
# overlap with the set of port names passed in here.
assert set(configs) & set(self.configs_with_no_results) == set()
# The ports with no results are generally ports of builders that
# failed, maybe for unrelated reasons. At this point, we add ports
# with no results to the list of platforms because we're guessing
# that this new expectation might be cross-platform and should
# also apply to any ports that we weren't able to get results for.
configs = tuple(list(configs) + self.configs_with_no_results)
expectations = '[ %s ]' % \
' '.join(self.get_expectations(result, test_name))
for specifier in self.normalized_specifiers(test_name, configs):
line_parts = []
if specifier:
line_parts.append('[ %s ]' % specifier)
# Escape literal asterisks for typ (https://crbug.com/1036130).
line_parts.append(test_name.replace('*', '\\*'))
line_parts.append(expectations)
# Only add the bug link if the expectations do not include WontFix.
if 'WontFix' not in expectations and result.bug:
line_parts.insert(0, result.bug)
lines.append(' '.join(line_parts))
return lines
def normalized_specifiers(self, test_name, configs):
"""Converts and simplifies ports into platform specifiers.
Args:
test_name: The test name string.
configs: A list of full configs that the line should apply to.
Returns:
A list of specifier string, e.g. ["Mac", "Win"].
[''] will be returned if the line should apply to all platforms.
"""
specifiers = []
for config in configs:
specifiers.append(
self.host.builders.version_specifier_for_port_name(
config.port_name))
if self.specifiers_can_extend_to_all_platforms(specifiers, test_name):
return ['']
specifiers = self.simplify_specifiers(
specifiers, self.port.configuration_specifier_macros())
if not specifiers:
return ['']
return specifiers
def specifiers_can_extend_to_all_platforms(self, specifiers, test_name):
"""Tests whether a list of specifiers can be extended to all platforms.
Tries to add skipped platform specifiers to the list and tests if the
extended list covers all platforms.
"""
extended_specifiers = specifiers + self.skipped_specifiers(test_name)
# If the list is simplified to empty, then all platforms are covered.
return not self.simplify_specifiers(
extended_specifiers, self.port.configuration_specifier_macros())
def skipped_specifiers(self, test_name):
"""Returns a list of platform specifiers for which the test is skipped."""
specifiers = []
for port in self.all_try_builder_ports():
if port.skips_test(test_name):
specifiers.append(
self.host.builders.version_specifier_for_port_name(
port.name()))
return specifiers
@memoized
def all_try_builder_ports(self):
"""Returns a list of Port objects for all try builders."""
return [
self.host.port_factory.get_from_builder_name(name)
for name in self._get_try_bots()
]
def simplify_specifiers(self, specifiers, specifier_macros):
"""Simplifies the specifier part of an expectation line if possible.
"Simplifying" means finding the shortest list of platform specifiers
that is equivalent to the given list of specifiers. This can be done
because there are "macro specifiers" that stand in for multiple version
specifiers, and an empty list stands in for "all platforms".
Args:
specifiers: A collection of specifiers (case insensitive).
specifier_macros: A dict mapping "macros" for groups of specifiers
to lists of version specifiers. e.g. {"win": ["win7", "win10"]}.
If there are versions in this dict for that have no corresponding
try bots, they are ignored.
Returns:
A shortened list of specifiers (capitalized). For example, ["win7",
"win10"] would be converted to ["Win"]. If the given list covers
all supported platforms, then an empty list is returned.
"""
specifiers = {s.lower() for s in specifiers}
covered_by_try_bots = self._platform_specifiers_covered_by_try_bots()
for macro, versions in specifier_macros.iteritems():
macro = macro.lower()
# Only consider version specifiers that have corresponding try bots.
versions = {
s.lower()
for s in versions if s.lower() in covered_by_try_bots
}
if len(versions) == 0:
continue
if versions <= specifiers:
specifiers -= versions
specifiers.add(macro)
if specifiers == {macro.lower() for macro in specifier_macros}:
return []
return sorted(specifier.capitalize() for specifier in specifiers)
def _platform_specifiers_covered_by_try_bots(self):
all_platform_specifiers = set()
for builder_name in self._get_try_bots():
all_platform_specifiers.add(
self.host.builders.platform_specifier_for_builder(
builder_name).lower())
return frozenset(all_platform_specifiers)
def write_to_test_expectations(self, test_expectations):
"""Writes the given lines to the TestExpectations file.
The place in the file where the new lines are inserted is after a marker
comment line. If this marker comment line is not found, then everything
including the marker line is appended to the end of the file.
All WontFix tests are inserted to NeverFixTests file instead of TextExpectations
file.
Args:
test_expectations: A dictionary mapping test names to a dictionary
mapping platforms and test results.
Returns:
Dictionary mapping test names to lists of test expectation strings.
"""
line_dict = self.create_line_dict(test_expectations)
if not line_dict:
_log.info('No lines to write to TestExpectations,'
' WebdriverExpectations or NeverFixTests.')
return {}
line_list = []
wont_fix_list = []
webdriver_list = []
for lines in line_dict.itervalues():
for line in lines:
if 'Skip' in line and '-manual.' in line:
wont_fix_list.append(line)
elif self.finder.webdriver_prefix() in line:
webdriver_list.append(line)
else:
line_list.append(line)
list_to_expectation = {
self.port.path_to_generic_test_expectations_file(): line_list,
self.port.path_to_webdriver_expectations_file(): webdriver_list
}
for expectations_file_path, lines in list_to_expectation.iteritems():
if not lines:
continue
_log.info('Lines to write to %s:\n %s', expectations_file_path,
'\n'.join(lines))
# Writes to TestExpectations file.
file_contents = self.host.filesystem.read_text_file(
expectations_file_path)
marker_comment_index = file_contents.find(self.MARKER_COMMENT)
if marker_comment_index == -1:
file_contents += '\n%s\n' % self.MARKER_COMMENT
file_contents += '\n'.join(lines)
else:
end_of_marker_line = (file_contents[marker_comment_index:].
find('\n')) + marker_comment_index
file_contents = (file_contents[:end_of_marker_line + 1] +
'\n'.join(lines) +
file_contents[end_of_marker_line:])
self.host.filesystem.write_text_file(expectations_file_path,
file_contents)
if wont_fix_list:
_log.info('Lines to write to NeverFixTests:\n %s',
'\n'.join(wont_fix_list))
# Writes to NeverFixTests file.
wont_fix_path = self.port.path_to_never_fix_tests_file()
wont_fix_file_content = self.host.filesystem.read_text_file(
wont_fix_path)
if not wont_fix_file_content.endswith('\n'):
wont_fix_file_content += '\n'
wont_fix_file_content += '\n'.join(wont_fix_list)
wont_fix_file_content += '\n'
self.host.filesystem.write_text_file(wont_fix_path,
wont_fix_file_content)
return line_dict
def cleanup_test_expectations_files(self):
"""Removes deleted tests from expectations files.
Removes expectations for deleted tests or renames test names in
expectation files for tests that were renamed. If the
--clean-up-affected-tests-only command line argument is used then
only tests deleted in the CL will have their expectations removed
through this script. If that command line argument is not used then
expectations for test files that no longer exist will be deleted.
"""
deleted_files = self._list_deleted_files()
renamed_files = self._list_renamed_files()
for path in self._test_expectations.expectations_dict:
_log.info('Updating %s for any removed or renamed tests.',
self.host.filesystem.basename(path))
self._clean_single_test_expectations_file(path, deleted_files,
renamed_files)
self._test_expectations.commit_changes()
def _list_deleted_files(self):
# TODO(robertma): Improve Git.changed_files so that we can use
# it here.
paths = self.git.run(
['diff', 'origin/master', '--diff-filter=D',
'--name-only']).splitlines()
deleted_files = []
for p in paths:
rel_path = self._relative_to_web_test_dir(p)
if rel_path:
deleted_files.append(rel_path)
return deleted_files
def _list_renamed_files(self):
"""Returns a dictionary mapping tests to their new name.
Regardless of the command line arguments used this test will only
return a dictionary for tests affected in the current CL.
Returns a dictionary mapping source name to destination name.
"""
out = self.git.run([
'diff', 'origin/master', '-M90%', '--diff-filter=R',
'--name-status'
])
renamed_tests = {}
for line in out.splitlines():
_, source_path, dest_path = line.split()
source_test = self._relative_to_web_test_dir(source_path)
dest_test = self._relative_to_web_test_dir(dest_path)
if source_test and dest_test:
renamed_tests[source_test] = dest_test
return renamed_tests
def _clean_single_test_expectations_file(self, path, deleted_files,
renamed_files):
"""Cleans up a single test expectations file.
Args:
path: Path of expectations file that is being cleaned up.
deleted_files: List of file paths relative to the web tests
directory which were deleted.
renamed_files: Dictionary mapping file paths to their new file
name after renaming.
"""
deleted_files = set(deleted_files)
for line in self._test_expectations.get_updated_lines(path):
# if a test is a glob type expectation or empty line or comment then
# add it to the updated expectations file without modifications
if not line.test or line.is_glob:
continue
root_file = self._get_root_file(line.test)
if root_file in deleted_files:
self._test_expectations.remove_expectations(path, [line])
elif root_file in renamed_files:
self._test_expectations.remove_expectations(path, [line])
new_file_name = renamed_files[root_file]
if self.finder.is_webdriver_test_path(line.test):
_, subtest_suffix = self.port.split_webdriver_test_name(
line.test)
line.test = self.port.add_webdriver_subtest_suffix(
new_file_name, subtest_suffix)
elif self.port.is_wpt_test(line.test):
# Based on logic in Base._wpt_test_urls_matching_paths
line.test = line.test.replace(
re.sub(r'\.js$', '.', root_file),
re.sub(r'\.js$', '.', new_file_name))
else:
line.test = new_file_name
self._test_expectations.add_expectations(path, [line],
lineno=line.lineno)
elif not root_file or not self.port.test_isfile(root_file):
if not self.options.clean_up_affected_tests_only:
self._test_expectations.remove_expectations(path, [line])
@memoized
def _get_root_file(self, test_name):
"""Finds the physical file in web tests directory for a test
If a test is a WPT test then it will look in each of the WPT manifests
for the physical file. If test name cannot be found in any of the manifests
then the test no longer exists and the function will return None. If a file
is webdriver test then it will strip all subtest arguments and return the
file path. If a test is a legacy web test then it will return the test name.
Args:
test_name: Test name which may include test arguments.
Returns:
Returns the path of the physical file that backs
up a test. The path is relative to the web_tests directory.
"""
if self.finder.is_webdriver_test_path(test_name):
root_test_file, _ = (
self.port.split_webdriver_test_name(test_name))
return root_test_file
elif self.port.is_wpt_test(test_name):
for wpt_manifest in self.wpt_manifests:
if test_name.startswith(wpt_manifest.wpt_dir):
wpt_test = test_name[len(wpt_manifest.wpt_dir) + 1:]
if wpt_manifest.is_test_url(wpt_test):
return self.host.filesystem.join(
wpt_manifest.wpt_dir,
wpt_manifest.file_path_for_test_url(wpt_test))
# The test was not found in any of the wpt manifests, therefore
# the test does not exist. So we will return None in this case.
return None
else:
# Non WPT and non webdriver tests have no file parameters, and
# the physical file path is the actual name of the test.
return test_name
def _relative_to_web_test_dir(self, path_relative_to_repo_root):
"""Returns a path that's relative to the web tests directory."""
abs_path = self.finder.path_from_chromium_base(
path_relative_to_repo_root)
if not abs_path.startswith(self.finder.web_tests_dir()):
return None
return self.host.filesystem.relpath(abs_path,
self.finder.web_tests_dir())
# TODO(robertma): Unit test this method.
def download_text_baselines(self, test_results):
"""Fetches new baseline files for tests that should be rebaselined.
Invokes `blink_tool.py rebaseline-cl` in order to download new baselines
(-expected.txt files) for testharness.js tests that did not crash or
time out. Then, the platform-specific test is removed from the overall
failure test dictionary and the resulting dictionary is returned.
Args:
test_results: A dictionary of failing test results, mapping test
names to lists of platforms to SimpleTestResult.
Returns:
A pair: A set of tests that are rebaselined, and a modified copy of
the test_results dictionary containing only tests that couldn't be
rebaselined.
"""
tests_to_rebaseline, test_results = self.get_tests_to_rebaseline(
test_results)
if not tests_to_rebaseline:
_log.info('No tests to rebaseline.')
return tests_to_rebaseline, test_results
_log.info('Tests to rebaseline:')
for test in tests_to_rebaseline:
_log.info(' %s', test)
blink_tool = self.finder.path_from_blink_tools('blink_tool.py')
command = [
'python',
blink_tool,
'rebaseline-cl',
'--verbose',
'--no-trigger-jobs',
'--fill-missing',
]
if self.patchset:
command.append('--patchset=' + str(self.patchset))
command += tests_to_rebaseline
self.host.executive.run_command(command)
return tests_to_rebaseline, test_results
def get_tests_to_rebaseline(self, test_results):
"""Filters failing tests that can be rebaselined.
Creates a list of tests to rebaseline depending on the tests' platform-
specific results. In general, this will be non-ref tests that failed
due to a baseline mismatch (rather than crash or timeout).
Args:
test_results: A dictionary of failing test results, mapping test
names to lists of platforms to SimpleTestResult.
Returns:
A pair: A set of tests to be rebaselined, and a modified copy of
the test_results dictionary. The tests to be rebaselined should
include testharness.js tests that failed due to a baseline mismatch.
"""
new_test_results = copy.deepcopy(test_results)
tests_to_rebaseline = set()
for test_name in test_results:
for platforms, result in test_results[test_name].iteritems():
if self.can_rebaseline(test_name, result):
del new_test_results[test_name][platforms]
tests_to_rebaseline.add(test_name)
return sorted(tests_to_rebaseline), new_test_results
def can_rebaseline(self, test_name, result):
"""Checks if a test can be rebaselined.
Args:
test_name: The test name string.
result: A SimpleTestResult.
"""
if self.is_reference_test(test_name):
return False
if any(x in result.actual for x in ('CRASH', 'TIMEOUT', 'MISSING')):
return False
if self.is_webdriver_test(test_name):
return False
return True
def is_reference_test(self, test_name):
"""Checks whether a given test is a reference test."""
return bool(self.port.reference_files(test_name))
def is_webdriver_test(self, test_name):
"""Checks whether a given test is a WebDriver test."""
return self.finder.is_webdriver_test_path(test_name)
@memoized
def _get_try_bots(self):
return self.host.builders.filter_builders(
is_try=True, exclude_specifiers={'android'})
def _test_base_path(self):
"""Returns the relative path from the repo root to the web tests."""
finder = PathFinder(self._tool.filesystem)
return self._tool.filesystem.relpath(
finder.web_tests_dir(), finder.path_from_chromium_base()) + '/'
class TestCopier(object):
def __init__(self, host, source_repo_path):
"""Initializes variables to prepare for copying and converting files.
Args:
host: An instance of Host.
source_repo_path: Path to the local checkout of web-platform-tests.
"""
self.host = host
assert self.host.filesystem.exists(source_repo_path)
self.source_repo_path = source_repo_path
self.filesystem = self.host.filesystem
self.path_finder = PathFinder(self.filesystem)
self.layout_tests_dir = self.path_finder.layout_tests_dir()
self.destination_directory = self.filesystem.normpath(
self.filesystem.join(
self.layout_tests_dir, DEST_DIR_NAME,
self.filesystem.basename(self.source_repo_path)))
self.import_in_place = (
self.source_repo_path == self.destination_directory)
self.dir_above_repo = self.filesystem.dirname(self.source_repo_path)
self.import_list = []
# This is just a FYI list of CSS properties that still need to be prefixed,
# which may be output after importing.
self._prefixed_properties = {}
def do_import(self):
_log.info('Importing %s into %s', self.source_repo_path,
self.destination_directory)
self.find_importable_tests()
self.import_tests()
def find_importable_tests(self):
"""Walks through the source directory to find what tests should be imported.
This function sets self.import_list, which contains information about how many
tests are being imported, and their source and destination paths.
"""
paths_to_skip = self.find_paths_to_skip()
for root, dirs, files in self.filesystem.walk(self.source_repo_path):
cur_dir = root.replace(self.dir_above_repo + '/', '') + '/'
_log.debug('Scanning %s...', cur_dir)
dirs_to_skip = ('.git', )
if dirs:
for name in dirs_to_skip:
if name in dirs:
dirs.remove(name)
for path in paths_to_skip:
path_base = path.replace(DEST_DIR_NAME + '/', '')
path_base = path_base.replace(cur_dir, '')
path_full = self.filesystem.join(root, path_base)
if path_base in dirs:
_log.info('Skipping: %s', path_full)
dirs.remove(path_base)
if self.import_in_place:
self.filesystem.rmtree(path_full)
copy_list = []
for filename in files:
path_full = self.filesystem.join(root, filename)
path_base = path_full.replace(self.source_repo_path + '/', '')
path_base = self.destination_directory.replace(
self.layout_tests_dir + '/', '') + '/' + path_base
if path_base in paths_to_skip:
if self.import_in_place:
_log.debug('Pruning: %s', path_base)
self.filesystem.remove(path_full)
continue
else:
continue
# FIXME: This block should really be a separate function, but the early-continues make that difficult.
if is_basename_skipped(filename):
_log.debug('Skipping: %s', path_full)
_log.debug(
' Reason: This file may cause Chromium presubmit to fail.'
)
continue
copy_list.append({'src': path_full, 'dest': filename})
if copy_list:
# Only add this directory to the list if there's something to import
self.import_list.append({
'dirname': root,
'copy_list': copy_list
})
def find_paths_to_skip(self):
paths_to_skip = set()
port = self.host.port_factory.get()
w3c_import_expectations_path = self.path_finder.path_from_layout_tests(
'W3CImportExpectations')
w3c_import_expectations = self.filesystem.read_text_file(
w3c_import_expectations_path)
parser = TestExpectationParser(port, all_tests=(), is_lint_mode=False)
expectation_lines = parser.parse(w3c_import_expectations_path,
w3c_import_expectations)
for line in expectation_lines:
if 'SKIP' in line.expectations:
if line.specifiers:
_log.warning(
'W3CImportExpectations:%s should not have any specifiers',
line.line_numbers)
continue
paths_to_skip.add(line.name)
return paths_to_skip
def import_tests(self):
"""Reads |self.import_list|, and converts and copies files to their destination."""
for dir_to_copy in self.import_list:
if not dir_to_copy['copy_list']:
continue
orig_path = dir_to_copy['dirname']
relative_dir = self.filesystem.relpath(orig_path,
self.source_repo_path)
dest_dir = self.filesystem.join(self.destination_directory,
relative_dir)
if not self.filesystem.exists(dest_dir):
self.filesystem.maybe_make_directory(dest_dir)
for file_to_copy in dir_to_copy['copy_list']:
self.copy_file(file_to_copy, dest_dir)
_log.info('')
_log.info('Import complete')
_log.info('')
if self._prefixed_properties:
_log.info('Properties needing prefixes (by count):')
for prefixed_property in sorted(
self._prefixed_properties,
key=lambda p: self._prefixed_properties[p]):
_log.info(' %s: %s', prefixed_property,
self._prefixed_properties[prefixed_property])
def copy_file(self, file_to_copy, dest_dir):
"""Converts and copies a file, if it should be copied.
Args:
file_to_copy: A dict in a file copy list constructed by
find_importable_tests, which represents one file to copy, including
the keys:
"src": Absolute path to the source location of the file.
"destination": File name of the destination file.
And possibly also the keys "reference_support_info" or "is_jstest".
dest_dir: Path to the directory where the file should be copied.
"""
source_path = self.filesystem.normpath(file_to_copy['src'])
dest_path = self.filesystem.join(dest_dir, file_to_copy['dest'])
if self.filesystem.isdir(source_path):
_log.error('%s refers to a directory', source_path)
return
if not self.filesystem.exists(source_path):
_log.error('%s not found. Possible error in the test.',
source_path)
return
if not self.filesystem.exists(self.filesystem.dirname(dest_path)):
if not self.import_in_place:
self.filesystem.maybe_make_directory(
self.filesystem.dirname(dest_path))
relpath = self.filesystem.relpath(dest_path, self.layout_tests_dir)
# FIXME: Maybe doing a file diff is in order here for existing files?
# In other words, there's no sense in overwriting identical files, but
# there's no harm in copying the identical thing.
_log.debug(' copying %s', relpath)
if not self.import_in_place:
self.filesystem.copyfile(source_path, dest_path)
if self.filesystem.read_binary_file(source_path)[:2] == '#!':
self.filesystem.make_executable(dest_path)
class TryFlag(object):
def __init__(self, argv, host, git_cl):
self._args = parse_args(argv)
self._host = host
self._git_cl = git_cl
self._expectations_model = TestExpectationsModel()
self._test_configuration_converter = TestConfigurationConverter(
set(BUILDER_CONFIGS.values()))
self._filesystem = self._host.filesystem
self._path_finder = PathFinder(self._filesystem)
self._git = self._host.git()
def _force_flag_for_test_runner(self):
flag = self._args.flag
path = self._path_finder.path_from_web_tests(FLAG_FILE)
self._filesystem.write_text_file(path, flag + '\n')
self._git.add_list([path])
self._git.commit_locally_with_message(
'Flag try job: force %s for run_web_tests.py.' % flag)
def _flag_expectations_path(self):
return self._path_finder.path_from_web_tests(
'FlagExpectations', self._args.flag.lstrip('-'))
def _clear_expectations(self):
path = self._flag_expectations_path()
self._filesystem.write_text_file(path, '')
self._git.add_list([path])
self._git.commit_locally_with_message(
'Flag try job: clear expectations for %s.' % self._args.flag)
def _tests_in_flag_expectations(self):
result = set()
path = self._flag_expectations_path()
for line in self._filesystem.read_text_file(path).split('\n'):
expectation_line = TestExpectationLine.tokenize_line(path, line, 0)
test_name = expectation_line.name
if test_name:
result.add(test_name)
return result
def trigger(self):
self._force_flag_for_test_runner()
if self._args.regenerate:
self._clear_expectations()
self._git_cl.run([
'upload', '--bypass-hooks', '-f', '-m',
'Flag try job for %s.' % self._args.flag
])
for builder in sorted(BUILDER_BUCKETS):
bucket = BUILDER_BUCKETS[builder]
self._git_cl.trigger_try_jobs([builder], bucket)
def _create_expectation_line(self, result, test_configuration):
test_name = result.test_name()
line = TestExpectationLine()
line.name = test_name
line.path = test_name
line.matching_tests = [test_name]
line.filename = ''
if self._args.bug:
line.bugs = ['crbug.com/%s' % self._args.bug]
else:
line.bugs = ['Bug(none)']
line.expectations = result.actual_results().split()
line.parsed_expectations = [
TestExpectations.expectation_from_string(expectation)
for expectation in line.expectations
]
line.specifiers = [test_configuration.version]
line.matching_configurations = set([test_configuration])
return line
def _process_result(self, build, result):
if not result.did_run_as_expected():
self._expectations_model.add_expectation_line(
self._create_expectation_line(
result, BUILDER_CONFIGS[build.builder_name]),
model_all_expectations=True)
def update(self):
self._host.print_('Fetching results...')
# TODO: Get jobs from the _tryflag branch. Current branch for now.
jobs = self._git_cl.latest_try_jobs(BUILDER_CONFIGS.keys())
buildbot = self._host.buildbot
for build in sorted(jobs):
self._host.print_(
'-- %s: %s/results.html' %
(BUILDER_CONFIGS[build.builder_name].version,
buildbot.results_url(build.builder_name, build.build_number)))
results = buildbot.fetch_results(build, True)
results.for_each_test(
lambda result, b=build: self._process_result(b, result))
# TODO: Write to flag expectations file. For now, stdout. :)
unexpected_failures = []
unexpected_passes = []
tests_in_flag_expectations = self._tests_in_flag_expectations()
for line in self._expectations_model.all_lines():
is_pass = (TestExpectations.EXPECTATIONS['pass']
in line.parsed_expectations)
if not is_pass:
unexpected_failures.append(line)
elif line.name in tests_in_flag_expectations:
unexpected_passes.append(line)
self._print_all(unexpected_passes, 'unexpected passes')
self._print_all(unexpected_failures, 'unexpected failures')
def _print_all(self, lines, description):
self._host.print_('\n### %s %s:\n' % (len(lines), description))
for line in lines:
self._host.print_(
line.to_string(self._test_configuration_converter))
def run(self):
action = self._args.action
if action == 'trigger':
self.trigger()
elif action == 'update':
self.update()
else:
print >> self._host.stderr, 'specify "trigger" or "update"'
return 1
return 0
class ImportNotifier(object):
def __init__(self, host, chromium_git, local_wpt):
self.host = host
self.git = chromium_git
self.local_wpt = local_wpt
self.default_port = host.port_factory.get()
self.finder = PathFinder(host.filesystem)
self.owners_extractor = DirectoryOwnersExtractor(host.filesystem)
self.new_failures_by_directory = defaultdict(list)
def main(self,
wpt_revision_start,
wpt_revision_end,
rebaselined_tests,
test_expectations,
issue,
patchset,
dry_run=True,
service_account_key_json=None):
"""Files bug reports for new failures.
Args:
wpt_revision_start: The start of the imported WPT revision range
(exclusive), i.e. the last imported revision.
wpt_revision_end: The end of the imported WPT revision range
(inclusive), i.e. the current imported revision.
rebaselined_tests: A list of test names that have been rebaselined.
test_expectations: A dictionary mapping names of tests that cannot
be rebaselined to a list of new test expectation lines.
issue: The issue number of the import CL (a string).
patchset: The patchset number of the import CL (a string).
dry_run: If True, no bugs will be actually filed to crbug.com.
service_account_key_json: The path to a JSON private key of a
service account for accessing Monorail. If None, try to load
from the default location, i.e. the path stored in the
environment variable GOOGLE_APPLICATION_CREDENTIALS.
Note: "test names" are paths of the tests relative to LayoutTests.
"""
gerrit_url = SHORT_GERRIT_PREFIX + issue
gerrit_url_with_ps = gerrit_url + '/' + patchset + '/'
changed_test_baselines = self.find_changed_baselines_of_tests(
rebaselined_tests)
self.examine_baseline_changes(changed_test_baselines,
gerrit_url_with_ps)
self.examine_new_test_expectations(test_expectations)
bugs = self.create_bugs_from_new_failures(wpt_revision_start,
wpt_revision_end, gerrit_url)
self.file_bugs(bugs, dry_run, service_account_key_json)
def find_changed_baselines_of_tests(self, rebaselined_tests):
"""Finds the corresponding changed baselines of each test.
Args:
rebaselined_tests: A list of test names that have been rebaselined.
Returns:
A dictionary mapping test names to paths of their baselines changed
in this import CL (paths relative to the root of Chromium repo).
"""
test_baselines = {}
changed_files = self.git.changed_files()
for test_name in rebaselined_tests:
test_without_ext, _ = self.host.filesystem.splitext(test_name)
changed_baselines = []
# TODO(robertma): Refactor this into layout_tests.port.base.
baseline_name = test_without_ext + '-expected.txt'
for changed_file in changed_files:
if changed_file.endswith(baseline_name):
changed_baselines.append(changed_file)
if changed_baselines:
test_baselines[test_name] = changed_baselines
return test_baselines
def examine_baseline_changes(self, changed_test_baselines,
gerrit_url_with_ps):
"""Examines all changed baselines to find new failures.
Args:
changed_test_baselines: A dictionary mapping test names to paths of
changed baselines.
gerrit_url_with_ps: Gerrit URL of this CL with the patchset number.
"""
for test_name, changed_baselines in changed_test_baselines.iteritems():
directory = self.find_owned_directory(test_name)
if not directory:
_log.warning('Cannot find OWNERS of %s', test_name)
continue
for baseline in changed_baselines:
if self.more_failures_in_baseline(baseline):
self.new_failures_by_directory[directory].append(
TestFailure(TestFailure.BASELINE_CHANGE,
test_name,
baseline_path=baseline,
gerrit_url_with_ps=gerrit_url_with_ps))
def more_failures_in_baseline(self, baseline):
diff = self.git.run(['diff', '-U0', 'origin/master', '--', baseline])
delta_failures = 0
for line in diff.splitlines():
if line.startswith('+FAIL'):
delta_failures += 1
if line.startswith('-FAIL'):
delta_failures -= 1
return delta_failures > 0
def examine_new_test_expectations(self, test_expectations):
"""Examines new test expectations to find new failures.
Args:
test_expectations: A dictionary mapping names of tests that cannot
be rebaselined to a list of new test expectation lines.
"""
for test_name, expectation_lines in test_expectations.iteritems():
directory = self.find_owned_directory(test_name)
if not directory:
_log.warning('Cannot find OWNERS of %s', test_name)
continue
for expectation_line in expectation_lines:
self.new_failures_by_directory[directory].append(
TestFailure(TestFailure.NEW_EXPECTATION,
test_name,
expectation_line=expectation_line))
def create_bugs_from_new_failures(self, wpt_revision_start,
wpt_revision_end, gerrit_url):
"""Files bug reports for new failures.
Args:
wpt_revision_start: The start of the imported WPT revision range
(exclusive), i.e. the last imported revision.
wpt_revision_end: The end of the imported WPT revision range
(inclusive), i.e. the current imported revision.
gerrit_url: Gerrit URL of the CL.
Return:
A list of MonorailIssue objects that should be filed.
"""
imported_commits = self.local_wpt.commits_in_range(
wpt_revision_start, wpt_revision_end)
bugs = []
for directory, failures in self.new_failures_by_directory.iteritems():
summary = '[WPT] New failures introduced in {} by import {}'.format(
directory, gerrit_url)
full_directory = self.host.filesystem.join(
self.finder.layout_tests_dir(), directory)
owners_file = self.host.filesystem.join(full_directory, 'OWNERS')
is_wpt_notify_enabled = self.owners_extractor.is_wpt_notify_enabled(
owners_file)
owners = self.owners_extractor.extract_owners(owners_file)
# owners may be empty but not None.
cc = owners + ['*****@*****.**']
component = self.owners_extractor.extract_component(owners_file)
# component could be None.
components = [component] if component else None
prologue = ('WPT import {} introduced new failures in {}:\n\n'
'List of new failures:\n'.format(
gerrit_url, directory))
failure_list = ''
for failure in failures:
failure_list += str(failure) + '\n'
epilogue = '\nThis import contains upstream changes from {} to {}:\n'.format(
wpt_revision_start, wpt_revision_end)
commit_list = self.format_commit_list(imported_commits,
full_directory)
description = prologue + failure_list + epilogue + commit_list
bug = MonorailIssue.new_chromium_issue(summary, description, cc,
components)
_log.info(str(bug))
if is_wpt_notify_enabled:
_log.info(
"WPT-NOTIFY enabled in this directory; adding the bug to the pending list."
)
bugs.append(bug)
else:
_log.info(
"WPT-NOTIFY disabled in this directory; discarding the bug."
)
return bugs
def format_commit_list(self, imported_commits, directory):
"""Formats the list of imported WPT commits.
Imports affecting the given directory will be highlighted.
Args:
imported_commits: A list of (SHA, commit subject) pairs.
directory: An absolute path of a directory in the Chromium repo, for
which the list is formatted.
Returns:
A multi-line string.
"""
path_from_wpt = self.host.filesystem.relpath(
directory, self.finder.path_from_layout_tests('external', 'wpt'))
commit_list = ''
for sha, subject in imported_commits:
# subject is a Unicode string and can contain non-ASCII characters.
line = u'{}: {}'.format(subject, GITHUB_COMMIT_PREFIX + sha)
if self.local_wpt.is_commit_affecting_directory(
sha, path_from_wpt):
line += ' [affecting this directory]'
commit_list += line + '\n'
return commit_list
def find_owned_directory(self, test_name):
"""Finds the lowest directory that contains the test and has OWNERS.
Args:
The name of the test (a path relative to LayoutTests).
Returns:
The path of the found directory relative to LayoutTests.
"""
# Always use non-virtual test names when looking up OWNERS.
if self.default_port.lookup_virtual_test_base(test_name):
test_name = self.default_port.lookup_virtual_test_base(test_name)
# find_owners_file takes either a relative path from the *root* of the
# repository, or an absolute path.
abs_test_path = self.finder.path_from_layout_tests(test_name)
owners_file = self.owners_extractor.find_owners_file(
self.host.filesystem.dirname(abs_test_path))
if not owners_file:
return None
owned_directory = self.host.filesystem.dirname(owners_file)
short_directory = self.host.filesystem.relpath(
owned_directory, self.finder.layout_tests_dir())
return short_directory
def file_bugs(self, bugs, dry_run, service_account_key_json=None):
"""Files a list of bugs to Monorail.
Args:
bugs: A list of MonorailIssue objects.
dry_run: A boolean, whether we are in dry run mode.
service_account_key_json: Optional, see docs for main().
"""
# TODO(robertma): Better error handling in this method.
if dry_run:
_log.info(
'[dry_run] Would have filed the %d bugs in the pending list.',
len(bugs))
return
_log.info('Filing %d bugs in the pending list to Monorail', len(bugs))
api = self._get_monorail_api(service_account_key_json)
for index, bug in enumerate(bugs, start=1):
response = api.insert_issue(bug)
_log.info('[%d] Filed bug: %s', index,
MonorailIssue.crbug_link(response['id']))
def _get_monorail_api(self, service_account_key_json):
return MonorailAPI(service_account_key_json=service_account_key_json)
class TryFlag(object):
def __init__(self, argv, host, git_cl):
self._args = parse_args(argv)
self._host = host
self._git_cl = git_cl
self._expectations = []
self._filesystem = self._host.filesystem
self._path_finder = PathFinder(self._filesystem)
self._git = self._host.git()
def _force_flag_for_test_runner(self):
flag = self._args.flag
path = self._path_finder.path_from_web_tests(FLAG_FILE)
self._filesystem.write_text_file(path, flag + '\n')
self._git.add_list([path])
self._git.commit_locally_with_message(
'Flag try job: force %s for run_web_tests.py.' % flag)
def _flag_expectations_path(self):
return self._path_finder.path_from_web_tests(
'FlagExpectations', self._args.flag.lstrip('-'))
def _clear_expectations(self):
path = self._flag_expectations_path()
self._filesystem.write_text_file(path, '')
self._git.add_list([path])
self._git.commit_locally_with_message(
'Flag try job: clear expectations for %s.' % self._args.flag)
def _tests_in_flag_expectations(self):
path = self._flag_expectations_path()
content = self._filesystem.read_text_file(path)
test_expectations = TestExpectations()
test_expectations.parse_tagged_list(content)
return {test_name for test_name in test_expectations.individual_exps.keys()}
def trigger(self):
self._force_flag_for_test_runner()
if self._args.regenerate:
self._clear_expectations()
self._git_cl.run(['upload', '--bypass-hooks', '-f',
'-m', 'Flag try job for %s.' % self._args.flag])
for builder in sorted(BUILDER_BUCKETS):
bucket = BUILDER_BUCKETS[builder]
self._git_cl.trigger_try_jobs([builder], bucket)
def _create_expectation_line(self, result, test_configuration):
expected_results = set([res for res in result.actual_results().split()])
tag = test_configuration.version
reason = ''
if self._args.bug:
reason = 'crbug.com/' + self._args.bug
return Expectation(
test=result.test_name(), results=expected_results, tags=set([tag]), reason=reason)
def _process_result(self, build, result):
if not result.did_run_as_expected():
self._expectations.append(
self._create_expectation_line(
result, BUILDER_CONFIGS[build.builder_name]))
def update(self):
self._host.print_('Fetching results...')
# TODO: Get jobs from the _tryflag branch. Current branch for now.
jobs = self._git_cl.latest_try_jobs(builder_names=BUILDER_CONFIGS.keys())
results_fetcher = self._host.results_fetcher
for build in sorted(jobs):
self._host.print_('-- %s: %s/results.html' % (
BUILDER_CONFIGS[build.builder_name].version,
results_fetcher.results_url(build.builder_name, build.build_number)))
results = results_fetcher.fetch_results(build, True)
results.for_each_test(
lambda result, b=build: self._process_result(b, result))
# TODO: Write to flag expectations file. For now, stdout. :)
unexpected_failures = []
unexpected_passes = []
tests_in_flag_expectations = self._tests_in_flag_expectations()
for exp in self._expectations:
if ResultType.Pass not in exp.results:
unexpected_failures.append(exp)
elif exp.test in tests_in_flag_expectations:
unexpected_passes.append(exp)
unexpected_passes = sorted(unexpected_passes, key=lambda e: e.test)
unexpected_failures = sorted(unexpected_failures, key=lambda e: e.test)
self._print_all(unexpected_passes, 'unexpected passes')
self._print_all(unexpected_failures, 'unexpected failures')
def _print_all(self, exps, description):
self._host.print_('\n### %s %s:\n' % (len(exps), description))
for exp in exps:
self._host.print_(exp.to_string())
def run(self):
action = self._args.action
if action == 'trigger':
self.trigger()
elif action == 'update':
self.update()
else:
print >> self._host.stderr, 'specify "trigger" or "update"'
return 1
return 0
def test_chromium_base(self):
finder = PathFinder(MockFileSystem())
self.assertEqual(finder.chromium_base(), '/mock-checkout')
class DirectoryOwnersExtractor(object):
def __init__(self, host):
self.filesystem = host.filesystem
self.finder = PathFinder(self.filesystem)
self.executive = host.executive
self.owner_map = None
def list_owners(self, changed_files):
"""Looks up the owners for the given set of changed files.
Args:
changed_files: A list of file paths relative to the repository root.
Returns:
A dict mapping tuples of owner email addresses to lists of
owned directories (paths relative to the root of web tests).
"""
email_map = collections.defaultdict(set)
external_root_owners = self.finder.path_from_web_tests(
'external', 'OWNERS')
for relpath in changed_files:
# Try to find the first *non-empty* OWNERS file.
absolute_path = self.finder.path_from_chromium_base(relpath)
owners = None
owners_file = self.find_owners_file(absolute_path)
while owners_file:
owners = self.extract_owners(owners_file)
if owners:
break
# Found an empty OWNERS file. Try again from the parent directory.
absolute_path = self.filesystem.dirname(
self.filesystem.dirname(owners_file))
owners_file = self.find_owners_file(absolute_path)
# Skip web_tests/external/OWNERS.
if not owners or owners_file == external_root_owners:
continue
owned_directory = self.filesystem.dirname(owners_file)
owned_directory_relpath = self.filesystem.relpath(
owned_directory, self.finder.web_tests_dir())
email_map[tuple(owners)].add(owned_directory_relpath)
return {
owners: sorted(owned_directories)
for owners, owned_directories in email_map.iteritems()
}
def find_owners_file(self, start_path):
"""Finds the first enclosing OWNERS file for a given path.
Starting from the given path, walks up the directory tree until the
first OWNERS file is found or web_tests/external is reached.
Args:
start_path: A relative path from the root of the repository, or an
absolute path. The path can be a file or a directory.
Returns:
The absolute path to the first OWNERS file found; None if not found
or if start_path is outside of web_tests/external.
"""
abs_start_path = (start_path if self.filesystem.isabs(start_path) else
self.finder.path_from_chromium_base(start_path))
directory = (abs_start_path if self.filesystem.isdir(abs_start_path)
else self.filesystem.dirname(abs_start_path))
external_root = self.finder.path_from_web_tests('external')
if not directory.startswith(external_root):
return None
# Stop at web_tests, which is the parent of external_root.
while directory != self.finder.web_tests_dir():
owners_file = self.filesystem.join(directory, 'OWNERS')
if self.filesystem.isfile(
self.finder.path_from_chromium_base(owners_file)):
return owners_file
directory = self.filesystem.dirname(directory)
return None
def extract_owners(self, owners_file):
"""Extracts owners from an OWNERS file.
Args:
owners_file: An absolute path to an OWNERS file.
Returns:
A list of valid owners (email addresses).
"""
contents = self._read_text_file(owners_file)
email_regexp = re.compile(BASIC_EMAIL_REGEXP)
addresses = []
for line in contents.splitlines():
line = line.strip()
if email_regexp.match(line):
addresses.append(line)
return addresses
def extract_component(self, owners_file):
"""Extracts the component from an OWNERS file.
Args:
owners_file: An absolute path to an OWNERS file.
Returns:
A string, or None if not found.
"""
dir_metadata = self._read_dir_metadata(owners_file)
if dir_metadata and dir_metadata.component:
return dir_metadata.component
contents = self._read_text_file(owners_file)
search = re.search(COMPONENT_REGEXP, contents, re.MULTILINE)
if search:
return search.group(1)
return None
def is_wpt_notify_enabled(self, owners_file):
"""Checks if the OWNERS file enables WPT-NOTIFY.
Args:
owners_file: An absolute path to an OWNERS file.
Returns:
A boolean.
"""
dir_metadata = self._read_dir_metadata(owners_file)
if dir_metadata and dir_metadata.should_notify is not None:
return dir_metadata.should_notify
contents = self._read_text_file(owners_file)
return bool(re.search(WPT_NOTIFY_REGEXP, contents, re.MULTILINE))
@memoized
def _read_text_file(self, path):
return self.filesystem.read_text_file(path)
@memoized
def _read_dir_metadata(self, path):
"""Read the content from a path.
Args:
path: An absolute path.
Returns:
A WPTDirMetadata object, or None if not found.
"""
root_path = self.finder.web_tests_dir()
dir_path = self.filesystem.dirname(path)
# dirmd starts with an absolute directory path, `dir_path`, traverses all
# parent directories and stops at `root_path` to find the first available DIR_METADATA
# file. `root_path` is the web_tests directory.
json_data = self.executive.run_command([
self.finder.path_from_depot_tools_base('dirmd'), 'compute',
'-root', root_path, dir_path
])
try:
data = json.loads(json_data)
except ValueError:
return None
relative_path = self.filesystem.relpath(dir_path, root_path)
return WPTDirMetadata(data, relative_path)
