def test_changed_files_no_results(self):
rietveld = Rietveld(self.mock_web())
self.assertIsNone(rietveld.changed_files(11113333))
class RebaselineCL(AbstractParallelRebaselineCommand):
name = "rebaseline-cl"
help_text = "Fetches new baselines for a CL from test runs on try bots."
long_help = ("By default, this command will check the latest try job results "
"for all platforms, and start try jobs for platforms with no "
"try jobs. Then, new baselines are downloaded for any tests "
"that are being rebaselined. After downloading, the baselines "
"for different platforms will be optimized (consolidated).")
show_in_main_help = True
def __init__(self):
super(RebaselineCL, self).__init__(options=[
optparse.make_option(
'--issue', type='int', default=None,
help='Rietveld issue number; if none given, this will be obtained via `git cl issue`.'),
optparse.make_option(
'--dry-run', action='store_true', default=False,
help='Dry run mode; list actions that would be performed but do not do anything.'),
optparse.make_option(
'--only-changed-tests', action='store_true', default=False,
help='Only download new baselines for tests that are changed in the CL.'),
optparse.make_option(
'--no-trigger-jobs', dest='trigger_jobs', action='store_false', default=True,
help='Do not trigger any try jobs.'),
self.no_optimize_option,
self.results_directory_option,
])
self.rietveld = Rietveld(Web())
def execute(self, options, args, tool):
self._tool = tool
unstaged_baselines = self.unstaged_baselines()
if unstaged_baselines:
_log.error('Aborting: there are unstaged baselines:')
for path in unstaged_baselines:
_log.error(' %s', path)
return
issue_number = self._get_issue_number(options)
if not issue_number:
return
# TODO(qyearsley): Replace this with git cl try-results to remove
# dependency on Rietveld. See crbug.com/671684.
builds = self.rietveld.latest_try_jobs(issue_number, self._try_bots())
if options.trigger_jobs:
if self.trigger_jobs_for_missing_builds(builds):
_log.info('Please re-run webkit-patch rebaseline-cl once all pending try jobs have finished.')
return
if not builds:
_log.info('No builds to download baselines from.')
_log.debug('Getting results for Rietveld issue %d.', issue_number)
builds_to_results = self._fetch_results(builds)
if builds_to_results is None:
return
test_prefix_list = {}
if args:
for test in args:
test_prefix_list[test] = {b: BASELINE_SUFFIX_LIST for b in builds}
else:
test_prefix_list = self._test_prefix_list(
issue_number,
builds_to_results,
only_changed_tests=options.only_changed_tests)
self._log_test_prefix_list(test_prefix_list)
if options.dry_run:
return
self.rebaseline(options, test_prefix_list)
def _get_issue_number(self, options):
"""Gets the Rietveld CL number from either |options| or from the current local branch."""
if options.issue:
return options.issue
issue_number = self.git_cl().get_issue_number()
_log.debug('Issue number for current branch: %s', issue_number)
if not issue_number.isdigit():
_log.error('No issue number given and no issue for current branch. This tool requires a CL\n'
'to operate on; please run `git cl upload` on this branch first, or use the --issue\n'
'option to download baselines for another existing CL.')
return None
return int(issue_number)
def git_cl(self):
"""Returns a GitCL instance; can be overridden for tests."""
return GitCL(self._tool)
def trigger_jobs_for_missing_builds(self, builds):
"""Triggers try jobs for any builders that have no builds started.
Args:
builds: A list of Build objects; if the build number of a Build is None,
then that indicates that the job is pending.
Returns:
True if there are pending jobs to wait for, including jobs just started.
"""
builders_with_builds = {b.builder_name for b in builds}
builders_without_builds = set(self._try_bots()) - builders_with_builds
builders_with_pending_builds = {b.builder_name for b in builds if b.build_number is None}
if builders_with_pending_builds:
_log.info('There are existing pending builds for:')
for builder in sorted(builders_with_pending_builds):
_log.info(' %s', builder)
if builders_without_builds:
_log.info('Triggering try jobs for:')
command = ['try']
for builder in sorted(builders_without_builds):
_log.info(' %s', builder)
command.extend(['-b', builder])
self.git_cl().run(command)
return bool(builders_with_pending_builds or builders_without_builds)
def _try_bots(self):
"""Returns a collection of try bot builders to fetch results for."""
return self._tool.builders.all_try_builder_names()
def _fetch_results(self, builds):
"""Fetches results for each build.
There should be a one-to-one correspondence between Builds, supported
platforms, and try bots. If not all of the builds can be fetched, then
continuing with rebaselining may yield incorrect results, when the new
baselines are deduped, an old baseline may be kept for the platform
that's missing results.
Returns:
A dict mapping Build to LayoutTestResults, or None if any results
were not available.
"""
buildbot = self._tool.buildbot
results = {}
for build in builds:
results_url = buildbot.results_url(build.builder_name, build.build_number)
layout_test_results = buildbot.fetch_results(build)
if layout_test_results is None:
_log.error(
'Failed to fetch results from "%s".\n'
'Try starting a new job for %s by running :\n'
' git cl try -b %s',
results_url, build.builder_name, build.builder_name)
return None
results[build] = layout_test_results
return results
def _test_prefix_list(self, issue_number, builds_to_results, only_changed_tests):
"""Returns a collection of tests, builders and file extensions to get new baselines for.
Args:
issue_number: The CL number of the change which needs new baselines.
builds_to_results: A dict mapping Builds to LayoutTestResults.
only_changed_tests: Whether to only include baselines for tests that
are changed in this CL. If False, all new baselines for failing
tests will be downloaded, even for tests that were not modified.
Returns:
A dict containing information about which new baselines to download.
"""
builds_to_tests = {}
for build, results in builds_to_results.iteritems():
builds_to_tests[build] = self._tests_to_rebaseline(build, results)
if only_changed_tests:
files_in_cl = self.rietveld.changed_files(issue_number)
# Note, in the changed files list from Rietveld, paths always
# use / as the separator, and they're always relative to repo root.
# TODO(qyearsley): Do this without using a hard-coded constant.
test_base = 'third_party/WebKit/LayoutTests/'
tests_in_cl = [f[len(test_base):] for f in files_in_cl if f.startswith(test_base)]
result = {}
for build, tests in builds_to_tests.iteritems():
for test in tests:
if only_changed_tests and test not in tests_in_cl:
continue
if test not in result:
result[test] = {}
result[test][build] = BASELINE_SUFFIX_LIST
return result
def _tests_to_rebaseline(self, build, layout_test_results):
"""Fetches a list of tests that should be rebaselined for some build ."""
unexpected_results = layout_test_results.didnt_run_as_expected_results()
tests = sorted(r.test_name() for r in unexpected_results
if r.is_missing_baseline() or r.has_mismatch_result())
new_failures = self._fetch_tests_with_new_failures(build)
if new_failures is None:
_log.warning('No retry summary available for build %s.', build)
else:
tests = [t for t in tests if t in new_failures]
return tests
def _fetch_tests_with_new_failures(self, build):
"""Fetches a list of tests that failed with a patch in a given try job but not without."""
buildbot = self._tool.buildbot
content = buildbot.fetch_retry_summary_json(build)
if content is None:
return None
try:
retry_summary = json.loads(content)
return retry_summary['failures']
except (ValueError, KeyError):
_log.warning('Unexpected retry summary content:\n%s', content)
return None
@staticmethod
def _log_test_prefix_list(test_prefix_list):
"""Logs the tests to download new baselines for."""
if not test_prefix_list:
_log.info('No tests to rebaseline; exiting.')
return
_log.debug('Tests to rebaseline:')
for test, builds in test_prefix_list.iteritems():
_log.debug(' %s:', test)
for build in sorted(builds):
_log.debug(' %s', build)
def test_changed_files(self):
rietveld = Rietveld(self.mock_web())
self.assertEqual(
rietveld.changed_files(11112222),
['some/path/bar.html', 'some/path/foo.cc'])
class RebaselineCL(AbstractParallelRebaselineCommand):
name = "rebaseline-cl"
help_text = "Fetches new baselines for a CL from test runs on try bots."
long_help = ("By default, this command will check the latest try job results "
"for all platforms, and start try jobs for platforms with no "
"try jobs. Then, new baselines are downloaded for any tests "
"that are being rebaselined. After downloading, the baselines "
"for different platforms will be optimized (consolidated).")
show_in_main_help = True
def __init__(self):
super(RebaselineCL, self).__init__(options=[
optparse.make_option(
'--issue', type='int', default=None,
help='Rietveld issue number; if none given, this will be obtained via `git cl issue`.'),
optparse.make_option(
'--dry-run', action='store_true', default=False,
help='Dry run mode; list actions that would be performed but do not do anything.'),
optparse.make_option(
'--only-changed-tests', action='store_true', default=False,
help='Only download new baselines for tests that are changed in the CL.'),
optparse.make_option(
'--no-trigger-jobs', dest='trigger_jobs', action='store_false', default=True,
help='Do not trigger any try jobs.'),
self.no_optimize_option,
self.results_directory_option,
])
self.rietveld = Rietveld(Web())
def execute(self, options, args, tool):
self._tool = tool
issue_number = self._get_issue_number(options)
if not issue_number:
return
builds = self.rietveld.latest_try_jobs(issue_number, self._try_bots())
if options.trigger_jobs:
if self.trigger_jobs_for_missing_builds(builds):
_log.info('Please re-run webkit-patch rebaseline-cl once all pending try jobs have finished.')
return
if not builds:
_log.info('No builds to download baselines from.')
if args:
test_prefix_list = {}
for test in args:
test_prefix_list[test] = {b: BASELINE_SUFFIX_LIST for b in builds}
else:
test_prefix_list = self._test_prefix_list(
issue_number, only_changed_tests=options.only_changed_tests)
# TODO(qyearsley): Fix places where non-existing tests may be added:
# 1. Make sure that the tests obtained when passing --only-changed-tests include only existing tests.
test_prefix_list = self._filter_existing(test_prefix_list)
self._log_test_prefix_list(test_prefix_list)
if options.dry_run:
return
self.rebaseline(options, test_prefix_list)
def _filter_existing(self, test_prefix_list):
"""Filters out entries in |test_prefix_list| for tests that don't exist."""
new_test_prefix_list = {}
port = self._tool.port_factory.get()
for test in test_prefix_list:
path = port.abspath_for_test(test)
if self._tool.filesystem.exists(path):
new_test_prefix_list[test] = test_prefix_list[test]
else:
_log.warning('%s not found, removing from list.', path)
return new_test_prefix_list
def _get_issue_number(self, options):
"""Gets the Rietveld CL number from either |options| or from the current local branch."""
if options.issue:
return options.issue
issue_number = self.git_cl().get_issue_number()
_log.debug('Issue number for current branch: %s', issue_number)
if not issue_number.isdigit():
_log.error('No issue number given and no issue for current branch. This tool requires a CL\n'
'to operate on; please run `git cl upload` on this branch first, or use the --issue\n'
'option to download baselines for another existing CL.')
return None
return int(issue_number)
def git_cl(self):
"""Returns a GitCL instance; can be overridden for tests."""
return GitCL(self._tool)
def trigger_jobs_for_missing_builds(self, builds):
"""Triggers try jobs for any builders that have no builds started.
Args:
builds: A list of Build objects; if the build number of a Build is None,
then that indicates that the job is pending.
Returns:
True if there are pending jobs to wait for, including jobs just started.
"""
builders_with_builds = {b.builder_name for b in builds}
builders_without_builds = set(self._try_bots()) - builders_with_builds
builders_with_pending_builds = {b.builder_name for b in builds if b.build_number is None}
if builders_with_pending_builds:
_log.info('There are existing pending builds for:')
for builder in sorted(builders_with_pending_builds):
_log.info(' %s', builder)
if builders_without_builds:
_log.info('Triggering try jobs for:')
command = ['try']
for builder in sorted(builders_without_builds):
_log.info(' %s', builder)
command.extend(['-b', builder])
self.git_cl().run(command)
return bool(builders_with_pending_builds or builders_without_builds)
def _test_prefix_list(self, issue_number, only_changed_tests):
"""Returns a collection of test, builder and file extensions to get new baselines for.
Args:
issue_number: The CL number of the change which needs new baselines.
only_changed_tests: Whether to only include baselines for tests that
are changed in this CL. If False, all new baselines for failing
tests will be downloaded, even for tests that were not modified.
Returns:
A dict containing information about which new baselines to download.
"""
builds_to_tests = self._builds_to_tests(issue_number)
if only_changed_tests:
files_in_cl = self.rietveld.changed_files(issue_number)
# Note, in the changed files list from Rietveld, paths always
# use / as the separator, and they're always relative to repo root.
# TODO(qyearsley): Do this without using a hard-coded constant.
test_base = 'third_party/WebKit/LayoutTests/'
tests_in_cl = [f[len(test_base):] for f in files_in_cl if f.startswith(test_base)]
result = {}
for build, tests in builds_to_tests.iteritems():
for test in tests:
if only_changed_tests and test not in tests_in_cl:
continue
if test not in result:
result[test] = {}
result[test][build] = BASELINE_SUFFIX_LIST
return result
def _builds_to_tests(self, issue_number):
"""Fetches a list of try bots, and for each, fetches tests with new baselines."""
_log.debug('Getting results for Rietveld issue %d.', issue_number)
builds = self.rietveld.latest_try_jobs(issue_number, self._try_bots())
if not builds:
_log.debug('No try job results for builders in: %r.', self._try_bots())
return {build: self._tests_to_rebaseline(build) for build in builds}
def _try_bots(self):
"""Returns a collection of try bot builders to fetch results for."""
return self._tool.builders.all_try_builder_names()
def _tests_to_rebaseline(self, build):
"""Fetches a list of tests that should be rebaselined."""
buildbot = self._tool.buildbot
results_url = buildbot.results_url(build.builder_name, build.build_number)
layout_test_results = buildbot.fetch_layout_test_results(results_url)
if layout_test_results is None:
_log.warning('Failed to request layout test results from "%s".', results_url)
return []
unexpected_results = layout_test_results.didnt_run_as_expected_results()
tests = sorted(r.test_name() for r in unexpected_results
if r.is_missing_baseline() or r.has_mismatch_result())
new_failures = self._fetch_tests_with_new_failures(build)
if new_failures is None:
_log.warning('No retry summary available for build %s.', build)
else:
tests = [t for t in tests if t in new_failures]
return tests
def _fetch_tests_with_new_failures(self, build):
"""Fetches a list of tests that failed with a patch in a given try job but not without."""
buildbot = self._tool.buildbot
content = buildbot.fetch_retry_summary_json(build)
if content is None:
return None
try:
retry_summary = json.loads(content)
return retry_summary['failures']
except (ValueError, KeyError):
_log.warning('Unexepected retry summary content:\n%s', content)
return None
@staticmethod
def _log_test_prefix_list(test_prefix_list):
"""Logs the tests to download new baselines for."""
if not test_prefix_list:
_log.info('No tests to rebaseline; exiting.')
return
_log.debug('Tests to rebaseline:')
for test, builds in test_prefix_list.iteritems():
_log.debug(' %s:', test)
for build in sorted(builds):
_log.debug(' %s', build)
def test_changed_files_no_results(self):
rietveld = Rietveld(self.mock_web())
self.assertIsNone(rietveld.changed_files(11113333))
def test_changed_files(self):
rietveld = Rietveld(self.mock_web())
self.assertEqual(rietveld.changed_files(11112222),
['some/path/bar.html', 'some/path/foo.cc'])
class RebaselineCL(AbstractParallelRebaselineCommand):
name = "rebaseline-cl"
help_text = "Fetches new baselines for a CL from test runs on try bots."
long_help = ("By default, this command will check the latest try job results "
"for all platforms, and start try jobs for platforms with no "
"try jobs. Then, new baselines are downloaded for any tests "
"that are being rebaselined. After downloading, the baselines "
"for different platforms will be optimized (consolidated).")
show_in_main_help = True
def __init__(self):
super(RebaselineCL, self).__init__(options=[
optparse.make_option(
'--issue', type='int', default=None,
help='Rietveld issue number; if none given, this will be obtained via `git cl issue`.'),
optparse.make_option(
'--dry-run', action='store_true', default=False,
help='Dry run mode; list actions that would be performed but do not do anything.'),
optparse.make_option(
'--only-changed-tests', action='store_true', default=False,
help='Only download new baselines for tests that are changed in the CL.'),
optparse.make_option(
'--no-trigger-jobs', dest='trigger_jobs', action='store_false', default=True,
help='Do not trigger any try jobs.'),
self.no_optimize_option,
self.results_directory_option,
])
self.rietveld = Rietveld(Web())
def execute(self, options, args, tool):
self._tool = tool
issue_number = self._get_issue_number(options)
if not issue_number:
return
builds = self.rietveld.latest_try_jobs(issue_number, self._try_bots())
if options.trigger_jobs:
self.trigger_jobs_for_missing_builds(builds)
if not builds:
# TODO(qyearsley): Also check that there are *finished* builds.
# The current behavior would still proceed if there are queued
# or started builds.
_log.info('No builds to download baselines from.')
if args:
test_prefix_list = {}
for test in args:
test_prefix_list[test] = {b: BASELINE_SUFFIX_LIST for b in builds}
else:
test_prefix_list = self._test_prefix_list(
issue_number, only_changed_tests=options.only_changed_tests)
# TODO(qyearsley): Fix places where non-existing tests may be added:
# 1. Make sure that the tests obtained when passing --only-changed-tests include only existing tests.
# 2. Make sure that update-w3c-test-expectations doesn't specify non-existing tests (http://crbug.com/649691).
test_prefix_list = self._filter_existing(test_prefix_list)
self._log_test_prefix_list(test_prefix_list)
if options.dry_run:
return
# NOTE(qyearsley): If this is changed to stage all new files with git,
# e.g. if update_scm is not False, then update_w3c_test_expectations.py
# should be changed to not call git add --all.
self.rebaseline(options, test_prefix_list, update_scm=False)
def _filter_existing(self, test_prefix_list):
"""Filters out entries in |test_prefix_list| for tests that don't exist."""
new_test_prefix_list = {}
port = self._tool.port_factory.get()
for test in test_prefix_list:
path = port.abspath_for_test(test)
if self._tool.filesystem.exists(path):
new_test_prefix_list[test] = test_prefix_list[test]
else:
_log.warning('%s not found, removing from list.', path)
return new_test_prefix_list
def _get_issue_number(self, options):
"""Gets the Rietveld CL number from either |options| or from the current local branch."""
if options.issue:
return options.issue
issue_number = self.git_cl().get_issue_number()
_log.debug('Issue number for current branch: %s', issue_number)
if not issue_number.isdigit():
_log.error('No issue number given and no issue for current branch. This tool requires a CL\n'
'to operate on; please run `git cl upload` on this branch first, or use the --issue\n'
'option to download baselines for another existing CL.')
return None
return int(issue_number)
def git_cl(self):
"""Returns a GitCL instance; can be overridden for tests."""
return GitCL(self._tool)
def trigger_jobs_for_missing_builds(self, builds):
builders_with_builds = {b.builder_name for b in builds}
builders_without_builds = set(self._try_bots()) - builders_with_builds
if not builders_without_builds:
return
_log.info('Triggering try jobs for:')
for builder in sorted(builders_without_builds):
_log.info(' %s', builder)
# If the builders may be under different masters, then they cannot
# all be started in one invocation of git cl try without providing
# master names. Doing separate invocations is slower, but always works
# even when there are builders under different master names.
for builder in sorted(builders_without_builds):
self.git_cl().run(['try', '-b', builder])
def _test_prefix_list(self, issue_number, only_changed_tests):
"""Returns a collection of test, builder and file extensions to get new baselines for.
Args:
issue_number: The CL number of the change which needs new baselines.
only_changed_tests: Whether to only include baselines for tests that
are changed in this CL. If False, all new baselines for failing
tests will be downloaded, even for tests that were not modified.
Returns:
A dict containing information about which new baselines to download.
"""
builds_to_tests = self._builds_to_tests(issue_number)
if only_changed_tests:
files_in_cl = self.rietveld.changed_files(issue_number)
finder = WebKitFinder(self._tool.filesystem)
tests_in_cl = [finder.layout_test_name(f) for f in files_in_cl]
result = {}
for build, tests in builds_to_tests.iteritems():
for test in tests:
if only_changed_tests and test not in tests_in_cl:
continue
if test not in result:
result[test] = {}
result[test][build] = BASELINE_SUFFIX_LIST
return result
def _builds_to_tests(self, issue_number):
"""Fetches a list of try bots, and for each, fetches tests with new baselines."""
_log.debug('Getting results for Rietveld issue %d.', issue_number)
try_jobs = self.rietveld.latest_try_jobs(issue_number, self._try_bots())
if not try_jobs:
_log.debug('No try job results for builders in: %r.', self._try_bots())
builds_to_tests = {}
for job in try_jobs:
test_results = self._unexpected_mismatch_results(job)
build = Build(job.builder_name, job.build_number)
builds_to_tests[build] = sorted(r.test_name() for r in test_results)
return builds_to_tests
def _try_bots(self):
"""Returns a collection of try bot builders to fetch results for."""
return self._tool.builders.all_try_builder_names()
def _unexpected_mismatch_results(self, try_job):
"""Fetches a list of LayoutTestResult objects for unexpected results with new baselines."""
buildbot = self._tool.buildbot
results_url = buildbot.results_url(try_job.builder_name, try_job.build_number)
layout_test_results = buildbot.fetch_layout_test_results(results_url)
if layout_test_results is None:
_log.warning('Failed to request layout test results from "%s".', results_url)
return []
return layout_test_results.unexpected_mismatch_results()
@staticmethod
def _log_test_prefix_list(test_prefix_list):
"""Logs the tests to download new baselines for."""
if not test_prefix_list:
_log.info('No tests to rebaseline; exiting.')
return
_log.info('Tests to rebaseline:')
for test, builds in test_prefix_list.iteritems():
builds_str = ', '.join(sorted('%s (%s)' % (b.builder_name, b.build_number) for b in builds))
_log.info(' %s: %s', test, builds_str)