def FormatAndPrintResults(self, bisect_results):
"""Prints the results from a bisection run in a readable format.
Also prints annotations creating buildbot step "Results".
Args:
bisect_results: BisectResult object containing results to be printed.
"""
if bisect_results.abort_reason:
self._PrintAbortResults(bisect_results.abort_reason)
return
if self.opts.output_buildbot_annotations:
bisect_utils.OutputAnnotationStepStart('Build Status Per Revision')
print
print 'Full results of bisection:'
for revision_state in bisect_results.state.GetRevisionStates():
build_status = revision_state.passed
if type(build_status) is bool:
if build_status:
build_status = 'Good'
else:
build_status = 'Bad'
print ' %20s %40s %s' % (revision_state.depot,
revision_state.revision, build_status)
print
if self.opts.output_buildbot_annotations:
bisect_utils.OutputAnnotationStepClosed()
# The perf dashboard scrapes the "results" step in order to comment on
# bugs. If you change this, please update the perf dashboard as well.
bisect_utils.OutputAnnotationStepStart('Results')
self._PrintBanner(bisect_results)
self._PrintWarnings(bisect_results.warnings)
if bisect_results.culprit_revisions and bisect_results.confidence:
for culprit in bisect_results.culprit_revisions:
cl, info, depot = culprit
self._PrintRevisionInfo(cl, info, depot)
if bisect_results.other_regressions:
self._PrintOtherRegressions(bisect_results.other_regressions)
self._PrintRetestResults(bisect_results)
self._PrintTestedCommitsTable(bisect_results.state.GetRevisionStates(),
bisect_results.first_working_revision,
bisect_results.last_broken_revision,
bisect_results.confidence,
final_step=True)
self._PrintStepTime(bisect_results.state.GetRevisionStates())
self._PrintReproSteps()
self._PrintThankYou()
if self.opts.output_buildbot_annotations:
bisect_utils.OutputAnnotationStepClosed()
def _PrintAbortResults(self, abort_reason):
if self.opts.output_buildbot_annotations:
bisect_utils.OutputAnnotationStepStart('Results')
print ABORT_REASON_TEMPLATE % {
'abort_reason': abort_reason,
'bug_id': self.opts.bug_id or 'NOT SPECIFIED',
'command': self.opts.command,
'metric': '/'.join(self.opts.metric),
'good_revision': self.opts.good_revision,
'bad_revision': self.opts.bad_revision,
}
self._PrintThankYou()
if self.opts.output_buildbot_annotations:
bisect_utils.OutputAnnotationStepClosed()
def _SetupAndRunPerformanceTest(config, path_to_file, path_to_goma):
"""Attempts to build and run the current revision with and without the
current patch, with the parameters passed in.
Args:
config: The config read from run-perf-test.cfg.
path_to_file: Path to the bisect-perf-regression.py script.
path_to_goma: Path to goma directory.
Returns:
0 on success, otherwise 1.
"""
try:
with Goma(path_to_goma) as goma:
config['use_goma'] = bool(path_to_goma)
_RunPerformanceTest(config, path_to_file)
return 0
except RuntimeError, e:
bisect_utils.OutputAnnotationStepClosed()
_OutputFailedResults('Error: %s' % e.message)
return 1
def _PrintAbortResults(self, abort_reason):
if self.opts.output_buildbot_annotations:
bisect_utils.OutputAnnotationStepStart('Results')
# Metric string in config is not split in case of return code mode.
if (self.opts.metric and
self.opts.bisect_mode != bisect_utils.BISECT_MODE_RETURN_CODE):
metric = '/'.join(self.opts.metric)
else:
metric = self.opts.metric
print ABORT_REASON_TEMPLATE % {
'abort_reason': abort_reason,
'bug_id': self.opts.bug_id or 'NOT SPECIFIED',
'command': self.opts.command,
'metric': metric,
'good_revision': self.opts.good_revision,
'bad_revision': self.opts.bad_revision,
}
self._PrintThankYou()
if self.opts.output_buildbot_annotations:
bisect_utils.OutputAnnotationStepClosed()
def _RunBisectionScript(config, working_directory, path_to_file, path_to_goma,
dry_run):
"""Attempts to execute src/tools/bisect-perf-regression.py with the parameters
passed in.
Args:
config: A dict containing the parameters to pass to the script.
working_directory: A working directory to provide to the
bisect-perf-regression.py script, where it will store it's own copy of
the depot.
path_to_file: Path to the bisect-perf-regression.py script.
path_to_goma: Path to goma directory.
dry_run: Do a dry run, skipping sync, build, and performance testing steps.
Returns:
0 on success, otherwise 1.
"""
bisect_utils.OutputAnnotationStepStart('Config')
print
for k, v in config.iteritems():
print ' %s : %s' % (k, v)
print
bisect_utils.OutputAnnotationStepClosed()
cmd = [
'python',
os.path.join(path_to_file, 'bisect-perf-regression.py'), '-c',
config['command'], '-g', config['good_revision'], '-b',
config['bad_revision'], '-m', config['metric'], '--working_directory',
working_directory, '--output_buildbot_annotations'
]
if config['repeat_count']:
cmd.extend(['-r', config['repeat_count']])
if config['truncate_percent']:
cmd.extend(['-t', config['truncate_percent']])
if config['max_time_minutes']:
cmd.extend(['--max_time_minutes', config['max_time_minutes']])
cmd.extend(['--build_preference', 'ninja'])
if '--browser=cros' in config['command']:
cmd.extend(['--target_platform', 'cros'])
if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]:
cmd.extend(['--cros_board', os.environ[CROS_BOARD_ENV]])
cmd.extend(['--cros_remote_ip', os.environ[CROS_IP_ENV]])
else:
print 'Error: Cros build selected, but BISECT_CROS_IP or'\
'BISECT_CROS_BOARD undefined.'
print
return 1
if 'android' in config['command']:
cmd.extend(['--target_platform', 'android'])
if path_to_goma:
cmd.append('--use_goma')
if dry_run:
cmd.extend([
'--debug_ignore_build', '--debug_ignore_sync',
'--debug_ignore_perf_test'
])
cmd = [str(c) for c in cmd]
with Goma(path_to_goma) as goma:
return_code = subprocess.call(cmd)
if return_code:
print 'Error: bisect-perf-regression.py returned with error %d' %\
return_code
print
return return_code
def _RunPerformanceTest(config, path_to_file):
# Bisect script expects to be run from src
os.chdir(os.path.join(path_to_file, '..'))
bisect_utils.OutputAnnotationStepStart('Building With Patch')
opts = _CreateBisectOptionsFromConfig(config)
b = bisect.BisectPerformanceMetrics(None, opts)
if bisect_utils.RunGClient(['runhooks']):
raise RuntimeError('Failed to run gclient runhooks')
if not b.BuildCurrentRevision('chromium'):
raise RuntimeError('Patched version failed to build.')
bisect_utils.OutputAnnotationStepClosed()
bisect_utils.OutputAnnotationStepStart('Running With Patch')
results_with_patch = b.RunPerformanceTestAndParseResults(
opts.command,
opts.metric,
reset_on_first_run=True,
results_label='Patch')
if results_with_patch[1]:
raise RuntimeError('Patched version failed to run performance test.')
bisect_utils.OutputAnnotationStepClosed()
bisect_utils.OutputAnnotationStepStart('Reverting Patch')
if bisect_utils.RunGClient(['revert']):
raise RuntimeError('Failed to run gclient runhooks')
bisect_utils.OutputAnnotationStepClosed()
bisect_utils.OutputAnnotationStepStart('Building Without Patch')
if bisect_utils.RunGClient(['runhooks']):
raise RuntimeError('Failed to run gclient runhooks')
if not b.BuildCurrentRevision('chromium'):
raise RuntimeError('Unpatched version failed to build.')
bisect_utils.OutputAnnotationStepClosed()
bisect_utils.OutputAnnotationStepStart('Running Without Patch')
results_without_patch = b.RunPerformanceTestAndParseResults(
opts.command,
opts.metric,
upload_on_last_run=True,
results_label='ToT')
if results_without_patch[1]:
raise RuntimeError('Unpatched version failed to run performance test.')
# Find the link to the cloud stored results file.
output = results_without_patch[2]
cloud_file_link = [
t for t in output.splitlines()
if 'storage.googleapis.com/chromium-telemetry/html-results/' in t
]
if cloud_file_link:
# What we're getting here is basically "View online at http://..." so parse
# out just the url portion.
cloud_file_link = cloud_file_link[0]
cloud_file_link = [
t for t in cloud_file_link.split(' ')
if 'storage.googleapis.com/chromium-telemetry/html-results/' in t
]
assert cloud_file_link, "Couldn't parse url from output."
cloud_file_link = cloud_file_link[0]
else:
cloud_file_link = ''
# Calculate the % difference in the means of the 2 runs.
percent_diff_in_means = (results_with_patch[0]['mean'] / max(
0.0001, results_without_patch[0]['mean'])) * 100.0 - 100.0
std_err = bisect.CalculatePooledStandardError(
[results_with_patch[0]['values'], results_without_patch[0]['values']])
bisect_utils.OutputAnnotationStepClosed()
bisect_utils.OutputAnnotationStepStart('Results - %.02f +- %0.02f delta' %
(percent_diff_in_means, std_err))
print ' %s %s %s' % (''.center(10, ' '), 'Mean'.center(
20, ' '), 'Std. Error'.center(20, ' '))
print ' %s %s %s' % (
'Patch'.center(10, ' '),
('%.02f' % results_with_patch[0]['mean']).center(20, ' '),
('%.02f' % results_with_patch[0]['std_err']).center(20, ' '))
print ' %s %s %s' % (
'No Patch'.center(10, ' '),
('%.02f' % results_without_patch[0]['mean']).center(20, ' '),
('%.02f' % results_without_patch[0]['std_err']).center(20, ' '))
if cloud_file_link:
bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link)
bisect_utils.OutputAnnotationStepClosed()
def _OutputFailedResults(text_to_print):
bisect_utils.OutputAnnotationStepStart('Results - Failed')
print
print text_to_print
print
bisect_utils.OutputAnnotationStepClosed()
def _RunBisectionScript(config, working_directory, path_to_file, path_to_goma,
path_to_extra_src, dry_run):
"""Attempts to execute src/tools/bisect-perf-regression.py with the parameters
passed in.
Args:
config: A dict containing the parameters to pass to the script.
working_directory: A working directory to provide to the
bisect-perf-regression.py script, where it will store it's own copy of
the depot.
path_to_file: Path to the bisect-perf-regression.py script.
path_to_goma: Path to goma directory.
path_to_extra_src: Path to extra source file.
dry_run: Do a dry run, skipping sync, build, and performance testing steps.
Returns:
0 on success, otherwise 1.
"""
bisect_utils.OutputAnnotationStepStart('Config')
print
for k, v in config.iteritems():
print ' %s : %s' % (k, v)
print
bisect_utils.OutputAnnotationStepClosed()
cmd = [
'python',
os.path.join(path_to_file, 'bisect-perf-regression.py'), '-c',
config['command'], '-g', config['good_revision'], '-b',
config['bad_revision'], '-m', config['metric'], '--working_directory',
working_directory, '--output_buildbot_annotations'
]
if config['repeat_count']:
cmd.extend(['-r', config['repeat_count']])
if config['truncate_percent']:
cmd.extend(['-t', config['truncate_percent']])
if config['max_time_minutes']:
cmd.extend(['--max_time_minutes', config['max_time_minutes']])
if config.has_key('bisect_mode'):
cmd.extend(['--bisect_mode', config['bisect_mode']])
cmd.extend(['--build_preference', 'ninja'])
if '--browser=cros' in config['command']:
cmd.extend(['--target_platform', 'cros'])
if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]:
cmd.extend(['--cros_board', os.environ[CROS_BOARD_ENV]])
cmd.extend(['--cros_remote_ip', os.environ[CROS_IP_ENV]])
else:
print 'Error: Cros build selected, but BISECT_CROS_IP or'\
'BISECT_CROS_BOARD undefined.'
print
return 1
if 'android' in config['command']:
if 'android-chrome' in config['command']:
cmd.extend(['--target_platform', 'android-chrome'])
else:
cmd.extend(['--target_platform', 'android'])
if path_to_goma:
# crbug.com/330900. For Windows XP platforms, GOMA service is not supported.
# Moreover we don't compile chrome when gs_bucket flag is set instead
# use builds archives, therefore ignore GOMA service for Windows XP.
if config.get('gs_bucket') and platform.release() == 'XP':
print(
'Goma doesn\'t have a win32 binary, therefore it is not supported '
'on Windows XP platform. Please refer to crbug.com/330900.')
path_to_goma = None
cmd.append('--use_goma')
if path_to_extra_src:
cmd.extend(['--extra_src', path_to_extra_src])
# These flags are used to download build archives from cloud storage if
# available, otherwise will post a try_job_http request to build it on
# tryserver.
if config.get('gs_bucket'):
if config.get('builder_host') and config.get('builder_port'):
cmd.extend([
'--gs_bucket', config['gs_bucket'], '--builder_host',
config['builder_host'], '--builder_port',
config['builder_port']
])
else:
print(
'Error: Specified gs_bucket, but missing builder_host or '
'builder_port information in config.')
return 1
if dry_run:
cmd.extend([
'--debug_ignore_build', '--debug_ignore_sync',
'--debug_ignore_perf_test'
])
cmd = [str(c) for c in cmd]
with Goma(path_to_goma) as goma:
return_code = subprocess.call(cmd)
if return_code:
print 'Error: bisect-perf-regression.py returned with error %d' %\
return_code
print
return return_code
