def STEP_LOG_END_PERF(self, log_label, perf_dashboard_name):
# Support: @@@STEP_LOG_END_PERF@<label>@<line>@@@
# (finalizes log to step, marks it as being a perf step
# requiring logs to be stored on the master)
current_logs = self.cursor['annotated_logs']
log_text = '\n'.join(current_logs.get(log_label, [])) + '\n'
report_link = None
output_dir = None
if self.perf_id:
report_link, output_dir, _ = self._PerfStepMappings(
self.show_perf, self.perf_id, perf_dashboard_name,
self.perf_report_url_suffix)
PERF_EXPECTATIONS_PATH = ('../../scripts/master/log_parser/'
'perf_expectations/')
perf_output_dir = None
if output_dir:
output_dir = chromium_utils.AbsoluteCanonicalPath(output_dir)
perf_output_dir = chromium_utils.AbsoluteCanonicalPath(output_dir,
PERF_EXPECTATIONS_PATH)
if report_link and output_dir:
MakeOutputDirectory(output_dir)
if log_label == self.GRAPH_LIST:
self._SaveGraphInfo(log_text, output_dir)
else:
Prepend(log_label, log_text, output_dir, perf_output_dir)
def setUp(self):
# Must use chromium_utils.AbsoluteCanonicalPath to avoid getting different
# paths on Mac in the implementation of chromium_step.Prepend().
self.output_dir = chromium_utils.AbsoluteCanonicalPath(
tempfile.gettempdir())
self.perf_output_dir = chromium_utils.AbsoluteCanonicalPath(
os.path.join(tempfile.gettempdir(), 'perf'))
def _canonical_file(self, filename, ignore_basedir=False):
"""Returns an absolute path for a config or log file."""
if ignore_basedir:
full_filename = filename
else:
full_filename = os.path.join(self.basedir, filename)
return chromium_utils.AbsoluteCanonicalPath(full_filename)
def main():
parser = argparse.ArgumentParser(
description='Run a REINDEX TABLE command on postgres.')
parser.add_argument('directory', help='location of the master to reindex.')
parser.add_argument('--dbconfig-filename',
default='.dbconfig',
help='name of the dbconfig, defaults to %(default)s.')
parser.add_argument(
'--prod',
action='store_true',
help='actually execute command instead of just displaying it.')
args = parser.parse_args()
filename = chromium_utils.AbsoluteCanonicalPath(
os.path.join(args.directory, args.dbconfig_filename))
dbconfig = get_database_creds(filename)
cmd = [
'psql', '-h', 'localhost', '-U', dbconfig['username'], '-d',
dbconfig['dbname'], '-c', 'REINDEX TABLE buildrequests;'
]
new_env = os.environ.copy()
new_env['PGPASSWORD'] = dbconfig['password']
if args.prod:
return chromium_utils.RunCommand(cmd, env=new_env)
else:
print 'Would have run %s.' % cmd
print 'If this looks good, re-run with --prod.'
return 0
def Prepend(filename, data, output_dir, perf_output_dir):
READABLE_FILE_PERMISSIONS = int('644', 8)
fullfn = chromium_utils.AbsoluteCanonicalPath(output_dir, filename)
# This whitelists writing to files only directly under output_dir
# or perf_expectations_dir for security reasons.
if os.path.dirname(fullfn) != output_dir and (os.path.dirname(fullfn) !=
perf_output_dir):
raise Exception(
'Attempted to write to log file outside of \'%s\' or '
'\'%s\': \'%s\'' %
(output_dir, perf_output_dir, os.path.join(output_dir, filename)))
chromium_utils.Prepend(fullfn, data)
os.chmod(fullfn, READABLE_FILE_PERMISSIONS)
def ReplaceCoveragePath(cov_file, build_dir):
"""Replace the coverage path to this system 'src' directory."""
src_dir = chromium_utils.AbsoluteCanonicalPath(
os.path.join(build_dir, '..'))
src_dir = os.path.normpath(src_dir)
input_file = open(cov_file)
cov_lines = input_file.readlines()
input_file.close()
fhandler = open(cov_file, 'w')
for line in cov_lines:
line = re.sub(r'SF:.*[\\/]src[\\/]', 'SF:%s/' % src_dir, line, 1)
line = line.replace('\\', '/')
fhandler.write(line)
fhandler.close()
def add_perf_step(self, params, perf_file, perf_base_url, perf_output_dir):
"""Adds step for uploading perf results using the given file.
Args:
params: Extra parameters for cbuildbot.
perf_file: Name of the perf file to upload. Note the name of this file
will be used as the testname and params[0] will be used as the platform
name.
perf_base_url: If set, base url to build into references.
perf_output_dir: If set, where the perf files are to update.
"""
# Name of platform is always taken as the first param.
platform = params.split()[0]
# Name of the test is based off the name of the file.
test = os.path.splitext(perf_file)[0]
# Assuming all perf files will be stored in the cbuildbot log directory.
perf_file_path = os.path.join(self.buildroot, 'cbuildbot_logs',
perf_file)
if not perf_base_url:
perf_base_url = config.Master.perf_base_url
if not perf_output_dir:
perf_output_dir = config.Master.perf_output_dir
report_link = '/'.join([
perf_base_url, platform, test, config.Master.perf_report_url_suffix
])
output_dir = chromium_utils.AbsoluteCanonicalPath('/'.join(
[perf_output_dir, platform, test]))
cmd = ['cat', perf_file_path]
# Hmm - I wonder how dry_run should affect this.
perf_class = commands.CreatePerformanceStepClass(
process_log.GraphingLogProcessor,
report_link=report_link,
output_dir=output_dir,
factory_properties={},
perf_name=platform,
test_name=test)
self.f_cbuild.addStep(perf_class,
command=cmd,
name='Upload Perf Results',
description='upload_perf_results')
def handleOutputLine(self, line):
"""This is called once with each line of the test log."""
# Handle initial setup here, as step_status might not exist yet at init.
self.initialSection()
# All annotator directives start with @.
if not line.startswith('@'):
return
# Add \n if not there, which seems to be the case for log lines from
# windows agents, but not others.
if not line.endswith('\n'):
line += '\n'
# Support: @@@STEP_LOG_LINE@<label>@<line>@@@ (add log to step)
# Appends a line to the log's array. When STEP_LOG_END is called,
# that will finalize the log and call addCompleteLog().
m = annotator.Match.log_line(line)
if m:
log_label = m[0]
log_line = m[1]
current_logs = self.cursor['annotated_logs']
current_logs[log_label] = current_logs.get(log_label,
[]) + [log_line]
# Support: @@@STEP_LOG_END@<label>@@@ (finalizes log to step)
m = annotator.Match.log_end(line)
if m:
log_label = m[0]
current_logs = self.cursor['annotated_logs']
log_text = '\n'.join(current_logs.get(log_label, []))
addLogToStep(self.cursor['step'], log_label, log_text)
# Support: @@@STEP_LOG_END_PERF@<label>@<line>@@@
# (finalizes log to step, marks it as being a perf step
# requiring logs to be stored on the master)
m = annotator.Match.log_end_perf(line)
if m:
log_label = m[0]
perf_dashboard_name = m[1]
current_logs = self.cursor['annotated_logs']
log_text = '\n'.join(current_logs.get(log_label, [])) + '\n'
report_link = None
output_dir = None
if self.perf_id:
report_link, output_dir, _ = self._PerfStepMappings(
self.show_perf, self.perf_id, perf_dashboard_name,
self.perf_report_url_suffix)
if report_link:
# It's harmless to send the results URL more than once, but it
# clutters up the logs.
if 'results' not in (x for x, _ in self.cursor['links']):
self.addLinkToCursor('results', report_link)
PERF_EXPECTATIONS_PATH = ('../../scripts/master/log_parser/'
'perf_expectations/')
perf_output_dir = None
if output_dir:
output_dir = chromium_utils.AbsoluteCanonicalPath(output_dir)
perf_output_dir = chromium_utils.AbsoluteCanonicalPath(
output_dir, PERF_EXPECTATIONS_PATH)
if report_link and output_dir:
MakeOutputDirectory(output_dir)
if log_label == self.GRAPH_LIST:
self._SaveGraphInfo(log_text, output_dir)
else:
Prepend(log_label, log_text, output_dir, perf_output_dir)
# Support: @@@STEP_LINK@<name>@<url>@@@ (emit link)
# Also support depreceated @@@link@<name>@<url>@@@
m = annotator.Match.step_link(line)
if m:
link_label = m[0]
link_url = m[1]
self.addLinkToCursor(link_label, link_url)
# Support: @@@STEP_STARTED@@@ (start a step at cursor)
if annotator.Match.step_started(line):
self.startStep(self.cursor)
# Support: @@@STEP_CLOSED@@@
if annotator.Match.step_closed(line):
self.finishCursor()
self.cursor = self.sections[0]
# Support: @@@STEP_WARNINGS@@@ (warn on a stage)
# Also support deprecated @@@BUILD_WARNINGS@@@
if annotator.Match.step_warnings(line):
self.updateStepStatus(builder.WARNINGS)
# Support: @@@STEP_FAILURE@@@ (fail a stage)
# Also support deprecated @@@BUILD_FAILED@@@
if annotator.Match.step_failure(line):
self.updateStepStatus(builder.FAILURE)
# Support: @@@STEP_EXCEPTION@@@ (exception on a stage)
# Also support deprecated @@@BUILD_FAILED@@@
if annotator.Match.step_exception(line):
self.updateStepStatus(builder.EXCEPTION)
# Support: @@@HALT_ON_FAILURE@@@ (halt if a step fails immediately)
if annotator.Match.halt_on_failure(line):
self.halt_on_failure = True
# Support: @@@HONOR_ZERO_RETURN_CODE@@@ (succeed on 0 return, even if some
# steps have failed)
if annotator.Match.honor_zero_return_code(line):
self.honor_zero_return_code = True
# Support: @@@STEP_CLEAR@@@ (reset step description)
if annotator.Match.step_clear(line):
self.cursor['step_text'] = []
self.updateCursorText()
# Support: @@@STEP_SUMMARY_CLEAR@@@ (reset step summary)
if annotator.Match.step_summary_clear(line):
self.cursor['step_summary_text'] = []
self.updateCursorText()
# Support: @@@STEP_TEXT@<msg>@@@
m = annotator.Match.step_text(line)
if m:
self.cursor['step_text'].append(m[0])
self.updateCursorText()
# Support: @@@STEP_SUMMARY_TEXT@<msg>@@@
m = annotator.Match.step_summary_text(line)
if m:
self.cursor['step_summary_text'].append(m[0])
self.updateCursorText()
# Support: @@@SEED_STEP <stepname>@@@ (seed a new section)
m = annotator.Match.seed_step(line)
if m:
step_name = m[0]
# Add new one.
self.addSection(step_name)
# Support: @@@SEED_STEP_TEXT@<stepname>@<step text@@@ (change step text of a
# seeded step)
m = annotator.Match.seed_step_text(line)
if m:
step_name = m[0]
step_text = m[1]
target = self.lookupCursor(step_name)
target['step_text'].append(step_text)
updateText(target)
# Support: @@@STEP_CURSOR <stepname>@@@ (set cursor to specified section)
m = annotator.Match.step_cursor(line)
if m:
step_name = m[0]
self.cursor = self.lookupCursor(step_name)
# Support: @@@BUILD_STEP <step_name>@@@ (start a new section)
m = annotator.Match.build_step(line)
if m:
step_name = m[0]
# Ignore duplicate consecutive step labels (for robustness).
if step_name != self.sections[-1]['name']:
# Don't close already closed steps or the initial step
# when using BUILD_STEP.
if not (self.cursor['step'].isFinished()
or self.cursor == self.sections[0]):
# Finish up last section.
self.finishCursor()
section = self.addSection(step_name)
self.startStep(section)
self.cursor = section