def RunPythonCommandInBuildDir(build_dir,
target,
command_line_args,
server_dir=None):
if sys.platform == 'win32':
python_exe = 'python.exe'
setup_mount = chromium_utils.FindUpward(build_dir, 'third_party',
'cygwin', 'setup_mount.bat')
chromium_utils.RunCommand([setup_mount])
else:
os.environ['PYTHONPATH'] = (
chromium_utils.FindUpward(build_dir, 'tools', 'python') + ":" +
os.environ.get('PYTHONPATH', ''))
python_exe = 'python'
if chromium_utils.IsLinux():
slave_name = SlaveBuildName(build_dir)
xvfb.StartVirtualX(slave_name,
os.path.join(build_dir, '..', 'out', target),
server_dir=server_dir)
command = [python_exe]
# The list of tests is given as arguments.
command.extend(command_line_args)
result = chromium_utils.RunCommand(command)
if chromium_utils.IsLinux():
xvfb.StopVirtualX(slave_name)
return result
def goma_teardown(options, env, exit_status, cloudtail_proc):
"""Tears down goma if necessary. """
if (options.compiler in ('goma', 'goma-clang') and options.goma_dir):
override_gsutil = None
if options.gsutil_py_path:
override_gsutil = [sys.executable, options.gsutil_py_path]
# If goma compiler_proxy crashes during the build, there could be crash
# dump.
if options.build_data_dir:
env['GOMACTL_CRASH_REPORT_ID_FILE'] = os.path.join(
options.build_data_dir, 'crash_report_id_file')
goma_ctl_cmd = [
sys.executable,
os.path.join(options.goma_dir, 'goma_ctl.py')
]
if options.goma_jsonstatus:
chromium_utils.RunCommand(goma_ctl_cmd +
['jsonstatus', options.goma_jsonstatus],
env=env)
goma_utils.SendGomaTsMon(options.goma_jsonstatus, exit_status)
# Always stop the proxy for now to allow in-place update.
chromium_utils.RunCommand(goma_ctl_cmd + ['stop'], env=env)
goma_utils.UploadGomaCompilerProxyInfo(override_gsutil=override_gsutil)
if env.get('GOMA_DUMP_STATS_FILE'):
goma_utils.SendGomaStats(env['GOMA_DUMP_STATS_FILE'],
env.get('GOMACTL_CRASH_REPORT_ID_FILE'),
options.build_data_dir)
if cloudtail_proc:
cloudtail_proc.terminate()
cloudtail_proc.wait()
def _RunGTestCommand(command, results_tracker=None, pipes=None):
if results_tracker and pipes:
# This is not supported by RunCommand.
print 'Invalid test invocation. (results_tracker and pipes)'
return 1
if results_tracker:
return chromium_utils.RunCommand(
command, parser_func=results_tracker.OnReceiveLine)
else:
return chromium_utils.RunCommand(command, pipes=pipes)
def goma_teardown(options, env):
"""Tears down goma if necessary. """
if (options.compiler in ('goma', 'goma-clang') and
options.goma_dir):
goma_ctl_cmd = [sys.executable,
os.path.join(options.goma_dir, 'goma_ctl.py')]
if options.goma_jsonstatus:
chromium_utils.RunCommand(
goma_ctl_cmd + ['jsonstatus', options.goma_jsonstatus], env=env)
# Always stop the proxy for now to allow in-place update.
chromium_utils.RunCommand(goma_ctl_cmd + ['stop'], env=env)
UploadGomaCompilerProxyInfo()
def goma_setup(options, env):
"""Sets up goma if necessary.
If using the Goma compiler, first call goma_ctl with ensure_start
(or restart in clobber mode) to ensure the proxy is available, and returns
True.
If it failed to start up compiler_proxy, modify options.compiler and
options.goma_dir and returns False
"""
if options.compiler not in ('goma', 'goma-clang', 'jsonclang'):
# Unset goma_dir to make sure we'll not use goma.
options.goma_dir = None
return False
# HACK(agable,goma): This prevents native-client's build_nexe.py from using
# Goma, as doing so is currently overloading machines and hurting the
# waterfall. Remove this as soon as that is fixed. (12 June, 2013).
env['NO_NACL_GOMA'] = 'true'
# goma is requested.
goma_key = os.path.join(options.goma_dir, 'goma.key')
if os.path.exists(goma_key):
env['GOMA_API_KEY_FILE'] = goma_key
result = -1
if not chromium_utils.IsWindows():
goma_ctl_cmd = [os.path.join(options.goma_dir, 'goma_ctl.sh')]
goma_start_command = ['ensure_start']
if options.clobber:
goma_start_command = ['restart']
result = chromium_utils.RunCommand(goma_ctl_cmd + goma_start_command,
env=env)
else:
env['GOMA_RPC_EXTRA_PARAMS'] = '?win'
goma_ctl_cmd = [sys.executable,
os.path.join(options.goma_dir, 'goma_ctl.py')]
result = chromium_utils.RunCommand(goma_ctl_cmd + ['start'], env=env)
if not result:
# goma started sucessfully.
return True
print 'warning: failed to start goma. falling back to non-goma'
# Drop goma from options.compiler
options.compiler = options.compiler.replace('goma-', '')
if options.compiler == 'goma':
options.compiler = None
# Reset options.goma_dir.
# options.goma_dir will be used to check if goma is ready
# when options.compiler=jsonclang.
options.goma_dir = None
return False
def _RunGTestCommand(options, command, extra_env, pipes=None):
"""Runs a test, printing and possibly processing the output.
Args:
options: Options passed for this invocation of runtest.py.
command: A list of strings in a command (the command and its arguments).
extra_env: A dictionary of extra environment variables to set.
pipes: A list of command string lists which the output will be piped to.
Returns:
The process return code.
"""
env = os.environ.copy()
if extra_env:
print 'Additional test environment:'
for k, v in sorted(extra_env.items()):
print ' %s=%s' % (k, v)
env.update(extra_env or {})
# Trigger bot mode (test retries, redirection of stdio, possibly faster,
# etc.) - using an environment variable instead of command-line flags because
# some internal waterfalls run this (_RunGTestCommand) for totally non-gtest
# code.
# TODO(phajdan.jr): Clean this up when internal waterfalls are fixed.
env.update({'CHROMIUM_TEST_LAUNCHER_BOT_MODE': '1'})
return chromium_utils.RunCommand(command, pipes=pipes, env=env)
def main():
work_dir = os.path.abspath('.')
print 'Locating NaCl SDK update script at %s' % NACL_SDK_UPDATE_URL
file_name = NACL_SDK_UPDATE_URL.split('/')[-1]
response = requests.get(NACL_SDK_UPDATE_URL, verify=True, stream=True)
print 'Downloading: %s' % file_name
Retrieve(response, file_name)
print 'Unzipping %s into %s' % (file_name, work_dir)
chromium_utils.ExtractZip(file_name, work_dir, verbose=True)
result = chromium_utils.RunCommand([NACL_TOOL, 'update', '--force'])
if os.path.exists(CURRENT_PEPPER_BUNDLE):
print 'Removing current pepper bundle %s' % CURRENT_PEPPER_BUNDLE
shutil.rmtree(CURRENT_PEPPER_BUNDLE)
def PepperDirs():
for x in os.listdir('nacl_sdk'):
if re.match('pepper_\d+', x):
yield x
pepper_rev = max([int(i.split('_')[1]) for i in PepperDirs()])
pepper_rev_dir = os.path.join('nacl_sdk', 'pepper_' + str(pepper_rev))
print 'Copying pepper bundle %d to current' % pepper_rev
shutil.copytree(pepper_rev_dir, CURRENT_PEPPER_BUNDLE, symlinks=True)
return result
def CreateArchive(self, filepath):
"""Creates a zip archive containing the index pack.
Args:
filepath: The filepath where the index pack archive should be stored.
Raises:
Exception: The zip command failed to create the archive
"""
# Remove the old zip archive (if it exists). This avoids that the new index
# pack is just added to the old zip archive.
if os.path.exists(filepath):
os.remove(filepath)
# Run the command in the parent directory of the index pack and use a
# relative path for the index pack to get rid of any path prefix. The format
# specification requires that the archive contains one folder with an
# arbitrary name directly containing the 'units' and 'files' directories.
if chromium_utils.RunCommand(
['zip', '-r', filepath, os.path.basename(self.index_directory)],
cwd=os.path.dirname(self.index_directory)) != 0:
raise Exception('ERROR: failed to create %s, exiting' % filepath)
# Remove the temporary index pack directory. If there was no exception so
# far, the archive has been created successfully, so the temporary index
# pack directory is not needed anymore.
shutil.rmtree(self.index_directory)
def GSUtilDownloadFile(src, dst):
"""Copy a file from Google Storage."""
# Run the gsutil command. gsutil internally calls command_wrapper, which
# will try to run the command 10 times if it fails.
command = GSUtilSetup()
command.extend(['cp', src, dst])
return chromium_utils.RunCommand(command)
def ZipAndUpload(bucket, archive, *targets):
"""Uploads a zipped archive to the specified Google Storage bucket.
Args:
bucket: Google Storage bucket to upload to.
archive: Name of the .zip archive.
*targets: List of targets that should be included in the archive.
Returns:
Path to the uploaded archive on Google Storage.
"""
local_archive = os.path.join(tempfile.mkdtemp(archive), archive)
zip_cmd = [
'zip',
'-9',
'--filesync',
'--recurse-paths',
'--symlinks',
local_archive,
]
zip_cmd.extend(targets)
chromium_utils.RunCommand(zip_cmd)
GSUtilCopy(local_archive, 'gs://%s/%s' % (bucket, archive))
return 'https://storage.cloud.google.com/%s/%s' % (bucket, archive)
def GSUtilCopy(source, dest, mimetype=None, gs_acl=None):
"""Copy a file to Google Storage.
Runs the following command:
gsutil -h Content-Type:<mimetype> \
cp -a <gs_acl> file://<filename> <gs_base>/<subdir>/<filename w/o path>
Args:
source: the source URI
dest: the destination URI
mimetype: optional value to add as a Content-Type header
gs_acl: optional value to add as a canned-acl
Returns:
The status code returned from running the generated gsutil command.
"""
if not source.startswith('gs://') and not source.startswith('file://'):
source = 'file://' + source
if not dest.startswith('gs://') and not dest.startswith('file://'):
dest = 'file://' + dest
gsutil = GSUtilSetup()
# Run the gsutil command. gsutil internally calls command_wrapper, which
# will try to run the command 10 times if it fails.
command = [gsutil]
if mimetype:
command.extend(['-h', 'Content-Type:%s' % mimetype])
command.extend(['cp'])
if gs_acl:
command.extend(['-a', gs_acl])
command.extend([source, dest])
return chromium_utils.RunCommand(command)
def GSUtilCopyDir(src_dir,
gs_base,
dest_dir=None,
gs_acl=None,
cache_control=None,
add_quiet_flag=False):
"""Uploads the directory and its contents to Google Storage."""
if os.path.isfile(src_dir):
assert os.path.isdir(src_dir), '%s must be a directory' % src_dir
command = GSUtilSetup()
command += ['-m']
if add_quiet_flag:
command.append('-q')
if cache_control:
command.extend(['-h', 'Cache-Control:%s' % cache_control])
command.extend(['cp', '-R'])
if gs_acl:
command.extend(['-a', gs_acl])
if dest_dir:
command.extend([src_dir, gs_base + '/' + dest_dir])
else:
command.extend([src_dir, gs_base])
return chromium_utils.RunCommand(command)
def main():
parser = argparse.ArgumentParser(
description='Run a REINDEX TABLE command on postgres.')
parser.add_argument('directory', help='location of the master to reindex.')
parser.add_argument('--dbconfig-filename',
default='.dbconfig',
help='name of the dbconfig, defaults to %(default)s.')
parser.add_argument(
'--prod',
action='store_true',
help='actually execute command instead of just displaying it.')
args = parser.parse_args()
filename = chromium_utils.AbsoluteCanonicalPath(
os.path.join(args.directory, args.dbconfig_filename))
dbconfig = get_database_creds(filename)
cmd = [
'psql', '-h', 'localhost', '-U', dbconfig['username'], '-d',
dbconfig['dbname'], '-c', 'REINDEX TABLE buildrequests;'
]
new_env = os.environ.copy()
new_env['PGPASSWORD'] = dbconfig['password']
if args.prod:
return chromium_utils.RunCommand(cmd, env=new_env)
else:
print 'Would have run %s.' % cmd
print 'If this looks good, re-run with --prod.'
return 0
def run_gatekeeper(master_urls, extra_args=None):
env = {}
env['PYTHONPATH'] = os.pathsep.join(sys.path)
cmd = gen_gatekeeper_cmd(master_urls, extra_args=extra_args)
return chromium_utils.RunCommand(cmd, env=env)
def main(argv):
prog_desc = 'Invoke telemetry performance tests.'
parser = optparse.OptionParser(usage=('%prog [options]' + '\n\n' + prog_desc))
parser.add_option('--print-cmd', action='store_true',
help='only print command instead of running it')
parser.add_option('--target-android-browser',
default='android-chromium-testshell',
help='target browser used on Android')
parser.add_option('--factory-properties', action='callback',
callback=chromium_utils.convert_json, type='string',
nargs=1, default={},
help='factory properties in JSON format')
options, _ = parser.parse_args(argv[1:])
if not options.factory_properties:
print 'This program requires a factory properties to run.'
return 1
commands, env = _GenerateTelemetryCommandSequence(options)
retval = 0
for command in commands:
if options.print_cmd:
print ' '.join("'%s'" % c for c in command)
continue
retval = chromium_utils.RunCommand(command, env=env)
if retval != 0:
break
return retval
def goma_teardown(options, env):
"""Tears down goma if necessary. """
if (options.compiler in ('goma', 'goma-clang', 'jsonclang')
and options.goma_dir):
if not chromium_utils.IsWindows():
goma_ctl_cmd = [os.path.join(options.goma_dir, 'goma_ctl.sh')]
else:
goma_ctl_cmd = [
sys.executable,
os.path.join(options.goma_dir, 'goma_ctl.py')
]
# Show goma stats so that we can investigate goma when
# something weird happens.
chromium_utils.RunCommand(goma_ctl_cmd + ['stat'], env=env)
# Always stop the proxy for now to allow in-place update.
chromium_utils.RunCommand(goma_ctl_cmd + ['stop'], env=env)
UploadGomaCompilerProxyInfo()
def testRunCommandFilter(self):
mycmd = synthesizeCmd(['print "1\\n2"'])
filter_obj = FakeFilterObj()
retval = chromium_utils.RunCommand(mycmd,
print_cmd=False,
filter_obj=filter_obj)
self.assertEqual(0, retval)
self.assertEqual(['1\n', '2\n'], filter_obj.lines)
def testRunCommandFilterEndline(self):
mycmd = synthesizeCmd(['import sys; sys.stdout.write("test")'])
filter_obj = FakeFilterObj()
retval = chromium_utils.RunCommand(mycmd,
print_cmd=False,
filter_obj=filter_obj)
self.assertEqual(0, retval)
self.assertEqual(['test'], filter_obj.lines)
def testRunCommandParser(self):
mycmd = synthesizeCmd(['print "1\\n2"'])
parser = FakeParser()
retval = chromium_utils.RunCommand(mycmd,
print_cmd=False,
parser_func=parser.ProcessLine)
self.assertEqual(0, retval)
self.assertEqual(['1', '2', ''], parser.lines)
def GSUtilDeleteFile(filename):
"""Delete a file on Google Storage."""
# Run the gsutil command. gsutil internally calls command_wrapper, which
# will try to run the command 10 times if it fails.
command = GSUtilSetup()
command.extend(['rm', filename])
return chromium_utils.RunCommand(command)
def GSUtilMoveFile(source, dest, gs_acl=None):
"""Move a file on Google Storage."""
gsutil = GSUtilSetup()
# Run the gsutil command. gsutil internally calls command_wrapper, which
# will try to run the command 10 times if it fails.
command = gsutil + ['mv', source, dest]
status = chromium_utils.RunCommand(command)
if status:
return status
if gs_acl:
command = gsutil + ['setacl', gs_acl, dest]
status = chromium_utils.RunCommand(command)
return status
def GSUtilCopy(source,
dest,
mimetype=None,
gs_acl=None,
cache_control=None,
metadata=None,
override_gsutil=None,
add_quiet_flag=False):
"""Copy a file to Google Storage.
Runs the following command:
gsutil -h Content-Type:<mimetype> \
-h Cache-Control:<cache_control> \
cp -a <gs_acl> file://<filename> <dest>
Args:
source: the source URI
dest: the destination URI
mimetype: optional value to add as a Content-Type header
gs_acl: optional value to add as a canned-acl
cache_control: optional value to set Cache-Control header
metadata: (dict) A dictionary of string key/value metadata entries to set
(see `gsutil cp' '-h' option)
override_gsutil (list): optional argv to run gsutil
add_quiet_flag: add the -q (quiet) flag when invoking gsutil
Returns:
The status code returned from running the generated gsutil command.
"""
if not source.startswith('gs://') and not source.startswith('file://'):
source = 'file://' + source
if not dest.startswith('gs://') and not dest.startswith('file://'):
dest = 'file://' + dest
# The setup also sets up some env variables - for now always run that.
gsutil = GSUtilSetup()
# Run the gsutil command. gsutil internally calls command_wrapper, which
# will try to run the command 10 times if it fails.
command = list(override_gsutil or gsutil)
if add_quiet_flag:
command.append('-q')
if not metadata:
metadata = {}
if mimetype:
metadata['Content-Type'] = mimetype
if cache_control:
metadata['Cache-Control'] = cache_control
for k, v in sorted(metadata.iteritems(), key=lambda (k, _): k):
field = GSUtilGetMetadataField(k)
param = (field) if v is None else ('%s:%s' % (field, v))
command += ['-h', param]
command.extend(['cp'])
if gs_acl:
command.extend(['-a', gs_acl])
command.extend([source, dest])
return chromium_utils.RunCommand(command)
def RunPythonCommandInBuildDir(build_dir, target, command_line_args,
server_dir=None, filter_obj=None):
if sys.platform == 'win32':
python_exe = 'python.exe'
setup_mount = chromium_utils.FindUpward(build_dir,
'third_party',
'cygwin',
'setup_mount.bat')
chromium_utils.RunCommand([setup_mount])
else:
os.environ['PYTHONPATH'] = (chromium_utils.FindUpward(build_dir, 'tools',
'python')
+ ':' +os.environ.get('PYTHONPATH', ''))
python_exe = 'python'
command = [python_exe] + command_line_args
return chromium_utils.RunCommand(command, filter_obj=filter_obj)
def main():
builder_name = os.getenv('BUILDBOT_BUILDERNAME', default='')
is_release_bot = builder_name.startswith('release')
script = ''
if is_release_bot:
script = 'src/dartium_tools/buildbot_release_annotated_steps.py'
else:
script = 'src/dartium_tools/buildbot_annotated_steps.py'
return chromium_utils.RunCommand([sys.executable, script])
def _runAnnotator(self, cmdlist, env=None):
json.dump(cmdlist, self.temp)
self.temp.close()
cmd = [sys.executable, self.script, self.tempfn]
cmd_env = os.environ.copy()
cmd_env['PYTHONPATH'] = os.pathsep.join(sys.path)
if env:
cmd_env.update(env)
return chromium_utils.RunCommand(cmd, filter_obj=self.capture, env=cmd_env,
print_cmd=False)
def testRunCommandPipesFailure(self):
firstcmd = synthesizeCmd(['print "1"'])
secondcmd = synthesizeCmd(["exit(1)"])
filter_obj = FakeFilterObj()
retval = chromium_utils.RunCommand(firstcmd,
print_cmd=False,
pipes=[secondcmd],
filter_obj=filter_obj)
self.assertEqual(1, retval)
def goma_teardown(options, env):
"""Tears down goma if necessary. """
if (options.compiler in ('goma', 'goma-clang', 'jsonclang') and
options.goma_dir):
if not chromium_utils.IsWindows():
goma_ctl_cmd = [os.path.join(options.goma_dir, 'goma_ctl.sh')]
else:
goma_ctl_cmd = [sys.executable,
os.path.join(options.goma_dir, 'goma_ctl.py')]
# Always stop the proxy for now to allow in-place update.
chromium_utils.RunCommand(goma_ctl_cmd + ['stop'], env=env)
def main():
option_parser = optparse.OptionParser()
chromium_utils.AddPropertiesOptions(option_parser)
(options, args) = option_parser.parse_args()
buildername = options.build_properties.get('buildername', '')
cmd = SDK_BUILDER_MAP.get(buildername) or SDK_BUILDER_MAP.get('DEFAULT')
build_tools_dir = chromium_utils.FindUpward(os.getcwd(), 'src',
'native_client_sdk', 'src',
'build_tools')
os.chdir(build_tools_dir)
return chromium_utils.RunCommand(cmd + args)
def main():
parser = optparse.OptionParser()
parser.add_option('--target', help='Release or Debug')
options, args = parser.parse_args()
assert not args
build_dir = build_directory.GetBuildOutputDirectory()
return chromium_utils.RunCommand([
sys.executable,
os.path.join('src', 'tools', 'checkbins', 'checkbins.py'),
os.path.join(build_dir, options.target)
])
def SendGomaStats(goma_stats_file, goma_crash_report, build_data_dir):
"""Send GomaStats monitoring event.
Note: this function also removes goma_stats_file.
"""
try:
goma_options = []
if goma_stats_file and os.path.exists(goma_stats_file):
# send GomaStats.
goma_options = [
'--build-event-goma-stats-path',
goma_stats_file,
]
elif goma_crash_report and os.path.exists(goma_crash_report):
# crash report.
goma_options = [
'--build-event-goma-error',
'GOMA_ERROR_CRASHED',
'--build-event-goma-crash-report-id-path',
goma_crash_report,
]
elif IsCompilerProxyKilledByFatalError():
goma_options = [
'--build-event-goma-error',
'GOMA_ERROR_LOG_FATAL',
]
else:
# unknown error.
goma_options = [
'--build-event-goma-error',
'GOMA_ERROR_UNKNOWN',
]
run_cmd = PLATFORM_RUN_CMD.get(os.name)
if not run_cmd:
print 'Unknown os.name: %s' % os.name
return
send_monitoring_event_cmd = [
sys.executable, run_cmd, 'infra.tools.send_monitoring_event',
'--event-mon-run-type', 'prod', '--build-event-type', 'BUILD',
'--event-mon-timestamp-kind', 'POINT', '--event-logrequest-path',
os.path.join(build_data_dir, 'log_request_proto')
] + goma_options
cmd_filter = chromium_utils.FilterCapture()
retcode = chromium_utils.RunCommand(send_monitoring_event_cmd,
filter_obj=cmd_filter,
max_time=30)
if retcode:
print('Execution of send_monitoring_event failed with code %s' %
retcode)
print '\n'.join(cmd_filter.text)
except Exception, inst: # safety net
print('send_monitoring_event for goma failed: %s' % inst)
