def RunPythonCommandInBuildDir(build_dir, target, command_line_args, server_dir=None): if sys.platform == 'win32': python_exe = 'python.exe' setup_mount = chromium_utils.FindUpward(build_dir, 'third_party', 'cygwin', 'setup_mount.bat') chromium_utils.RunCommand([setup_mount]) else: os.environ['PYTHONPATH'] = ( chromium_utils.FindUpward(build_dir, 'tools', 'python') + ":" + os.environ.get('PYTHONPATH', '')) python_exe = 'python' if chromium_utils.IsLinux(): slave_name = SlaveBuildName(build_dir) xvfb.StartVirtualX(slave_name, os.path.join(build_dir, '..', 'out', target), server_dir=server_dir) command = [python_exe] # The list of tests is given as arguments. command.extend(command_line_args) result = chromium_utils.RunCommand(command) if chromium_utils.IsLinux(): xvfb.StopVirtualX(slave_name) return result
def goma_teardown(options, env, exit_status, cloudtail_proc): """Tears down goma if necessary. """ if (options.compiler in ('goma', 'goma-clang') and options.goma_dir): override_gsutil = None if options.gsutil_py_path: override_gsutil = [sys.executable, options.gsutil_py_path] # If goma compiler_proxy crashes during the build, there could be crash # dump. if options.build_data_dir: env['GOMACTL_CRASH_REPORT_ID_FILE'] = os.path.join( options.build_data_dir, 'crash_report_id_file') goma_ctl_cmd = [ sys.executable, os.path.join(options.goma_dir, 'goma_ctl.py') ] if options.goma_jsonstatus: chromium_utils.RunCommand(goma_ctl_cmd + ['jsonstatus', options.goma_jsonstatus], env=env) goma_utils.SendGomaTsMon(options.goma_jsonstatus, exit_status) # Always stop the proxy for now to allow in-place update. chromium_utils.RunCommand(goma_ctl_cmd + ['stop'], env=env) goma_utils.UploadGomaCompilerProxyInfo(override_gsutil=override_gsutil) if env.get('GOMA_DUMP_STATS_FILE'): goma_utils.SendGomaStats(env['GOMA_DUMP_STATS_FILE'], env.get('GOMACTL_CRASH_REPORT_ID_FILE'), options.build_data_dir) if cloudtail_proc: cloudtail_proc.terminate() cloudtail_proc.wait()
def _RunGTestCommand(command, results_tracker=None, pipes=None): if results_tracker and pipes: # This is not supported by RunCommand. print 'Invalid test invocation. (results_tracker and pipes)' return 1 if results_tracker: return chromium_utils.RunCommand( command, parser_func=results_tracker.OnReceiveLine) else: return chromium_utils.RunCommand(command, pipes=pipes)
def goma_teardown(options, env): """Tears down goma if necessary. """ if (options.compiler in ('goma', 'goma-clang') and options.goma_dir): goma_ctl_cmd = [sys.executable, os.path.join(options.goma_dir, 'goma_ctl.py')] if options.goma_jsonstatus: chromium_utils.RunCommand( goma_ctl_cmd + ['jsonstatus', options.goma_jsonstatus], env=env) # Always stop the proxy for now to allow in-place update. chromium_utils.RunCommand(goma_ctl_cmd + ['stop'], env=env) UploadGomaCompilerProxyInfo()
def goma_setup(options, env): """Sets up goma if necessary. If using the Goma compiler, first call goma_ctl with ensure_start (or restart in clobber mode) to ensure the proxy is available, and returns True. If it failed to start up compiler_proxy, modify options.compiler and options.goma_dir and returns False """ if options.compiler not in ('goma', 'goma-clang', 'jsonclang'): # Unset goma_dir to make sure we'll not use goma. options.goma_dir = None return False # HACK(agable,goma): This prevents native-client's build_nexe.py from using # Goma, as doing so is currently overloading machines and hurting the # waterfall. Remove this as soon as that is fixed. (12 June, 2013). env['NO_NACL_GOMA'] = 'true' # goma is requested. goma_key = os.path.join(options.goma_dir, 'goma.key') if os.path.exists(goma_key): env['GOMA_API_KEY_FILE'] = goma_key result = -1 if not chromium_utils.IsWindows(): goma_ctl_cmd = [os.path.join(options.goma_dir, 'goma_ctl.sh')] goma_start_command = ['ensure_start'] if options.clobber: goma_start_command = ['restart'] result = chromium_utils.RunCommand(goma_ctl_cmd + goma_start_command, env=env) else: env['GOMA_RPC_EXTRA_PARAMS'] = '?win' goma_ctl_cmd = [sys.executable, os.path.join(options.goma_dir, 'goma_ctl.py')] result = chromium_utils.RunCommand(goma_ctl_cmd + ['start'], env=env) if not result: # goma started sucessfully. return True print 'warning: failed to start goma. falling back to non-goma' # Drop goma from options.compiler options.compiler = options.compiler.replace('goma-', '') if options.compiler == 'goma': options.compiler = None # Reset options.goma_dir. # options.goma_dir will be used to check if goma is ready # when options.compiler=jsonclang. options.goma_dir = None return False
def _RunGTestCommand(options, command, extra_env, pipes=None): """Runs a test, printing and possibly processing the output. Args: options: Options passed for this invocation of runtest.py. command: A list of strings in a command (the command and its arguments). extra_env: A dictionary of extra environment variables to set. pipes: A list of command string lists which the output will be piped to. Returns: The process return code. """ env = os.environ.copy() if extra_env: print 'Additional test environment:' for k, v in sorted(extra_env.items()): print ' %s=%s' % (k, v) env.update(extra_env or {}) # Trigger bot mode (test retries, redirection of stdio, possibly faster, # etc.) - using an environment variable instead of command-line flags because # some internal waterfalls run this (_RunGTestCommand) for totally non-gtest # code. # TODO(phajdan.jr): Clean this up when internal waterfalls are fixed. env.update({'CHROMIUM_TEST_LAUNCHER_BOT_MODE': '1'}) return chromium_utils.RunCommand(command, pipes=pipes, env=env)
def main(): work_dir = os.path.abspath('.') print 'Locating NaCl SDK update script at %s' % NACL_SDK_UPDATE_URL file_name = NACL_SDK_UPDATE_URL.split('/')[-1] response = requests.get(NACL_SDK_UPDATE_URL, verify=True, stream=True) print 'Downloading: %s' % file_name Retrieve(response, file_name) print 'Unzipping %s into %s' % (file_name, work_dir) chromium_utils.ExtractZip(file_name, work_dir, verbose=True) result = chromium_utils.RunCommand([NACL_TOOL, 'update', '--force']) if os.path.exists(CURRENT_PEPPER_BUNDLE): print 'Removing current pepper bundle %s' % CURRENT_PEPPER_BUNDLE shutil.rmtree(CURRENT_PEPPER_BUNDLE) def PepperDirs(): for x in os.listdir('nacl_sdk'): if re.match('pepper_\d+', x): yield x pepper_rev = max([int(i.split('_')[1]) for i in PepperDirs()]) pepper_rev_dir = os.path.join('nacl_sdk', 'pepper_' + str(pepper_rev)) print 'Copying pepper bundle %d to current' % pepper_rev shutil.copytree(pepper_rev_dir, CURRENT_PEPPER_BUNDLE, symlinks=True) return result
def CreateArchive(self, filepath): """Creates a zip archive containing the index pack. Args: filepath: The filepath where the index pack archive should be stored. Raises: Exception: The zip command failed to create the archive """ # Remove the old zip archive (if it exists). This avoids that the new index # pack is just added to the old zip archive. if os.path.exists(filepath): os.remove(filepath) # Run the command in the parent directory of the index pack and use a # relative path for the index pack to get rid of any path prefix. The format # specification requires that the archive contains one folder with an # arbitrary name directly containing the 'units' and 'files' directories. if chromium_utils.RunCommand( ['zip', '-r', filepath, os.path.basename(self.index_directory)], cwd=os.path.dirname(self.index_directory)) != 0: raise Exception('ERROR: failed to create %s, exiting' % filepath) # Remove the temporary index pack directory. If there was no exception so # far, the archive has been created successfully, so the temporary index # pack directory is not needed anymore. shutil.rmtree(self.index_directory)
def GSUtilDownloadFile(src, dst): """Copy a file from Google Storage.""" # Run the gsutil command. gsutil internally calls command_wrapper, which # will try to run the command 10 times if it fails. command = GSUtilSetup() command.extend(['cp', src, dst]) return chromium_utils.RunCommand(command)
def ZipAndUpload(bucket, archive, *targets): """Uploads a zipped archive to the specified Google Storage bucket. Args: bucket: Google Storage bucket to upload to. archive: Name of the .zip archive. *targets: List of targets that should be included in the archive. Returns: Path to the uploaded archive on Google Storage. """ local_archive = os.path.join(tempfile.mkdtemp(archive), archive) zip_cmd = [ 'zip', '-9', '--filesync', '--recurse-paths', '--symlinks', local_archive, ] zip_cmd.extend(targets) chromium_utils.RunCommand(zip_cmd) GSUtilCopy(local_archive, 'gs://%s/%s' % (bucket, archive)) return 'https://storage.cloud.google.com/%s/%s' % (bucket, archive)
def GSUtilCopy(source, dest, mimetype=None, gs_acl=None): """Copy a file to Google Storage. Runs the following command: gsutil -h Content-Type:<mimetype> \ cp -a <gs_acl> file://<filename> <gs_base>/<subdir>/<filename w/o path> Args: source: the source URI dest: the destination URI mimetype: optional value to add as a Content-Type header gs_acl: optional value to add as a canned-acl Returns: The status code returned from running the generated gsutil command. """ if not source.startswith('gs://') and not source.startswith('file://'): source = 'file://' + source if not dest.startswith('gs://') and not dest.startswith('file://'): dest = 'file://' + dest gsutil = GSUtilSetup() # Run the gsutil command. gsutil internally calls command_wrapper, which # will try to run the command 10 times if it fails. command = [gsutil] if mimetype: command.extend(['-h', 'Content-Type:%s' % mimetype]) command.extend(['cp']) if gs_acl: command.extend(['-a', gs_acl]) command.extend([source, dest]) return chromium_utils.RunCommand(command)
def GSUtilCopyDir(src_dir, gs_base, dest_dir=None, gs_acl=None, cache_control=None, add_quiet_flag=False): """Uploads the directory and its contents to Google Storage.""" if os.path.isfile(src_dir): assert os.path.isdir(src_dir), '%s must be a directory' % src_dir command = GSUtilSetup() command += ['-m'] if add_quiet_flag: command.append('-q') if cache_control: command.extend(['-h', 'Cache-Control:%s' % cache_control]) command.extend(['cp', '-R']) if gs_acl: command.extend(['-a', gs_acl]) if dest_dir: command.extend([src_dir, gs_base + '/' + dest_dir]) else: command.extend([src_dir, gs_base]) return chromium_utils.RunCommand(command)
def main(): parser = argparse.ArgumentParser( description='Run a REINDEX TABLE command on postgres.') parser.add_argument('directory', help='location of the master to reindex.') parser.add_argument('--dbconfig-filename', default='.dbconfig', help='name of the dbconfig, defaults to %(default)s.') parser.add_argument( '--prod', action='store_true', help='actually execute command instead of just displaying it.') args = parser.parse_args() filename = chromium_utils.AbsoluteCanonicalPath( os.path.join(args.directory, args.dbconfig_filename)) dbconfig = get_database_creds(filename) cmd = [ 'psql', '-h', 'localhost', '-U', dbconfig['username'], '-d', dbconfig['dbname'], '-c', 'REINDEX TABLE buildrequests;' ] new_env = os.environ.copy() new_env['PGPASSWORD'] = dbconfig['password'] if args.prod: return chromium_utils.RunCommand(cmd, env=new_env) else: print 'Would have run %s.' % cmd print 'If this looks good, re-run with --prod.' return 0
def run_gatekeeper(master_urls, extra_args=None): env = {} env['PYTHONPATH'] = os.pathsep.join(sys.path) cmd = gen_gatekeeper_cmd(master_urls, extra_args=extra_args) return chromium_utils.RunCommand(cmd, env=env)
def main(argv): prog_desc = 'Invoke telemetry performance tests.' parser = optparse.OptionParser(usage=('%prog [options]' + '\n\n' + prog_desc)) parser.add_option('--print-cmd', action='store_true', help='only print command instead of running it') parser.add_option('--target-android-browser', default='android-chromium-testshell', help='target browser used on Android') parser.add_option('--factory-properties', action='callback', callback=chromium_utils.convert_json, type='string', nargs=1, default={}, help='factory properties in JSON format') options, _ = parser.parse_args(argv[1:]) if not options.factory_properties: print 'This program requires a factory properties to run.' return 1 commands, env = _GenerateTelemetryCommandSequence(options) retval = 0 for command in commands: if options.print_cmd: print ' '.join("'%s'" % c for c in command) continue retval = chromium_utils.RunCommand(command, env=env) if retval != 0: break return retval
def goma_teardown(options, env): """Tears down goma if necessary. """ if (options.compiler in ('goma', 'goma-clang', 'jsonclang') and options.goma_dir): if not chromium_utils.IsWindows(): goma_ctl_cmd = [os.path.join(options.goma_dir, 'goma_ctl.sh')] else: goma_ctl_cmd = [ sys.executable, os.path.join(options.goma_dir, 'goma_ctl.py') ] # Show goma stats so that we can investigate goma when # something weird happens. chromium_utils.RunCommand(goma_ctl_cmd + ['stat'], env=env) # Always stop the proxy for now to allow in-place update. chromium_utils.RunCommand(goma_ctl_cmd + ['stop'], env=env) UploadGomaCompilerProxyInfo()
def testRunCommandFilter(self): mycmd = synthesizeCmd(['print "1\\n2"']) filter_obj = FakeFilterObj() retval = chromium_utils.RunCommand(mycmd, print_cmd=False, filter_obj=filter_obj) self.assertEqual(0, retval) self.assertEqual(['1\n', '2\n'], filter_obj.lines)
def testRunCommandFilterEndline(self): mycmd = synthesizeCmd(['import sys; sys.stdout.write("test")']) filter_obj = FakeFilterObj() retval = chromium_utils.RunCommand(mycmd, print_cmd=False, filter_obj=filter_obj) self.assertEqual(0, retval) self.assertEqual(['test'], filter_obj.lines)
def testRunCommandParser(self): mycmd = synthesizeCmd(['print "1\\n2"']) parser = FakeParser() retval = chromium_utils.RunCommand(mycmd, print_cmd=False, parser_func=parser.ProcessLine) self.assertEqual(0, retval) self.assertEqual(['1', '2', ''], parser.lines)
def GSUtilDeleteFile(filename): """Delete a file on Google Storage.""" # Run the gsutil command. gsutil internally calls command_wrapper, which # will try to run the command 10 times if it fails. command = GSUtilSetup() command.extend(['rm', filename]) return chromium_utils.RunCommand(command)
def GSUtilMoveFile(source, dest, gs_acl=None): """Move a file on Google Storage.""" gsutil = GSUtilSetup() # Run the gsutil command. gsutil internally calls command_wrapper, which # will try to run the command 10 times if it fails. command = gsutil + ['mv', source, dest] status = chromium_utils.RunCommand(command) if status: return status if gs_acl: command = gsutil + ['setacl', gs_acl, dest] status = chromium_utils.RunCommand(command) return status
def GSUtilCopy(source, dest, mimetype=None, gs_acl=None, cache_control=None, metadata=None, override_gsutil=None, add_quiet_flag=False): """Copy a file to Google Storage. Runs the following command: gsutil -h Content-Type:<mimetype> \ -h Cache-Control:<cache_control> \ cp -a <gs_acl> file://<filename> <dest> Args: source: the source URI dest: the destination URI mimetype: optional value to add as a Content-Type header gs_acl: optional value to add as a canned-acl cache_control: optional value to set Cache-Control header metadata: (dict) A dictionary of string key/value metadata entries to set (see `gsutil cp' '-h' option) override_gsutil (list): optional argv to run gsutil add_quiet_flag: add the -q (quiet) flag when invoking gsutil Returns: The status code returned from running the generated gsutil command. """ if not source.startswith('gs://') and not source.startswith('file://'): source = 'file://' + source if not dest.startswith('gs://') and not dest.startswith('file://'): dest = 'file://' + dest # The setup also sets up some env variables - for now always run that. gsutil = GSUtilSetup() # Run the gsutil command. gsutil internally calls command_wrapper, which # will try to run the command 10 times if it fails. command = list(override_gsutil or gsutil) if add_quiet_flag: command.append('-q') if not metadata: metadata = {} if mimetype: metadata['Content-Type'] = mimetype if cache_control: metadata['Cache-Control'] = cache_control for k, v in sorted(metadata.iteritems(), key=lambda (k, _): k): field = GSUtilGetMetadataField(k) param = (field) if v is None else ('%s:%s' % (field, v)) command += ['-h', param] command.extend(['cp']) if gs_acl: command.extend(['-a', gs_acl]) command.extend([source, dest]) return chromium_utils.RunCommand(command)
def RunPythonCommandInBuildDir(build_dir, target, command_line_args, server_dir=None, filter_obj=None): if sys.platform == 'win32': python_exe = 'python.exe' setup_mount = chromium_utils.FindUpward(build_dir, 'third_party', 'cygwin', 'setup_mount.bat') chromium_utils.RunCommand([setup_mount]) else: os.environ['PYTHONPATH'] = (chromium_utils.FindUpward(build_dir, 'tools', 'python') + ':' +os.environ.get('PYTHONPATH', '')) python_exe = 'python' command = [python_exe] + command_line_args return chromium_utils.RunCommand(command, filter_obj=filter_obj)
def main(): builder_name = os.getenv('BUILDBOT_BUILDERNAME', default='') is_release_bot = builder_name.startswith('release') script = '' if is_release_bot: script = 'src/dartium_tools/buildbot_release_annotated_steps.py' else: script = 'src/dartium_tools/buildbot_annotated_steps.py' return chromium_utils.RunCommand([sys.executable, script])
def _runAnnotator(self, cmdlist, env=None): json.dump(cmdlist, self.temp) self.temp.close() cmd = [sys.executable, self.script, self.tempfn] cmd_env = os.environ.copy() cmd_env['PYTHONPATH'] = os.pathsep.join(sys.path) if env: cmd_env.update(env) return chromium_utils.RunCommand(cmd, filter_obj=self.capture, env=cmd_env, print_cmd=False)
def testRunCommandPipesFailure(self): firstcmd = synthesizeCmd(['print "1"']) secondcmd = synthesizeCmd(["exit(1)"]) filter_obj = FakeFilterObj() retval = chromium_utils.RunCommand(firstcmd, print_cmd=False, pipes=[secondcmd], filter_obj=filter_obj) self.assertEqual(1, retval)
def goma_teardown(options, env): """Tears down goma if necessary. """ if (options.compiler in ('goma', 'goma-clang', 'jsonclang') and options.goma_dir): if not chromium_utils.IsWindows(): goma_ctl_cmd = [os.path.join(options.goma_dir, 'goma_ctl.sh')] else: goma_ctl_cmd = [sys.executable, os.path.join(options.goma_dir, 'goma_ctl.py')] # Always stop the proxy for now to allow in-place update. chromium_utils.RunCommand(goma_ctl_cmd + ['stop'], env=env)
def main(): option_parser = optparse.OptionParser() chromium_utils.AddPropertiesOptions(option_parser) (options, args) = option_parser.parse_args() buildername = options.build_properties.get('buildername', '') cmd = SDK_BUILDER_MAP.get(buildername) or SDK_BUILDER_MAP.get('DEFAULT') build_tools_dir = chromium_utils.FindUpward(os.getcwd(), 'src', 'native_client_sdk', 'src', 'build_tools') os.chdir(build_tools_dir) return chromium_utils.RunCommand(cmd + args)
def main(): parser = optparse.OptionParser() parser.add_option('--target', help='Release or Debug') options, args = parser.parse_args() assert not args build_dir = build_directory.GetBuildOutputDirectory() return chromium_utils.RunCommand([ sys.executable, os.path.join('src', 'tools', 'checkbins', 'checkbins.py'), os.path.join(build_dir, options.target) ])
def SendGomaStats(goma_stats_file, goma_crash_report, build_data_dir): """Send GomaStats monitoring event. Note: this function also removes goma_stats_file. """ try: goma_options = [] if goma_stats_file and os.path.exists(goma_stats_file): # send GomaStats. goma_options = [ '--build-event-goma-stats-path', goma_stats_file, ] elif goma_crash_report and os.path.exists(goma_crash_report): # crash report. goma_options = [ '--build-event-goma-error', 'GOMA_ERROR_CRASHED', '--build-event-goma-crash-report-id-path', goma_crash_report, ] elif IsCompilerProxyKilledByFatalError(): goma_options = [ '--build-event-goma-error', 'GOMA_ERROR_LOG_FATAL', ] else: # unknown error. goma_options = [ '--build-event-goma-error', 'GOMA_ERROR_UNKNOWN', ] run_cmd = PLATFORM_RUN_CMD.get(os.name) if not run_cmd: print 'Unknown os.name: %s' % os.name return send_monitoring_event_cmd = [ sys.executable, run_cmd, 'infra.tools.send_monitoring_event', '--event-mon-run-type', 'prod', '--build-event-type', 'BUILD', '--event-mon-timestamp-kind', 'POINT', '--event-logrequest-path', os.path.join(build_data_dir, 'log_request_proto') ] + goma_options cmd_filter = chromium_utils.FilterCapture() retcode = chromium_utils.RunCommand(send_monitoring_event_cmd, filter_obj=cmd_filter, max_time=30) if retcode: print('Execution of send_monitoring_event failed with code %s' % retcode) print '\n'.join(cmd_filter.text) except Exception, inst: # safety net print('send_monitoring_event for goma failed: %s' % inst)