def goma_teardown(options, env, exit_status, cloudtail_proc): """Tears down goma if necessary. """ if (options.compiler in ('goma', 'goma-clang') and options.goma_dir): override_gsutil = None if options.gsutil_py_path: override_gsutil = [sys.executable, options.gsutil_py_path] # If goma compiler_proxy crashes during the build, there could be crash # dump. if options.build_data_dir: env['GOMACTL_CRASH_REPORT_ID_FILE'] = os.path.join( options.build_data_dir, 'crash_report_id_file') goma_ctl_cmd = [ sys.executable, os.path.join(options.goma_dir, 'goma_ctl.py') ] if options.goma_jsonstatus: chromium_utils.RunCommand(goma_ctl_cmd + ['jsonstatus', options.goma_jsonstatus], env=env) goma_utils.SendGomaTsMon(options.goma_jsonstatus, exit_status) # Always stop the proxy for now to allow in-place update. chromium_utils.RunCommand(goma_ctl_cmd + ['stop'], env=env) goma_utils.UploadGomaCompilerProxyInfo(override_gsutil=override_gsutil) if env.get('GOMA_DUMP_STATS_FILE'): goma_utils.SendGomaStats(env['GOMA_DUMP_STATS_FILE'], env.get('GOMACTL_CRASH_REPORT_ID_FILE'), options.build_data_dir) if cloudtail_proc: cloudtail_proc.terminate() cloudtail_proc.wait()
def main(): parser = argparse.ArgumentParser(description='Upload goma related logs') parser.add_argument('--upload-compiler-proxy-info', action='store_true', help='If set, the script will upload the latest ' 'compiler_proxy.INFO.') parser.add_argument('--ninja-log-outdir', metavar='DIR', help='Directory that has .ninja_log file.') parser.add_argument('--ninja-log-compiler', metavar='COMPILER', help='compiler name used for the build.') parser.add_argument('--ninja-log-command', metavar='COMMAND', help='command line options of the build.') parser.add_argument('--ninja-log-exit-status', type=int, metavar='EXIT_STATUS', help='ninja exit status.') parser.add_argument('--goma-stats-file', metavar='FILENAME', help='Filename of a GomaStats binary protobuf. ' 'If empty or non-existing file, it will report error ' 'to chrome infra monitoring system.') parser.add_argument('--goma-crash-report-id-file', metavar='FILENAME', help='Filename that has a crash report id.') parser.add_argument( '--build-data-dir', metavar='DIR', help='Directory that has build data used by event_mon.') args = parser.parse_args() if args.upload_compiler_proxy_info: goma_utils.UploadGomaCompilerProxyInfo() if args.ninja_log_outdir: goma_utils.UploadNinjaLog(args.ninja_log_outdir, args.ninja_log_compiler, args.ninja_log_command, args.ninja_log_exit_status) if args.goma_stats_file: goma_utils.SendGomaStats(args.goma_stats_file, args.goma_crash_report_id_file, args.build_data_dir) return 0
def goma_setup(options, env): """Sets up goma if necessary. If using the Goma compiler, first call goma_ctl to ensure the proxy is available, and returns (True, instance of cloudtail subprocess). If it failed to start up compiler_proxy, modify options.compiler and options.goma_dir and returns (False, None) """ if options.compiler not in ('goma', 'goma-clang'): # Unset goma_dir to make sure we'll not use goma. options.goma_dir = None return False, None hostname = goma_utils.GetShortHostname() # HACK(shinyak, yyanagisawa, goma): Windows NO_NACL_GOMA (crbug.com/390764) # Building NaCl untrusted code using goma brings large performance # improvement but it sometimes cause build failure by race condition. # Let me enable goma build on goma canary buildslaves to confirm the issue # has been fixed by a workaround. # vm*-m4 are trybots. build*-m1 and vm*-m1 are all goma canary bots. if hostname in [ 'build28-m1', 'build58-m1', 'vm191-m1', 'vm480-m1', 'vm820-m1', 'vm821-m1', 'vm848-m1' ]: env['NO_NACL_GOMA'] = 'false' if options.goma_fail_fast: # startup fails when initial ping failed. env['GOMA_FAIL_FAST'] = 'true' else: # If a network error continues 30 minutes, compiler_proxy make the compile # failed. When people use goma, they expect using goma is faster than # compile locally. If goma cannot guarantee that, let it make compile # as error. env['GOMA_ALLOWED_NETWORK_ERROR_DURATION'] = '1800' # HACK(yyanagisawa): reduce GOMA_BURST_MAX_PROCS crbug.com/592306 # Recently, I sometimes see buildbot slave time out, one possibility I come # up with is burst mode use up resource. # Let me temporary set small values to GOMA_BURST_MAX_PROCS to confirm # the possibility is true or false. max_subprocs = '3' max_heavy_subprocs = '1' number_of_processors = 0 try: number_of_processors = multiprocessing.cpu_count() except NotImplementedError: print 'cpu_count() is not implemented, using default value.' number_of_processors = 1 if number_of_processors > 3: max_subprocs = str(number_of_processors - 1) max_heavy_subprocs = str(number_of_processors / 2) env['GOMA_BURST_MAX_SUBPROCS'] = max_subprocs env['GOMA_BURST_MAX_SUBPROCS_LOW'] = max_subprocs env['GOMA_BURST_MAX_SUBPROCS_HEAVY'] = max_heavy_subprocs # Caches CRLs in GOMA_CACHE_DIR. # Since downloading CRLs is usually slow, caching them may improves # compiler_proxy start time. if not os.path.exists(options.goma_cache_dir): os.mkdir(options.goma_cache_dir, 0700) env['GOMA_CACHE_DIR'] = options.goma_cache_dir # Enable DepsCache. DepsCache caches the list of files to send goma server. # This will greatly improve build speed when cache is warmed. # The cache file is stored in the target output directory. env['GOMA_DEPS_CACHE_DIR'] = (options.goma_deps_cache_dir or options.target_output_dir) if not env.get('GOMA_HERMETIC'): env['GOMA_HERMETIC'] = options.goma_hermetic if options.goma_enable_remote_link: env['GOMA_ENABLE_REMOTE_LINK'] = options.goma_enable_remote_link if options.goma_store_local_run_output: env['GOMA_STORE_LOCAL_RUN_OUTPUT'] = options.goma_store_local_run_output if options.goma_enable_compiler_info_cache: # Will be stored in GOMA_CACHE_DIR. env['GOMA_COMPILER_INFO_CACHE_FILE'] = 'goma-compiler-info.cache' if options.build_data_dir: env['GOMA_DUMP_STATS_FILE'] = os.path.join(options.build_data_dir, 'goma_stats_proto') # goma is requested. goma_key = os.path.join(options.goma_dir, 'goma.key') if os.path.exists(goma_key): env['GOMA_API_KEY_FILE'] = goma_key if options.goma_service_account_json_file: env['GOMA_SERVICE_ACCOUNT_JSON_FILE'] = \ options.goma_service_account_json_file if chromium_utils.IsWindows(): env['GOMA_RPC_EXTRA_PARAMS'] = '?win' goma_start_command = ['restart'] goma_ctl_cmd = [ sys.executable, os.path.join(options.goma_dir, 'goma_ctl.py') ] result = chromium_utils.RunCommand(goma_ctl_cmd + goma_start_command, env=env) if not result: # goma started sucessfully. # Making cloudtail to upload the latest log. # TODO(yyanagisawa): install cloudtail from CIPD. cloudtail_path = '/opt/infra-tools/cloudtail' if chromium_utils.IsWindows(): cloudtail_path = 'C:\\infra-tools\\cloudtail' try: cloudtail_proc = subprocess.Popen([ cloudtail_path, 'tail', '--log-id', 'goma_compiler_proxy', '--path', goma_utils.GetLatestGomaCompilerProxyInfo() ]) except Exception as e: print 'failed to invoke cloudtail: %s' % e return True, None return True, cloudtail_proc if options.goma_jsonstatus: chromium_utils.RunCommand(goma_ctl_cmd + ['jsonstatus', options.goma_jsonstatus], env=env) goma_utils.SendGomaTsMon(options.goma_jsonstatus, -1) # Try to stop compiler_proxy so that it flushes logs and stores # GomaStats. if options.build_data_dir: env['GOMACTL_CRASH_REPORT_ID_FILE'] = os.path.join( options.build_data_dir, 'crash_report_id_file') chromium_utils.RunCommand(goma_ctl_cmd + ['stop'], env=env) override_gsutil = None if options.gsutil_py_path: override_gsutil = [sys.executable, options.gsutil_py_path] # Upload compiler_proxy.INFO to investigate the reason of compiler_proxy # start-up failure. goma_utils.UploadGomaCompilerProxyInfo(override_gsutil=override_gsutil) # Upload GomaStats to make it monitored. if env.get('GOMA_DUMP_STATS_FILE'): goma_utils.SendGomaStats(env['GOMA_DUMP_STATS_FILE'], env.get('GOMACTL_CRASH_REPORT_ID_FILE'), options.build_data_dir) if options.goma_disable_local_fallback: print 'error: failed to start goma; fallback has been disabled' raise Exception('failed to start goma') print 'warning: failed to start goma. falling back to non-goma' # Drop goma from options.compiler options.compiler = options.compiler.replace('goma-', '') if options.compiler == 'goma': options.compiler = None # Reset options.goma_dir. options.goma_dir = None env['GOMA_DISABLED'] = '1' return False, None
def main(): parser = argparse.ArgumentParser(description='Upload goma related logs') parser.add_argument('--upload-compiler-proxy-info', action='store_true', help='If set, the script will upload the latest ' 'compiler_proxy.INFO.') parser.add_argument('--log-url-json-file', help='If set, the script will write url of uploaded ' 'log visualizer.') parser.add_argument('--ninja-log-outdir', metavar='DIR', help='Directory that has .ninja_log file.') parser.add_argument('--ninja-log-compiler', metavar='COMPILER', help='compiler name used for the build.') parser.add_argument('--ninja-log-command', metavar='COMMAND', help='command line options of the build.') parser.add_argument('--ninja-log-exit-status', type=int, metavar='EXIT_STATUS', help='ninja exit status.') parser.add_argument('--goma-stats-file', metavar='FILENAME', help='Filename of a GomaStats binary protobuf. ' 'If empty or non-existing file, it will report error ' 'to chrome infra monitoring system.') parser.add_argument('--goma-crash-report-id-file', metavar='FILENAME', help='Filename that has a crash report id.') parser.add_argument( '--build-data-dir', metavar='DIR', help='Directory that has build data used by event_mon.') parser.add_argument('--json-status', metavar='JSON', help='path of json file generated from' ' ./goma_ctl.py jsonstatus') parser.add_argument('--skip-sendgomatsmon', action='store_true', help='Represent whether send jsonstatus' ' and goma or compile.py exit_status log to TsMon.' ' This option is used when no need to send goma status' ' to monitoring server.') parser.add_argument( '--gsutil-py-path', help='Specify path to gsutil.py script in depot_tools.') # Arguments set to os.environ parser.add_argument('--buildbot-buildername', default='unknown', help='buildbot buildername') parser.add_argument('--buildbot-mastername', default='unknown', help='buildbot mastername') parser.add_argument('--buildbot-slavename', default='unknown', help='buildbot slavename') parser.add_argument('--buildbot-clobber', default='', help='buildbot clobber') args = parser.parse_args() tsmon_counters = [] override_gsutil = None if args.gsutil_py_path: # Needs to add '--', otherwise gsutil options will be passed to gsutil.py. override_gsutil = [sys.executable, args.gsutil_py_path, '--'] viewer_urls = {} if args.upload_compiler_proxy_info: viewer_url = goma_utils.UploadGomaCompilerProxyInfo( builder=args.buildbot_buildername, master=args.buildbot_mastername, slave=args.buildbot_slavename, clobber=args.buildbot_clobber, override_gsutil=override_gsutil) if viewer_url is not None: viewer_urls['compiler_proxy_log'] = viewer_url if args.ninja_log_outdir: viewer_url = goma_utils.UploadNinjaLog(args.ninja_log_outdir, args.ninja_log_compiler, args.ninja_log_command, args.ninja_log_exit_status, override_gsutil=override_gsutil) if viewer_url is not None: viewer_urls['ninja_log'] = viewer_url if args.log_url_json_file: with open(args.log_url_json_file, 'w') as f: f.write(json.dumps(viewer_urls)) if args.goma_stats_file: # MakeGomaExitStatusCounter should be callbed before # goma_utils.SendGomaStats, since SendGomaStats removes stats file. counter = goma_utils.MakeGomaExitStatusCounter( args.goma_stats_file, args.build_data_dir, builder=args.buildbot_buildername, master=args.buildbot_mastername, slave=args.buildbot_slavename, clobber=args.buildbot_clobber) if counter: tsmon_counters.append(counter) goma_utils.SendGomaStats(args.goma_stats_file, args.goma_crash_report_id_file, args.build_data_dir) if not args.skip_sendgomatsmon: # In the case of goma_start is failed, # we want log to investigate failed reason. # So, let me send some logs instead of # error in parse_args() using required option. assert args.json_status is not None and os.path.exists( args.json_status) counter = goma_utils.MakeGomaStatusCounter( args.json_status, args.ninja_log_exit_status, builder=args.buildbot_buildername, master=args.buildbot_mastername, slave=args.buildbot_slavename, clobber=args.buildbot_clobber) if counter: tsmon_counters.append(counter) if tsmon_counters: goma_utils.SendCountersToTsMon(tsmon_counters) return 0