def goma_teardown(options, env, exit_status, cloudtail_proc):
"""Tears down goma if necessary. """
if (options.compiler in ('goma', 'goma-clang') and options.goma_dir):
override_gsutil = None
if options.gsutil_py_path:
override_gsutil = [sys.executable, options.gsutil_py_path]
# If goma compiler_proxy crashes during the build, there could be crash
# dump.
if options.build_data_dir:
env['GOMACTL_CRASH_REPORT_ID_FILE'] = os.path.join(
options.build_data_dir, 'crash_report_id_file')
goma_ctl_cmd = [
sys.executable,
os.path.join(options.goma_dir, 'goma_ctl.py')
]
if options.goma_jsonstatus:
chromium_utils.RunCommand(goma_ctl_cmd +
['jsonstatus', options.goma_jsonstatus],
env=env)
goma_utils.SendGomaTsMon(options.goma_jsonstatus, exit_status)
# Always stop the proxy for now to allow in-place update.
chromium_utils.RunCommand(goma_ctl_cmd + ['stop'], env=env)
goma_utils.UploadGomaCompilerProxyInfo(override_gsutil=override_gsutil)
if env.get('GOMA_DUMP_STATS_FILE'):
goma_utils.SendGomaStats(env['GOMA_DUMP_STATS_FILE'],
env.get('GOMACTL_CRASH_REPORT_ID_FILE'),
options.build_data_dir)
if cloudtail_proc:
cloudtail_proc.terminate()
cloudtail_proc.wait()
def main():
parser = argparse.ArgumentParser(description='Upload goma related logs')
parser.add_argument('--upload-compiler-proxy-info',
action='store_true',
help='If set, the script will upload the latest '
'compiler_proxy.INFO.')
parser.add_argument('--ninja-log-outdir',
metavar='DIR',
help='Directory that has .ninja_log file.')
parser.add_argument('--ninja-log-compiler',
metavar='COMPILER',
help='compiler name used for the build.')
parser.add_argument('--ninja-log-command',
metavar='COMMAND',
help='command line options of the build.')
parser.add_argument('--ninja-log-exit-status',
type=int,
metavar='EXIT_STATUS',
help='ninja exit status.')
parser.add_argument('--goma-stats-file',
metavar='FILENAME',
help='Filename of a GomaStats binary protobuf. '
'If empty or non-existing file, it will report error '
'to chrome infra monitoring system.')
parser.add_argument('--goma-crash-report-id-file',
metavar='FILENAME',
help='Filename that has a crash report id.')
parser.add_argument(
'--build-data-dir',
metavar='DIR',
help='Directory that has build data used by event_mon.')
args = parser.parse_args()
if args.upload_compiler_proxy_info:
goma_utils.UploadGomaCompilerProxyInfo()
if args.ninja_log_outdir:
goma_utils.UploadNinjaLog(args.ninja_log_outdir,
args.ninja_log_compiler,
args.ninja_log_command,
args.ninja_log_exit_status)
if args.goma_stats_file:
goma_utils.SendGomaStats(args.goma_stats_file,
args.goma_crash_report_id_file,
args.build_data_dir)
return 0
def goma_setup(options, env):
"""Sets up goma if necessary.
If using the Goma compiler, first call goma_ctl to ensure the proxy is
available, and returns (True, instance of cloudtail subprocess).
If it failed to start up compiler_proxy, modify options.compiler and
options.goma_dir and returns (False, None)
"""
if options.compiler not in ('goma', 'goma-clang'):
# Unset goma_dir to make sure we'll not use goma.
options.goma_dir = None
return False, None
hostname = goma_utils.GetShortHostname()
# HACK(shinyak, yyanagisawa, goma): Windows NO_NACL_GOMA (crbug.com/390764)
# Building NaCl untrusted code using goma brings large performance
# improvement but it sometimes cause build failure by race condition.
# Let me enable goma build on goma canary buildslaves to confirm the issue
# has been fixed by a workaround.
# vm*-m4 are trybots. build*-m1 and vm*-m1 are all goma canary bots.
if hostname in [
'build28-m1', 'build58-m1', 'vm191-m1', 'vm480-m1', 'vm820-m1',
'vm821-m1', 'vm848-m1'
]:
env['NO_NACL_GOMA'] = 'false'
if options.goma_fail_fast:
# startup fails when initial ping failed.
env['GOMA_FAIL_FAST'] = 'true'
else:
# If a network error continues 30 minutes, compiler_proxy make the compile
# failed. When people use goma, they expect using goma is faster than
# compile locally. If goma cannot guarantee that, let it make compile
# as error.
env['GOMA_ALLOWED_NETWORK_ERROR_DURATION'] = '1800'
# HACK(yyanagisawa): reduce GOMA_BURST_MAX_PROCS crbug.com/592306
# Recently, I sometimes see buildbot slave time out, one possibility I come
# up with is burst mode use up resource.
# Let me temporary set small values to GOMA_BURST_MAX_PROCS to confirm
# the possibility is true or false.
max_subprocs = '3'
max_heavy_subprocs = '1'
number_of_processors = 0
try:
number_of_processors = multiprocessing.cpu_count()
except NotImplementedError:
print 'cpu_count() is not implemented, using default value.'
number_of_processors = 1
if number_of_processors > 3:
max_subprocs = str(number_of_processors - 1)
max_heavy_subprocs = str(number_of_processors / 2)
env['GOMA_BURST_MAX_SUBPROCS'] = max_subprocs
env['GOMA_BURST_MAX_SUBPROCS_LOW'] = max_subprocs
env['GOMA_BURST_MAX_SUBPROCS_HEAVY'] = max_heavy_subprocs
# Caches CRLs in GOMA_CACHE_DIR.
# Since downloading CRLs is usually slow, caching them may improves
# compiler_proxy start time.
if not os.path.exists(options.goma_cache_dir):
os.mkdir(options.goma_cache_dir, 0700)
env['GOMA_CACHE_DIR'] = options.goma_cache_dir
# Enable DepsCache. DepsCache caches the list of files to send goma server.
# This will greatly improve build speed when cache is warmed.
# The cache file is stored in the target output directory.
env['GOMA_DEPS_CACHE_DIR'] = (options.goma_deps_cache_dir
or options.target_output_dir)
if not env.get('GOMA_HERMETIC'):
env['GOMA_HERMETIC'] = options.goma_hermetic
if options.goma_enable_remote_link:
env['GOMA_ENABLE_REMOTE_LINK'] = options.goma_enable_remote_link
if options.goma_store_local_run_output:
env['GOMA_STORE_LOCAL_RUN_OUTPUT'] = options.goma_store_local_run_output
if options.goma_enable_compiler_info_cache:
# Will be stored in GOMA_CACHE_DIR.
env['GOMA_COMPILER_INFO_CACHE_FILE'] = 'goma-compiler-info.cache'
if options.build_data_dir:
env['GOMA_DUMP_STATS_FILE'] = os.path.join(options.build_data_dir,
'goma_stats_proto')
# goma is requested.
goma_key = os.path.join(options.goma_dir, 'goma.key')
if os.path.exists(goma_key):
env['GOMA_API_KEY_FILE'] = goma_key
if options.goma_service_account_json_file:
env['GOMA_SERVICE_ACCOUNT_JSON_FILE'] = \
options.goma_service_account_json_file
if chromium_utils.IsWindows():
env['GOMA_RPC_EXTRA_PARAMS'] = '?win'
goma_start_command = ['restart']
goma_ctl_cmd = [
sys.executable,
os.path.join(options.goma_dir, 'goma_ctl.py')
]
result = chromium_utils.RunCommand(goma_ctl_cmd + goma_start_command,
env=env)
if not result:
# goma started sucessfully.
# Making cloudtail to upload the latest log.
# TODO(yyanagisawa): install cloudtail from CIPD.
cloudtail_path = '/opt/infra-tools/cloudtail'
if chromium_utils.IsWindows():
cloudtail_path = 'C:\\infra-tools\\cloudtail'
try:
cloudtail_proc = subprocess.Popen([
cloudtail_path, 'tail', '--log-id', 'goma_compiler_proxy',
'--path',
goma_utils.GetLatestGomaCompilerProxyInfo()
])
except Exception as e:
print 'failed to invoke cloudtail: %s' % e
return True, None
return True, cloudtail_proc
if options.goma_jsonstatus:
chromium_utils.RunCommand(goma_ctl_cmd +
['jsonstatus', options.goma_jsonstatus],
env=env)
goma_utils.SendGomaTsMon(options.goma_jsonstatus, -1)
# Try to stop compiler_proxy so that it flushes logs and stores
# GomaStats.
if options.build_data_dir:
env['GOMACTL_CRASH_REPORT_ID_FILE'] = os.path.join(
options.build_data_dir, 'crash_report_id_file')
chromium_utils.RunCommand(goma_ctl_cmd + ['stop'], env=env)
override_gsutil = None
if options.gsutil_py_path:
override_gsutil = [sys.executable, options.gsutil_py_path]
# Upload compiler_proxy.INFO to investigate the reason of compiler_proxy
# start-up failure.
goma_utils.UploadGomaCompilerProxyInfo(override_gsutil=override_gsutil)
# Upload GomaStats to make it monitored.
if env.get('GOMA_DUMP_STATS_FILE'):
goma_utils.SendGomaStats(env['GOMA_DUMP_STATS_FILE'],
env.get('GOMACTL_CRASH_REPORT_ID_FILE'),
options.build_data_dir)
if options.goma_disable_local_fallback:
print 'error: failed to start goma; fallback has been disabled'
raise Exception('failed to start goma')
print 'warning: failed to start goma. falling back to non-goma'
# Drop goma from options.compiler
options.compiler = options.compiler.replace('goma-', '')
if options.compiler == 'goma':
options.compiler = None
# Reset options.goma_dir.
options.goma_dir = None
env['GOMA_DISABLED'] = '1'
return False, None
def main():
parser = argparse.ArgumentParser(description='Upload goma related logs')
parser.add_argument('--upload-compiler-proxy-info',
action='store_true',
help='If set, the script will upload the latest '
'compiler_proxy.INFO.')
parser.add_argument('--log-url-json-file',
help='If set, the script will write url of uploaded '
'log visualizer.')
parser.add_argument('--ninja-log-outdir',
metavar='DIR',
help='Directory that has .ninja_log file.')
parser.add_argument('--ninja-log-compiler',
metavar='COMPILER',
help='compiler name used for the build.')
parser.add_argument('--ninja-log-command',
metavar='COMMAND',
help='command line options of the build.')
parser.add_argument('--ninja-log-exit-status',
type=int,
metavar='EXIT_STATUS',
help='ninja exit status.')
parser.add_argument('--goma-stats-file',
metavar='FILENAME',
help='Filename of a GomaStats binary protobuf. '
'If empty or non-existing file, it will report error '
'to chrome infra monitoring system.')
parser.add_argument('--goma-crash-report-id-file',
metavar='FILENAME',
help='Filename that has a crash report id.')
parser.add_argument(
'--build-data-dir',
metavar='DIR',
help='Directory that has build data used by event_mon.')
parser.add_argument('--json-status',
metavar='JSON',
help='path of json file generated from'
' ./goma_ctl.py jsonstatus')
parser.add_argument('--skip-sendgomatsmon',
action='store_true',
help='Represent whether send jsonstatus'
' and goma or compile.py exit_status log to TsMon.'
' This option is used when no need to send goma status'
' to monitoring server.')
parser.add_argument(
'--gsutil-py-path',
help='Specify path to gsutil.py script in depot_tools.')
# Arguments set to os.environ
parser.add_argument('--buildbot-buildername',
default='unknown',
help='buildbot buildername')
parser.add_argument('--buildbot-mastername',
default='unknown',
help='buildbot mastername')
parser.add_argument('--buildbot-slavename',
default='unknown',
help='buildbot slavename')
parser.add_argument('--buildbot-clobber',
default='',
help='buildbot clobber')
args = parser.parse_args()
tsmon_counters = []
override_gsutil = None
if args.gsutil_py_path:
# Needs to add '--', otherwise gsutil options will be passed to gsutil.py.
override_gsutil = [sys.executable, args.gsutil_py_path, '--']
viewer_urls = {}
if args.upload_compiler_proxy_info:
viewer_url = goma_utils.UploadGomaCompilerProxyInfo(
builder=args.buildbot_buildername,
master=args.buildbot_mastername,
slave=args.buildbot_slavename,
clobber=args.buildbot_clobber,
override_gsutil=override_gsutil)
if viewer_url is not None:
viewer_urls['compiler_proxy_log'] = viewer_url
if args.ninja_log_outdir:
viewer_url = goma_utils.UploadNinjaLog(args.ninja_log_outdir,
args.ninja_log_compiler,
args.ninja_log_command,
args.ninja_log_exit_status,
override_gsutil=override_gsutil)
if viewer_url is not None:
viewer_urls['ninja_log'] = viewer_url
if args.log_url_json_file:
with open(args.log_url_json_file, 'w') as f:
f.write(json.dumps(viewer_urls))
if args.goma_stats_file:
# MakeGomaExitStatusCounter should be callbed before
# goma_utils.SendGomaStats, since SendGomaStats removes stats file.
counter = goma_utils.MakeGomaExitStatusCounter(
args.goma_stats_file,
args.build_data_dir,
builder=args.buildbot_buildername,
master=args.buildbot_mastername,
slave=args.buildbot_slavename,
clobber=args.buildbot_clobber)
if counter:
tsmon_counters.append(counter)
goma_utils.SendGomaStats(args.goma_stats_file,
args.goma_crash_report_id_file,
args.build_data_dir)
if not args.skip_sendgomatsmon:
# In the case of goma_start is failed,
# we want log to investigate failed reason.
# So, let me send some logs instead of
# error in parse_args() using required option.
assert args.json_status is not None and os.path.exists(
args.json_status)
counter = goma_utils.MakeGomaStatusCounter(
args.json_status,
args.ninja_log_exit_status,
builder=args.buildbot_buildername,
master=args.buildbot_mastername,
slave=args.buildbot_slavename,
clobber=args.buildbot_clobber)
if counter:
tsmon_counters.append(counter)
if tsmon_counters:
goma_utils.SendCountersToTsMon(tsmon_counters)
return 0