def ParseGomaConfig(goma_message, chroot_path):
"""Parse a goma config message."""
assert isinstance(goma_message, common_pb2.GomaConfig)
if not goma_message.goma_dir:
return None
# Parse the goma config.
chromeos_goma_dir = goma_message.chromeos_goma_dir or None
if goma_message.goma_approach == common_pb2.GomaConfig.RBE_STAGING:
goma_approach = goma_util.GomaApproach('?staging',
'staging-goma.chromium.org', True)
else:
goma_approach = goma_util.GomaApproach('?prod', 'goma.chromium.org', True)
# Note that we are not specifying the goma log_dir so that goma will create
# and use a tmp dir for the logs.
stats_filename = goma_message.stats_file or None
counterz_filename = goma_message.counterz_file or None
return goma_util.Goma(goma_message.goma_dir,
goma_message.goma_client_json,
stage_name='BuildAPI',
chromeos_goma_dir=chromeos_goma_dir,
chroot_dir=chroot_path,
goma_approach=goma_approach,
stats_filename=stats_filename,
counterz_filename=counterz_filename)
def testExtraEnvCustomChroot(self):
"""Test the chroot env building with a custom chroot location."""
goma_dir = os.path.join(self.tempdir, 'goma')
goma_client_json = os.path.join(self.tempdir, 'goma_client.json')
chroot_dir = os.path.join(self.tempdir, 'chroot')
chroot_tmp = os.path.join(chroot_dir, 'tmp')
osutils.Touch(goma_client_json)
osutils.SafeMakedirs(goma_dir)
osutils.SafeMakedirs(chroot_tmp)
goma = goma_util.Goma(goma_dir,
goma_client_json,
chroot_dir=chroot_dir)
env = goma.GetExtraEnv()
chroot_env = goma.GetChrootExtraEnv()
# Make sure the chroot paths got translated.
self.assertStartsWith(chroot_env['GOMA_TMP_DIR'], '/tmp')
self.assertStartsWith(chroot_env['GLOG_log_dir'], '/tmp')
# Make sure the non-chroot paths didn't get translated.
self.assertStartsWith(env['GOMA_TMP_DIR'], chroot_tmp)
self.assertStartsWith(env['GLOG_log_dir'], chroot_tmp)
# Make sure they're based on the same path.
self.assertEndsWith(env['GOMA_TMP_DIR'], chroot_env['GOMA_TMP_DIR'])
self.assertEndsWith(env['GLOG_log_dir'], chroot_env['GLOG_log_dir'])
def _SetupGomaIfNecessary(self):
"""Sets up goma envs if necessary.
Updates related env vars, and returns args to chroot.
Returns:
args which should be provided to chroot in order to enable goma.
If goma is unusable or disabled, None is returned.
"""
if not self._ShouldEnableGoma():
return None
# TODO(crbug.com/751010): Revisit to enable DepsCache for non-chrome-pfq
# bots, too.
use_goma_deps_cache = self._run.config.name.endswith('chrome-pfq')
goma = goma_util.Goma(
self._run.options.goma_dir,
self._run.options.goma_client_json,
stage_name=self.StageNamePrefix() if use_goma_deps_cache else None)
# Set USE_GOMA env var so that chrome is built with goma.
self._portage_extra_env['USE_GOMA'] = 'true'
self._portage_extra_env.update(goma.GetChrootExtraEnv())
# Keep GOMA_TMP_DIR for Report stage to upload logs.
self._run.attrs.metadata.UpdateWithDict(
{'goma_tmp_dir': goma.goma_tmp_dir})
# Mount goma directory and service account json file (if necessary)
# into chroot.
chroot_args = ['--goma_dir', goma.goma_dir]
if goma.goma_client_json:
chroot_args.extend(['--goma_client_json', goma.goma_client_json])
return chroot_args
def _UploadAndLinkGomaLogIfNecessary(stage_name, cbb_config_name, goma_dir,
goma_client_json, goma_tmp_dir):
"""Uploads the logs for goma, if needed. Also create a link to the visualizer.
If |goma_tmp_dir| is given, |goma_dir| and |goma_client_json| must not be
None.
Args:
stage_name: Name of the stage where goma is used.
cbb_config_name: Name of cbb_config used for the build.
goma_dir: Path to goma installed directory.
goma_client_json: Path to the service account json file.
goma_tmp_dir: Goma's working directory.
"""
if not goma_tmp_dir:
return
goma = goma_util.Goma(goma_dir,
goma_client_json,
goma_tmp_dir=goma_tmp_dir)
# Just in case, stop the goma. E.g. In case of timeout, we do not want to
# keep goma compiler_proxy running.
goma.Stop()
goma_urls = goma.UploadLogs(cbb_config_name)
if goma_urls:
for label, url in goma_urls:
logging.PrintBuildbotLink('%s %s' % (stage_name, label), url)
def PerformStage(self):
with osutils.TempDir(prefix='chrome-sdk-cache') as tempdir:
cache_dir = os.path.join(tempdir, 'cache')
extra_args = [
'--cwd', self.chrome_src, '--sdk-path', self.archive_path
]
# Do not automatically run 'gn gen', that will be done in _BuildChrome.
extra_args.extend(['--nogn-gen'])
if self._ShouldEnableGoma():
# TODO(crbug.com/751010): Revisit to enable DepsCache for
# non-chrome-pfq bots, too.
use_goma_deps_cache = self._run.config.name.endswith(
'chrome-pfq')
goma = goma_util.Goma(self._run.options.goma_dir,
self._run.options.goma_client_json,
stage_name=self.StageNamePrefix()
if use_goma_deps_cache else None)
extra_args.extend(
['--nostart-goma', '--gomadir', goma.goma_dir])
self._run.attrs.metadata.UpdateWithDict(
{'goma_tmp_dir_for_simple_chrome': goma.goma_tmp_dir})
else:
goma = None
sdk_cmd = commands.ChromeSDK(self._build_root,
self._current_board,
chrome_src=self.chrome_src,
goma=bool(goma),
extra_args=extra_args,
cache_dir=cache_dir)
self._BuildChrome(sdk_cmd, goma)
self._TestDeploy(sdk_cmd)
def SimpleChromeWorkflowTest(input_proto, _output_proto, _config):
"""Run SimpleChromeWorkflow tests."""
if input_proto.goma_config.goma_dir:
chromeos_goma_dir = input_proto.goma_config.chromeos_goma_dir or None
goma = goma_util.Goma(input_proto.goma_config.goma_dir,
input_proto.goma_config.goma_client_json,
stage_name='BuildApiTestSimpleChrome',
chromeos_goma_dir=chromeos_goma_dir)
else:
goma = None
return test.SimpleChromeWorkflowTest(input_proto.sysroot.path,
input_proto.sysroot.build_target.name,
input_proto.chrome_root, goma)
def testExtraEnvCustomChroot(self):
"""Test the chroot env building with a custom chroot location."""
goma_dir = os.path.join(self.tempdir, 'goma')
goma_client_json = os.path.join(self.tempdir, 'goma_client.json')
chroot_dir = os.path.join(self.tempdir, 'chroot')
chroot_tmp = os.path.join(chroot_dir, 'tmp')
log_dir = os.path.join(chroot_tmp, 'log_dir')
stats_filename = 'stats_filename'
counterz_filename = 'counterz_filename'
osutils.Touch(goma_client_json)
osutils.SafeMakedirs(goma_dir)
osutils.SafeMakedirs(chroot_tmp)
goma = goma_util.Goma(goma_dir, goma_client_json, chroot_dir=chroot_dir,
log_dir=log_dir, stats_filename=stats_filename,
counterz_filename=counterz_filename)
env = goma.GetExtraEnv()
chroot_env = goma.GetChrootExtraEnv()
# Make sure the chroot paths got translated.
self.assertStartsWith(chroot_env['GOMA_TMP_DIR'], '/tmp')
self.assertStartsWith(chroot_env['GLOG_log_dir'], '/tmp/log_dir')
# Make sure the non-chroot paths didn't get translated.
self.assertStartsWith(env['GOMA_TMP_DIR'], chroot_tmp)
self.assertStartsWith(env['GLOG_log_dir'], log_dir)
# Make sure they're based on the same path.
self.assertEndsWith(env['GOMA_TMP_DIR'], chroot_env['GOMA_TMP_DIR'])
self.assertEndsWith(env['GLOG_log_dir'], chroot_env['GLOG_log_dir'])
# Make sure the stats file gets set correctly.
self.assertStartsWith(env['GOMA_DUMP_STATS_FILE'], log_dir)
self.assertEndsWith(env['GOMA_DUMP_STATS_FILE'], stats_filename)
self.assertStartsWith(chroot_env['GOMA_DUMP_STATS_FILE'], '/tmp/log_dir')
self.assertEndsWith(chroot_env['GOMA_DUMP_STATS_FILE'], stats_filename)
self.assertEndsWith(env['GOMA_DUMP_STATS_FILE'],
chroot_env['GOMA_DUMP_STATS_FILE'])
# Make sure the counterz file gets set correctly.
self.assertStartsWith(env['GOMA_DUMP_COUNTERZ_FILE'], log_dir)
self.assertEndsWith(env['GOMA_DUMP_COUNTERZ_FILE'], counterz_filename)
self.assertStartsWith(chroot_env['GOMA_DUMP_COUNTERZ_FILE'], '/tmp/log_dir')
self.assertEndsWith(chroot_env['GOMA_DUMP_COUNTERZ_FILE'],
counterz_filename)
self.assertEndsWith(env['GOMA_DUMP_COUNTERZ_FILE'],
chroot_env['GOMA_DUMP_COUNTERZ_FILE'])
def testExtraEnvGomaApproach(self):
"""Test the chroot env building with a goma approach."""
goma_dir = os.path.join(self.tempdir, 'goma')
goma_client_json = os.path.join(self.tempdir, 'goma_client.json')
chroot_dir = os.path.join(self.tempdir, 'chroot')
chroot_tmp = os.path.join(chroot_dir, 'tmp')
osutils.Touch(goma_client_json)
osutils.SafeMakedirs(goma_dir)
osutils.SafeMakedirs(chroot_tmp)
goma_approach = goma_util.GomaApproach('foo', 'bar', True)
goma = goma_util.Goma(goma_dir, goma_client_json,
chroot_dir=chroot_dir, goma_approach=goma_approach)
env = goma.GetExtraEnv()
# Make sure the extra environment specified by goma_approach is present.
self.assertEqual(env['GOMA_RPC_EXTRA_PARAMS'], 'foo')
self.assertEqual(env['GOMA_SERVER_HOST'], 'bar')
self.assertEqual(env['GOMA_ARBITRARY_TOOLCHAIN_SUPPORT'], 'true')
def testSimpleChromeWorkflowTest(self):
goma_test_dir = os.path.join(self.tempdir, 'goma_test_dir')
goma_test_json_string = os.path.join(self.tempdir,
'goma_json_string.txt')
chromeos_goma_dir = os.path.join(self.tempdir, 'chromeos_goma_dir')
goma_config = common_pb2.GomaConfig(
goma_dir=goma_test_dir, goma_client_json=goma_test_json_string)
osutils.SafeMakedirs(goma_test_dir)
osutils.SafeMakedirs(chromeos_goma_dir)
osutils.Touch(goma_test_json_string)
goma = goma_util.Goma(goma_config.goma_dir,
goma_config.goma_client_json,
stage_name='BuildApiTestSimpleChrome',
chromeos_goma_dir=chromeos_goma_dir)
mock_goma_log_dir = os.path.join(self.tempdir, 'goma_log_dir')
osutils.SafeMakedirs(mock_goma_log_dir)
goma.goma_log_dir = mock_goma_log_dir
# For this test, we avoid running test._VerifySDKEnvironment because use of
# other mocks prevent creating the SDK dir that _VerifySDKEnvironment checks
# for
self.PatchObject(test, '_VerifySDKEnvironment')
self.PatchObject(os.path, 'exists', return_value=True)
ninja_cmd = self.PatchObject(commands.ChromeSDK,
'GetNinjaCommand',
return_value='ninja command')
test.SimpleChromeWorkflowTest(self.sysroot_path, self.build_target,
self.chrome_root, goma)
# Verify ninja_cmd calls.
ninja_calls = [mock.call(), mock.call(debug=False)]
ninja_cmd.assert_has_calls(ninja_calls)
# Verify calls with args to chrome_sdk_run made by service/test.py.
gn_dir = os.path.join(self.chrome_root, 'buildtools/linux64/gn')
board_out_dir = os.path.join(self.chrome_root, 'out_board/Release')
self.chrome_sdk_run_mock.assert_any_call(['gclient', 'runhooks'])
self.chrome_sdk_run_mock.assert_any_call(['true'])
self.chrome_sdk_run_mock.assert_any_call([
'bash', '-c',
('%s gen "%s" --args="$GN_ARGS"' % (gn_dir, board_out_dir))
])
self.chrome_sdk_run_mock.assert_any_call(['env', '--null'],
run_args=mock.ANY)
self.chrome_sdk_run_mock.assert_any_call('ninja command',
run_args=mock.ANY)
# Create expected paths from constants so that the tests work inside or
# outside the SDK.
deploy_chrome_path = os.path.join(constants.SOURCE_ROOT,
constants.CHROMITE_BIN_SUBDIR,
'deploy_chrome')
image_dir_symlink = image_lib.GetLatestImageLink(self.build_target)
image_path = os.path.join(image_dir_symlink, constants.VM_IMAGE_BIN)
self.chrome_sdk_run_mock.assert_any_call([
deploy_chrome_path, '--build-dir', board_out_dir, '--staging-only',
'--staging-dir', mock.ANY
])
self.chrome_sdk_run_mock.assert_any_call([
'cros_run_test', '--copy-on-write', '--deploy', '--board=board',
('--image-path=%s' % (image_path)), '--build-dir=out_board/Release'
])
# Verify goma mock was started and stopped.
self.goma_mock.Start.assert_called_once()
self.goma_mock.Stop.assert_called_once()
