def testArrays(self):
env_dict = osutils.SourceEnvironment(self.env_file, ('ENVA', ))
self.assertEquals(env_dict, {'ENVA': 'a b c,d,e 1234 %'})
env_dict = osutils.SourceEnvironment(self.env_file, ('ENVA', ),
ifs=' ')
self.assertEquals(env_dict, {'ENVA': 'a b c d e 1234 %'})
def testArrays(self):
env_dict = osutils.SourceEnvironment(self.env_file, ('ENVA',))
self.assertEquals(env_dict, {'ENVA': 'a b c,d,e 1234 %'})
env_dict = osutils.SourceEnvironment(self.env_file, ('ENVA',), ifs=' ')
self.assertEquals(env_dict, {'ENVA': 'a b c d e 1234 %'})
env_dict = osutils.SourceEnvironment(self.env_file_multiline, ('ENVM',),
multiline=True)
self.assertEquals(env_dict, {'ENVM': 'gentil\nmechant'})
def DetermineAndroidBranch(self, board):
"""Returns the Android branch in use by the active container ebuild."""
try:
android_package = self.DetermineAndroidPackage(board)
except cros_build_lib.RunCommandError:
raise NoAndroidBranchError(
'Android branch could not be determined for %s' % board)
if not android_package:
raise NoAndroidBranchError(
'Android branch could not be determined for %s (no package?)' %
board)
ebuild_path = portage_util.FindEbuildForBoardPackage(
android_package, board)
host_ebuild_path = path_util.FromChrootPath(ebuild_path)
# We assume all targets pull from the same branch and that we always
# have at least one of the following targets.
targets = constants.ANDROID_ALL_BUILD_TARGETS
ebuild_content = osutils.SourceEnvironment(host_ebuild_path, targets)
for target in targets:
if target in ebuild_content:
branch = re.search(r'(.*?)-linux-', ebuild_content[target])
if branch is not None:
return branch.group(1)
raise NoAndroidBranchError(
'Android branch could not be determined for %s (ebuild empty?)' %
board)
def _VerifyGoma(user_rc):
"""Verify that the user's goma installation is working.
Arguments:
user_rc: User-supplied rc file.
"""
user_env = osutils.SourceEnvironment(user_rc, ['PATH'], env=True)
goma_ctl = osutils.Which('goma_ctl.sh', user_env.get('PATH'))
if goma_ctl is not None:
manifest = os.path.join(os.path.dirname(goma_ctl), 'MANIFEST')
platform_env = osutils.SourceEnvironment(manifest, ['PLATFORM'])
platform = platform_env.get('PLATFORM')
if platform is not None and platform != 'chromeos':
cros_build_lib.Warning('Found %s version of Goma in PATH.',
platform)
cros_build_lib.Warning('Goma will not work')
def DetermineAndroidBranch(self, package):
"""Returns the Android branch in use by the active container ebuild.
Workspace version of cbuildbot_run.DetermineAndroidBranch().
Args:
package: String name of Android package to get branch of.
Returns:
String with the android container branch name.
"""
ebuild_path = portage_util.FindEbuildForBoardPackage(
package, self._current_board, buildroot=self._build_root)
host_ebuild_path = path_util.FromChrootPath(ebuild_path,
source_path=self._build_root)
# We assume all targets pull from the same branch and that we always
# have an ARM_TARGET or an X86_USERDEBUG_TARGET.
targets = ['ARM_TARGET', 'X86_USERDEBUG_TARGET']
ebuild_content = osutils.SourceEnvironment(host_ebuild_path, targets)
logging.info('Got ebuild env: %s', ebuild_content)
for target in targets:
if target in ebuild_content:
branch = re.search(r'(.*?)-linux-', ebuild_content[target])
if branch is not None:
return branch.group(1)
raise cbuildbot_run.NoAndroidBranchError(
'Android branch could not be determined for %s (ebuild empty?)' %
ebuild_path)
def GetSourcePath(self, srcroot):
"""Get the project and path for this ebuild.
The path is guaranteed to exist, be a directory, and be absolute.
"""
workon_vars = (
'CROS_WORKON_LOCALNAME',
'CROS_WORKON_PROJECT',
'CROS_WORKON_SUBDIR',
)
env = {
'CROS_WORKON_LOCALNAME': self._pkgname,
'CROS_WORKON_PROJECT': self._pkgname,
'CROS_WORKON_SUBDIR': '',
}
settings = osutils.SourceEnvironment(self._unstable_ebuild_path,
workon_vars, env=env)
localnames = settings['CROS_WORKON_LOCALNAME'].split(',')
projects = settings['CROS_WORKON_PROJECT'].split(',')
subdirs = settings['CROS_WORKON_SUBDIR'].split(',')
# Sanity checks and completion.
# Each project specification has to have the same amount of items.
if len(projects) != len(localnames):
raise EbuildFormatIncorrectException(self._unstable_ebuild_path,
'Number of _PROJECT and _LOCALNAME items don\'t match.')
# Subdir must be either 0,1 or len(project)
if len(projects) != len(subdirs) and len(subdirs) > 1:
raise EbuildFormatIncorrectException(self._unstable_ebuild_path,
'Incorrect number of _SUBDIR items.')
# If there's one, apply it to all.
if len(subdirs) == 1:
subdirs = subdirs * len(projects)
# If there is none, make an empty list to avoid exceptions later.
if len(subdirs) == 0:
subdirs = [''] * len(projects)
# Calculate srcdir.
if self._category == 'chromeos-base':
dir_ = 'platform'
else:
dir_ = 'third_party'
subdir_paths = [os.path.realpath(os.path.join(srcroot, dir_, l, s))
for l, s in zip(localnames, subdirs)]
for subdir_path, project in zip(subdir_paths, projects):
if not os.path.isdir(subdir_path):
cros_build_lib.Die('Source repository %s '
'for project %s does not exist.' % (subdir_path,
self._pkgname))
# Verify that we're grabbing the commit id from the right project name.
real_project = self.GetGitProjectName(subdir_path)
if project != real_project:
cros_build_lib.Die('Project name mismatch for %s '
'(found %s, expected %s)' % (subdir_path,
real_project,
project))
return projects, subdir_paths
def GetStandardField(self, field):
"""Returns the value of a standard field.
Args:
field: Field from the standard configuration file to get.
One of STANDARD_FIELD_* from above.
"""
return osutils.SourceEnvironment(self._config_file, [field],
multiline=True).get(field)
def _GetRepoCheckoutVersion(self, repo_root):
"""Get the version specified in chromeos_version.sh.
Returns:
Version number in format '3929.0.0'.
"""
chromeos_version_sh = os.path.join(repo_root, constants.VERSION_FILE)
sourced_env = osutils.SourceEnvironment(chromeos_version_sh,
['CHROMEOS_VERSION_STRING'],
env={'CHROMEOS_OFFICIAL': '1'})
return sourced_env['CHROMEOS_VERSION_STRING']
def _VerifyGoma(user_rc):
"""Verify that the user has no goma installations set up in user_rc.
If the user does have a goma installation set up, verify that it's for
ChromeOS.
Arguments:
user_rc: User-supplied rc file.
"""
user_env = osutils.SourceEnvironment(user_rc, ['PATH'])
goma_ctl = osutils.Which('goma_ctl.sh', user_env.get('PATH'))
if goma_ctl is not None:
logging.warning(
'%s is adding Goma to the PATH. Using that Goma instead of the '
'managed Goma install.', user_rc)
manifest = os.path.join(os.path.dirname(goma_ctl), 'MANIFEST')
platform_env = osutils.SourceEnvironment(manifest, ['PLATFORM'])
platform = platform_env.get('PLATFORM')
if platform is not None and platform != 'chromeos':
logging.warning('Found %s version of Goma in PATH.', platform)
logging.warning('Goma will not work')
def _VerifyChromiteBin(user_rc):
"""Verify that the user has not set a chromite bin/ dir in user_rc.
Args:
user_rc: User-supplied rc file.
"""
user_env = osutils.SourceEnvironment(user_rc, ['PATH'])
chromite_bin = osutils.Which('parallel_emerge', user_env.get('PATH'))
if chromite_bin is not None:
logging.warning(
'%s is adding chromite/bin to the PATH. Remove it from the PATH to '
'use the the default Chromite.', user_rc)
def _VerifyClang(user_rc):
"""Verify that the user has not set a clang bin/ dir in user_rc.
Arguments:
user_rc: User-supplied rc file.
"""
user_env = osutils.SourceEnvironment(user_rc, ['PATH'])
clang_bin = osutils.Which('clang', user_env.get('PATH'))
if clang_bin is not None:
clang_dir = os.path.dirname(clang_bin)
if not osutils.Which('goma_ctl.sh', clang_dir):
logging.warning(
'%s is adding Clang to the PATH. Because of this, Goma is being '
'bypassed. Remove it from the PATH to use Goma with the default '
'Clang.', user_rc)
def _VerifyGoma(user_rc):
"""Verify that the user has no goma installations set up in user_rc.
If the user does have a goma installation set up, verify that it's for
ChromeOS.
Args:
user_rc: User-supplied rc file.
"""
user_env = osutils.SourceEnvironment(user_rc, ['PATH'])
goma_ctl = osutils.Which('goma_ctl.py', user_env.get('PATH'))
if goma_ctl is not None:
logging.warning(
'%s is adding Goma to the PATH. Using that Goma instead of the '
'managed Goma install.', user_rc)
def GetCachedField(self, field):
"""Returns the value of |field| in the sysroot cache file.
Access to the cache is thread-safe as long as we access it through this
methods or the bash helper in common.sh.
Args:
field: name of the field.
"""
if not os.path.exists(self._cache_file):
return None
with locking.FileLock(
self._cache_file_lock, locktype=locking.FLOCK,
world_writable=True).read_lock():
return osutils.SourceEnvironment(self._cache_file, [field]).get(field)
def _StripBinContext(options):
if not options.dostrip:
yield None
elif options.strip_bin:
yield options.strip_bin
else:
sdk = cros_chrome_sdk.SDKFetcher(options.cache_dir, options.board)
components = (sdk.TARGET_TOOLCHAIN_KEY, constants.CHROME_ENV_TAR)
with sdk.Prepare(components=components) as ctx:
env_path = os.path.join(ctx.key_map[constants.CHROME_ENV_TAR].path,
constants.CHROME_ENV_FILE)
strip_bin = osutils.SourceEnvironment(env_path, ['STRIP'])['STRIP']
strip_bin = os.path.join(
ctx.key_map[sdk.TARGET_TOOLCHAIN_KEY].path, 'bin',
os.path.basename(strip_bin))
yield strip_bin
def _GetDLCInfo(device, pkg_path, from_dut):
"""Returns information of a DLC given its package path.
Args:
device: commandline.Device object; None to use the default device.
pkg_path: path to the package.
from_dut: True if extracting DLC info from DUT, False if extracting DLC
info from host.
Returns:
A tuple (dlc_id, dlc_package).
"""
environment_content = ''
if from_dut:
# On DUT, |pkg_path| is the directory which contains environment file.
environment_path = os.path.join(pkg_path, _ENVIRONMENT_FILENAME)
result = device.run(['test', '-f', environment_path],
check=False,
encoding=None)
if result.returncode == 1:
# The package is not installed on DUT yet. Skip extracting info.
return None, None
result = device.run(['bzip2', '-d', '-c', environment_path],
encoding=None)
environment_content = result.output
else:
# On host, pkg_path is tbz2 file which contains environment file.
# Extract the metadata of the package file.
data = portage.xpak.tbz2(pkg_path).get_data()
# Extract the environment metadata.
environment_content = bz2.decompress(
data[_ENVIRONMENT_FILENAME.encode('utf-8')])
with tempfile.NamedTemporaryFile() as f:
# Dumps content into a file so we can use osutils.SourceEnvironment.
path = os.path.realpath(f.name)
osutils.WriteFile(path, environment_content, mode='wb')
content = osutils.SourceEnvironment(
path, (_DLC_ID, _DLC_PACKAGE, _DLC_ENABLED))
dlc_enabled = content.get(_DLC_ENABLED)
if dlc_enabled is not None and (dlc_enabled is False
or str(dlc_enabled) == 'false'):
logging.info('Installing DLC in rootfs.')
return None, None
return content.get(_DLC_ID), content.get(_DLC_PACKAGE)
def determine_android_branch(board):
"""Returns the Android branch in use by the active container ebuild."""
try:
android_package = determine_android_package(board)
except cros_build_lib.RunCommandError:
raise NoAndroidBranchError(
'Android branch could not be determined for %s' % board)
if not android_package:
return None
ebuild_path = portage_util.FindEbuildForBoardPackage(
android_package, board)
# We assume all targets pull from the same branch and that we always
# have an ARM_TARGET, ARM_USERDEBUG_TARGET, or an X86_USERDEBUG_TARGET.
targets = ['ARM_TARGET', 'ARM_USERDEBUG_TARGET', 'X86_USERDEBUG_TARGET']
ebuild_content = osutils.SourceEnvironment(ebuild_path, targets)
for target in targets:
if target in ebuild_content:
branch = re.search(r'(.*?)-linux-', ebuild_content[target])
if branch is not None:
return branch.group(1)
raise NoAndroidBranchError(
'Android branch could not be determined for %s (ebuild empty?)' %
board)
def _SetupEnvironment(self, board, version, key_map):
"""Sets environment variables to export to the SDK shell."""
sysroot = key_map[constants.CHROME_SYSROOT_TAR].path
environment = os.path.join(key_map[constants.CHROME_ENV_TAR].path,
'environment')
target_tc = key_map[self.sdk.TARGET_TOOLCHAIN_KEY].path
env = osutils.SourceEnvironment(environment, self.EBUILD_ENV)
os.environ[self.sdk.SDK_VERSION_ENV] = version
os.environ[self.sdk.SDK_BOARD_ENV] = board
# SYSROOT is necessary for Goma and the sysroot wrapper.
env['SYSROOT'] = sysroot
gyp_dict = chrome_util.ProcessGypDefines(env['GYP_DEFINES'])
gyp_dict['sysroot'] = sysroot
gyp_dict.pop('order_text_section', None)
env['GYP_DEFINES'] = chrome_util.DictToGypDefines(gyp_dict)
for tc_path in (target_tc, ):
env['PATH'] = '%s:%s' % (os.path.join(tc_path,
'bin'), os.environ['PATH'])
for var in ('CXX', 'CC', 'LD'):
env[var] = self._FixGoldPath(env[var], target_tc)
env['CXX_host'] = 'g++'
env['CC_host'] = 'gcc'
# PS1 sets the command line prompt and xterm window caption.
env['PS1'] = self._CreatePS1(self.board, version)
out_dir = 'out_%s' % self.board
env['builddir_name'] = out_dir
env['GYP_GENERATORS'] = 'ninja'
env['GYP_GENERATOR_FLAGS'] = 'output_dir=%s' % out_dir
return env
def _PrepareStagingDir(options, tempdir, staging_dir):
"""Place the necessary files in the staging directory.
The staging directory is the directory used to rsync the build artifacts over
to the device. Only the necessary Chrome build artifacts are put into the
staging directory.
"""
if options.build_dir:
sdk = cros_chrome_sdk.SDKFetcher(options.cache_dir, options.board)
components = (sdk.TARGET_TOOLCHAIN_KEY, constants.CHROME_ENV_TAR)
with sdk.Prepare(components=components) as ctx:
env_path = os.path.join(ctx.key_map[constants.CHROME_ENV_TAR].path,
constants.CHROME_ENV_FILE)
strip_bin = osutils.SourceEnvironment(env_path, ['STRIP'])['STRIP']
strip_bin = os.path.join(
ctx.key_map[sdk.TARGET_TOOLCHAIN_KEY].path, 'bin',
os.path.basename(strip_bin))
chrome_util.StageChromeFromBuildDir(
staging_dir,
options.build_dir,
strip_bin,
strict=options.strict,
sloppy=options.sloppy,
gyp_defines=options.gyp_defines,
staging_flags=options.staging_flags)
else:
pkg_path = options.local_pkg_path
if options.gs_path:
pkg_path = _FetchChromePackage(options.cache_dir, tempdir,
options.gs_path)
assert pkg_path
logging.info('Extracting %s...', pkg_path)
osutils.SafeMakedirs(staging_dir)
cros_build_lib.DebugRunCommand(['tar', '-xpf', pkg_path],
cwd=staging_dir)
def _SetupEnvironment(self,
board,
sdk_ctx,
options,
goma_dir=None,
goma_port=None):
"""Sets environment variables to export to the SDK shell."""
if options.chroot:
sysroot = os.path.join(options.chroot, 'build', board)
if not os.path.isdir(sysroot) and not options.cmd:
logging.warning(
"Because --chroot is set, expected a sysroot to be at "
"%s, but couldn't find one.", sysroot)
else:
sysroot = sdk_ctx.key_map[constants.CHROME_SYSROOT_TAR].path
environment = os.path.join(
sdk_ctx.key_map[constants.CHROME_ENV_TAR].path, 'environment')
env = osutils.SourceEnvironment(environment, self.EBUILD_ENV)
self._SetupTCEnvironment(sdk_ctx, options, env, goma_dir=goma_dir)
# Add managed components to the PATH.
env['PATH'] = '%s:%s' % (constants.CHROMITE_BIN_DIR, env['PATH'])
# Export internally referenced variables.
os.environ[self.sdk.SDK_VERSION_ENV] = sdk_ctx.version
os.environ[self.sdk.SDK_BOARD_ENV] = board
# Export the board/version info in a more accessible way, so developers can
# reference them in their chrome_sdk.bashrc files, as well as within the
# chrome-sdk shell.
for var in [self.sdk.SDK_VERSION_ENV, self.sdk.SDK_BOARD_ENV]:
env[var.lstrip('%')] = os.environ[var]
# Export Goma information.
if goma_dir:
env[self.SDK_GOMA_DIR_ENV] = goma_dir
env[self.SDK_GOMA_PORT_ENV] = goma_port
# SYSROOT is necessary for Goma and the sysroot wrapper.
env['SYSROOT'] = sysroot
gyp_dict = chrome_util.ProcessGypDefines(env['GYP_DEFINES'])
gyp_dict['sysroot'] = sysroot
gyp_dict.pop('order_text_section', None)
if options.clang:
gyp_dict['clang'] = 1
gyp_dict['werror'] = ''
gyp_dict['clang_use_chrome_plugins'] = 0
gyp_dict['linux_use_tcmalloc'] = 0
if options.internal:
gyp_dict['branding'] = 'Chrome'
gyp_dict['buildtype'] = 'Official'
else:
gyp_dict.pop('branding', None)
gyp_dict.pop('buildtype', None)
env['GYP_DEFINES'] = chrome_util.DictToGypDefines(gyp_dict)
# PS1 sets the command line prompt and xterm window caption.
env['PS1'] = self._CreatePS1(self.board,
self.sdk.GetFullVersion(sdk_ctx.version),
chroot=options.chroot)
out_dir = 'out_%s' % self.board
env['builddir_name'] = out_dir
env['GYP_GENERATORS'] = 'make' if options.make else 'ninja'
env['GYP_GENERATOR_FLAGS'] = 'output_dir=%s' % out_dir
return env
def _SetupEnvironment(self, board, sdk_ctx, options, goma_dir=None,
goma_port=None):
"""Sets environment variables to export to the SDK shell."""
if options.chroot:
sysroot = os.path.join(options.chroot, 'build', board)
if not os.path.isdir(sysroot) and not options.cmd:
logging.warning("Because --chroot is set, expected a sysroot to be at "
"%s, but couldn't find one.", sysroot)
else:
sysroot = sdk_ctx.key_map[constants.CHROME_SYSROOT_TAR].path
environment = os.path.join(sdk_ctx.key_map[constants.CHROME_ENV_TAR].path,
'environment')
if options.chroot:
# Override with the environment from the chroot if available (i.e.
# build_packages or emerge chromeos-chrome has been run for |board|).
env_path = os.path.join(sysroot, 'var', 'db', 'pkg', 'chromeos-base',
'chromeos-chrome-*')
env_glob = glob.glob(env_path)
if len(env_glob) != 1:
logging.warning('Multiple Chrome versions in %s. This can be resolved'
' by running "eclean-$BOARD -d packages". Using'
' environment from: %s', env_path, environment)
elif not os.path.isdir(env_glob[0]):
logging.warning('Environment path not found: %s. Using enviroment from:'
' %s.', env_path, environment)
else:
chroot_env_file = os.path.join(env_glob[0], 'environment.bz2')
if os.path.isfile(chroot_env_file):
# Log a warning here since this is new behavior that is not obvious.
logging.notice('Environment fetched from: %s', chroot_env_file)
# Uncompress enviornment.bz2 to pass to osutils.SourceEnvironment.
chroot_cache = os.path.join(options.cache_dir, COMMAND_NAME, 'chroot')
osutils.SafeMakedirs(chroot_cache)
environment = os.path.join(chroot_cache, 'environment_%s' % board)
cros_build_lib.UncompressFile(chroot_env_file, environment)
env = osutils.SourceEnvironment(environment, self.EBUILD_ENV)
gn_args = gn_helpers.FromGNArgs(env['GN_ARGS'])
self._SetupTCEnvironment(sdk_ctx, options, env, gn_args['is_clang'])
# Add managed components to the PATH.
env['PATH'] = '%s:%s' % (constants.CHROMITE_BIN_DIR, env['PATH'])
env['PATH'] = '%s:%s' % (os.path.dirname(self.sdk.gs_ctx.gsutil_bin),
env['PATH'])
# Export internally referenced variables.
os.environ[self.sdk.SDK_BOARD_ENV] = board
if options.sdk_path:
os.environ[self.sdk.SDK_PATH_ENV] = options.sdk_path
os.environ[self.sdk.SDK_VERSION_ENV] = sdk_ctx.version
# Add board and sdk version as gn args so that tests can bind them in
# test wrappers generated at compile time.
gn_args['cros_board'] = board
gn_args['cros_sdk_version'] = sdk_ctx.version
# Export the board/version info in a more accessible way, so developers can
# reference them in their chrome_sdk.bashrc files, as well as within the
# chrome-sdk shell.
for var in [self.sdk.SDK_VERSION_ENV, self.sdk.SDK_BOARD_ENV]:
env[var.lstrip('%')] = os.environ[var]
# Export Goma information.
if goma_dir:
env[self.SDK_GOMA_DIR_ENV] = goma_dir
if goma_port:
env[self.SDK_GOMA_PORT_ENV] = goma_port
# SYSROOT is necessary for Goma and the sysroot wrapper.
env['SYSROOT'] = sysroot
# Deprecated options warnings. TODO(stevenjb): Eliminate these entirely
# once removed from any builders.
if options.component:
logging.warning('--component is deprecated, ignoring')
if options.fastbuild:
logging.warning('--fastbuild is deprecated, ignoring')
gn_args['target_sysroot'] = sysroot
gn_args.pop('pkg_config', None)
# pkg_config only affects the target and comes from the sysroot.
# host_pkg_config is used for programs compiled for use later in the build.
gn_args['host_pkg_config'] = 'pkg-config'
if options.clang:
gn_args['is_clang'] = True
if options.internal:
gn_args['is_chrome_branded'] = True
gn_args['is_official_build'] = True
else:
gn_args.pop('is_chrome_branded', None)
gn_args.pop('is_official_build', None)
gn_args.pop('internal_gles2_conform_tests', None)
# For SimpleChrome, we use the binutils that comes bundled within Chrome.
# We should not use the binutils from the host system.
gn_args['linux_use_bundled_binutils'] = True
# Need to reset these after the env vars have been fixed by
# _SetupTCEnvironment.
gn_args['cros_host_is_clang'] = True
# v8 snapshot is built on the host, so we need to set this.
# See crosbug/618346.
gn_args['cros_v8_snapshot_is_clang'] = True
#
gn_args['cros_target_cc'] = env['CC']
gn_args['cros_target_cxx'] = env['CXX']
gn_args['cros_target_ld'] = env['LD']
gn_args['cros_target_extra_cflags'] = env.get('CFLAGS', '')
gn_args['cros_target_extra_cxxflags'] = env.get('CXXFLAGS', '')
gn_args['cros_host_cc'] = env['CC_host']
gn_args['cros_host_cxx'] = env['CXX_host']
gn_args['cros_host_ld'] = env['LD_host']
gn_args['cros_host_ar'] = env['AR_host']
gn_args['cros_v8_snapshot_cc'] = env['CC_host']
gn_args['cros_v8_snapshot_cxx'] = env['CXX_host']
gn_args['cros_v8_snapshot_ld'] = env['LD_host']
gn_args['cros_v8_snapshot_ar'] = env['AR_host']
# No need to adjust CFLAGS and CXXFLAGS for GN since the only
# adjustment made in _SetupTCEnvironment is for split debug which
# is done with 'use_debug_fission'.
# Enable goma if requested.
if goma_dir:
gn_args['use_goma'] = True
gn_args['goma_dir'] = goma_dir
elif not options.goma:
# If --nogoma option is explicitly set, disable goma, even if it is
# used in the original GN_ARGS.
gn_args['use_goma'] = False
gn_args.pop('internal_khronos_glcts_tests', None) # crbug.com/588080
# Disable ThinLTO and CFI for simplechrome. Tryjob machines do not have
# enough file descriptors to use. crbug.com/789607
if 'use_thin_lto' in gn_args:
gn_args['use_thin_lto'] = False
if 'is_cfi' in gn_args:
gn_args['is_cfi'] = False
if 'use_cfi_cast' in gn_args:
gn_args['use_cfi_cast'] = False
# We need to remove the flag below from cros_target_extra_ldflags.
# The format of ld flags is something like
# '-Wl,-O1 -Wl,-O2 -Wl,--as-needed -stdlib=libc++'
extra_thinlto_flag = '-Wl,-plugin-opt,-import-instr-limit=30'
extra_ldflags = gn_args.get('cros_target_extra_ldflags', '')
if extra_thinlto_flag in extra_ldflags:
gn_args['cros_target_extra_ldflags'] = extra_ldflags.replace(
extra_thinlto_flag, '')
# We removed webcore debug symbols on release builds on arm.
# See crbug.com/792999. However, we want to keep the symbols
# for simplechrome builds.
# TODO: remove the 'remove_webcore_debug_symbols' once we
# change the ebuild file.
gn_args['remove_webcore_debug_symbols'] = False
gn_args['blink_symbol_level'] = -1
if options.gn_extra_args:
gn_args.update(gn_helpers.FromGNArgs(options.gn_extra_args))
gn_args_env = gn_helpers.ToGNString(gn_args)
env['GN_ARGS'] = gn_args_env
# PS1 sets the command line prompt and xterm window caption.
full_version = sdk_ctx.version
if full_version != CUSTOM_VERSION:
full_version = self.sdk.GetFullVersion(sdk_ctx.version)
env['PS1'] = self._CreatePS1(self.board, full_version,
chroot=options.chroot)
# Set the useful part of PS1 for users with a custom PROMPT_COMMAND.
env['CROS_PS1_PREFIX'] = self._PS1Prefix(self.board, full_version,
chroot=options.chroot)
out_dir = 'out_%s' % self.board
env['builddir_name'] = out_dir
build_label = 'Release'
# This is used by landmines.py to prevent collisions when building both
# chromeos and android from shared source.
# For context, see crbug.com/407417
env['CHROMIUM_OUT_DIR'] = os.path.join(options.chrome_src, out_dir)
self._UpdateGnArgsIfStale(out_dir, build_label, gn_args, env['SDK_BOARD'])
return env
def _SetupEnvironment(self,
board,
sdk_ctx,
options,
goma_dir=None,
goma_port=None):
"""Sets environment variables to export to the SDK shell."""
if options.chroot:
sysroot = os.path.join(options.chroot, 'build', board)
if not os.path.isdir(sysroot) and not options.cmd:
logging.warning(
"Because --chroot is set, expected a sysroot to be at "
"%s, but couldn't find one.", sysroot)
else:
sysroot = sdk_ctx.key_map[constants.CHROME_SYSROOT_TAR].path
environment = os.path.join(
sdk_ctx.key_map[constants.CHROME_ENV_TAR].path, 'environment')
if options.chroot:
# Override with the environment from the chroot if available (i.e.
# build_packages or emerge chromeos-chrome has been run for |board|).
env_path = os.path.join(sysroot, 'var', 'db', 'pkg',
'chromeos-base', 'chromeos-chrome-*')
env_glob = glob.glob(env_path)
if len(env_glob) != 1:
logging.warning(
'Multiple Chrome versions in %s. This can be resolved'
' by running "eclean-$BOARD -d packages". Using'
' environment from: %s', env_path, environment)
elif not os.path.isdir(env_glob[0]):
logging.warning(
'Environment path not found: %s. Using enviroment from:'
' %s.', env_path, environment)
else:
chroot_env_file = os.path.join(env_glob[0], 'environment.bz2')
if os.path.isfile(chroot_env_file):
# Log a warning here since this is new behavior that is not obvious.
logging.notice('Environment fetched from: %s',
chroot_env_file)
# Uncompress enviornment.bz2 to pass to osutils.SourceEnvironment.
chroot_cache = os.path.join(self.options.cache_dir,
COMMAND_NAME, 'chroot')
osutils.SafeMakedirs(chroot_cache)
environment = os.path.join(chroot_cache,
'environment_%s' % board)
cros_build_lib.UncompressFile(chroot_env_file, environment)
env = osutils.SourceEnvironment(environment, self.EBUILD_ENV)
self._SetupTCEnvironment(sdk_ctx, options, env)
# Add managed components to the PATH.
env['PATH'] = '%s:%s' % (constants.CHROMITE_BIN_DIR, env['PATH'])
env['PATH'] = '%s:%s' % (os.path.dirname(
self.sdk.gs_ctx.gsutil_bin), env['PATH'])
# Export internally referenced variables.
os.environ[self.sdk.SDK_BOARD_ENV] = board
if self.options.sdk_path:
os.environ[self.sdk.SDK_PATH_ENV] = self.options.sdk_path
os.environ[self.sdk.SDK_VERSION_ENV] = sdk_ctx.version
# Export the board/version info in a more accessible way, so developers can
# reference them in their chrome_sdk.bashrc files, as well as within the
# chrome-sdk shell.
for var in [self.sdk.SDK_VERSION_ENV, self.sdk.SDK_BOARD_ENV]:
env[var.lstrip('%')] = os.environ[var]
# Export Goma information.
if goma_dir:
env[self.SDK_GOMA_DIR_ENV] = goma_dir
env[self.SDK_GOMA_PORT_ENV] = goma_port
# SYSROOT is necessary for Goma and the sysroot wrapper.
env['SYSROOT'] = sysroot
gyp_dict = chrome_util.ProcessGypDefines(env['GYP_DEFINES'])
gn_args = gn_helpers.FromGNArgs(env['GN_ARGS'])
gyp_dict['sysroot'] = sysroot
gn_args['target_sysroot'] = sysroot
gyp_dict.pop('pkg-config', None)
gn_args.pop('pkg_config', None)
if options.clang:
gyp_dict['clang'] = 1
gn_args['is_clang'] = True
if options.internal:
gyp_dict['branding'] = 'Chrome'
gn_args['is_chrome_branded'] = True
gyp_dict['buildtype'] = 'Official'
gn_args['is_official_build'] = True
else:
gyp_dict.pop('branding', None)
gn_args.pop('is_chrome_branded', None)
gyp_dict.pop('buildtype', None)
gn_args.pop('is_official_build', None)
gyp_dict.pop('internal_gles2_conform_tests', None)
gn_args.pop('internal_gles2_conform_tests', None)
if options.component:
gyp_dict['component'] = 'shared_library'
gn_args['is_component_build'] = True
if options.fastbuild:
gyp_dict['fastbuild'] = 1
gyp_dict.pop('release_extra_cflags', None)
# symbol_level corresponds to GYP's fastbuild (https://goo.gl/ZC4fUO).
gn_args['symbol_level'] = 1
else:
# Enable debug fission for GN.
gn_args['use_debug_fission'] = True
# For SimpleChrome, we use the binutils that comes bundled within Chrome.
# We should not use the binutils from the host system.
gn_args['linux_use_bundled_binutils'] = True
gyp_dict['host_clang'] = 1
# Need to reset these after the env vars have been fixed by
# _SetupTCEnvironment.
gn_args['cros_host_is_clang'] = True
gn_args['cros_target_cc'] = env['CC']
gn_args['cros_target_cxx'] = env['CXX']
gn_args['cros_target_ld'] = env['LD']
# We need to reset extra C/CXX flags to remove references to
# EBUILD_CFLAGS, EBUILD_CXXFLAGS
gn_args['cros_target_extra_cflags'] = env['CFLAGS']
gn_args['cros_target_extra_cxxflags'] = env['CXXFLAGS']
gn_args['cros_host_cc'] = env['CC_host']
gn_args['cros_host_cxx'] = env['CXX_host']
gn_args['cros_host_ld'] = env['LD_host']
gn_args['cros_host_ar'] = env['AR_host']
gn_args['cros_v8_snapshot_cc'] = env['CC_host']
gn_args['cros_v8_snapshot_cxx'] = env['CXX_host']
gn_args['cros_v8_snapshot_ld'] = env['LD_host']
gn_args['cros_v8_snapshot_ar'] = env['AR_host']
# No need to adjust CFLAGS and CXXFLAGS for GN since the only
# adjustment made in _SetupTCEnvironment is for split debug which
# is done with 'use_debug_fission'.
# Enable goma if requested.
if goma_dir:
gyp_dict['use_goma'] = 1
gn_args['use_goma'] = True
gyp_dict['gomadir'] = goma_dir
gn_args['goma_dir'] = goma_dir
gn_args.pop('internal_khronos_glcts_tests', None) # crbug.com/588080
env['GYP_DEFINES'] = chrome_util.DictToGypDefines(gyp_dict)
env['GN_ARGS'] = gn_helpers.ToGNString(gn_args)
# PS1 sets the command line prompt and xterm window caption.
full_version = sdk_ctx.version
if full_version != CUSTOM_VERSION:
full_version = self.sdk.GetFullVersion(sdk_ctx.version)
env['PS1'] = self._CreatePS1(self.board,
full_version,
chroot=options.chroot)
out_dir = 'out_%s' % self.board
env['builddir_name'] = out_dir
env['GYP_GENERATOR_FLAGS'] = 'output_dir=%s' % out_dir
env['GYP_CROSSCOMPILE'] = '1'
# deploy_chrome relies on the 'gn' USE flag to locate .so (and potentially
# other) files. Set this by default if GYP_CHROMIUM_NO_ACTION=1.
# TODO(stevenjb): Maybe figure out a better way to set this by default.
if os.environ.get('GYP_CHROMIUM_NO_ACTION', '') == '1':
env['USE'] = 'gn'
logging.notice(
'GYP_CHROMIUM_NO_ACTION=1, setting USE="gn" for deploy_chrome.'
)
return env
def GetLicenseTypesFromEbuild(ebuild_contents):
"""Returns a list of license types from an ebuild.
This function does not always return the correct list, but it is
faster than using portageq for not having to access chroot. It is
intended to be used for tasks such as presubmission checks.
Args:
ebuild_contents: contents of ebuild.
Returns:
list of licenses read from ebuild.
Raises:
ValueError: ebuild errors.
"""
ebuild_env_tmpl = """
has() { [[ " ${*:2} " == *" $1 "* ]]; }
inherit() {
local overlay_list="%(overlay_list)s"
local eclass overlay f
for eclass; do
has ${eclass} ${_INHERITED_} && continue
_INHERITED_+=" ${eclass}"
for overlay in %(overlay_list)s; do
f="${overlay}/eclass/${eclass}.eclass"
if [[ -e ${f} ]]; then
source "${f}"
break
fi
done
done
}
%(ebuild)s"""
# TODO: the overlay_list hard-coded here should be changed to look
# at the current overlay, and then the master overlays. E.g. for an
# ebuild file in overlay-parrot, we will look at parrot overlay
# first, and then look at portage-stable and chromiumos, which are
# listed as masters in overlay-parrot/metadata/layout.conf.
tmpl_env = {
'ebuild':
ebuild_contents,
'overlay_list':
'%s %s' %
(os.path.join(constants.SOURCE_ROOT,
'src/third_party/chromiumos-overlay'),
os.path.join(constants.SOURCE_ROOT, 'src/third_party/portage-stable'))
}
with tempfile.NamedTemporaryFile(bufsize=0) as f:
osutils.WriteFile(f.name, ebuild_env_tmpl % tmpl_env)
env = osutils.SourceEnvironment(f.name,
whitelist=['LICENSE'],
ifs=' ',
multiline=True)
if not env.get('LICENSE'):
raise ValueError('No LICENSE found in the ebuild.')
if re.search(r'[,;]', env['LICENSE']):
raise ValueError(
'LICENSE field in the ebuild should be whitespace-limited.')
return env['LICENSE'].split()
def testWhiteList(self):
env_dict = osutils.SourceEnvironment(
self.env_file, ('ENV1', 'ENV3', 'ENV5', 'ENV6'))
self.assertEquals(env_dict, self.ENV_WHITELIST)