def main(argv):
options = ParseArgs(argv)
if not cros_build_lib.IsInsideChroot():
raise commandline.ChrootRequiredError()
if os.geteuid() != 0:
cros_build_lib.SudoRunCommand(sys.argv)
return
# sysroot must have a trailing / as the tree dictionary produced by
# create_trees in indexed with a trailing /.
sysroot = cros_build_lib.GetSysroot(options.board) + '/'
trees = create_trees(target_root=sysroot, config_root=sysroot)
vartree = trees[sysroot]['vartree']
cache_dir = os.path.join(path_util.FindCacheDir(),
'cros_install_debug_syms-v' + CACHE_VERSION)
if options.clearcache:
osutils.RmDir(cache_dir, ignore_missing=True)
binhost_cache = None
if options.cachebinhost:
binhost_cache = cache.DiskCache(cache_dir)
boto_file = vartree.settings['BOTO_CONFIG']
if boto_file:
os.environ['BOTO_CONFIG'] = boto_file
gs_context = gs.GSContext()
symbols_mapping = RemoteSymbols(vartree, binhost_cache)
if options.all:
to_install = vartree.dbapi.cpv_all()
else:
to_install = [GetMatchingCPV(p, vartree.dbapi) for p in options.packages]
to_install = [p for p in to_install
if ShouldGetSymbols(p, vartree.dbapi, symbols_mapping)]
if not to_install:
logging.info('nothing to do, exit')
return
with DebugSymbolsInstaller(vartree, gs_context, sysroot,
not options.debug) as installer:
args = [(p, symbols_mapping[p]) for p in to_install]
parallel.RunTasksInProcessPool(installer.Install, args,
processes=options.jobs)
logging.debug('installation done, updating packages index file')
packages_dir = os.path.join(sysroot, 'packages')
packages_file = os.path.join(packages_dir, 'Packages')
# binpkg will set DEBUG_SYMBOLS automatically if it detects the debug symbols
# in the packages dir.
pkgindex = binpkg.GrabLocalPackageIndex(packages_dir)
with open(packages_file, 'w') as p:
pkgindex.Write(p)
def __init__(self,
cache_dir,
board,
clear_cache=False,
chrome_src=None,
sdk_path=None,
toolchain_path=None,
silent=False,
use_external_config=None):
"""Initialize the class.
Args:
cache_dir: The toplevel cache dir to use.
board: The board to manage the SDK for.
clear_cache: Clears the sdk cache during __init__.
chrome_src: The location of the chrome checkout. If unspecified, the
cwd is presumed to be within a chrome checkout.
sdk_path: The path (whether a local directory or a gs:// path) to fetch
SDK components from.
toolchain_path: The path (whether a local directory or a gs:// path) to
fetch toolchain components from.
silent: If set, the fetcher prints less output.
use_external_config: When identifying the configuration for a board,
force usage of the external configuration if both external and internal
are available.
"""
site_config = config_lib.GetConfig()
self.cache_base = os.path.join(cache_dir, COMMAND_NAME)
if clear_cache:
logging.warning('Clearing the SDK cache.')
osutils.RmDir(self.cache_base, ignore_missing=True)
self.tarball_cache = cache.TarballCache(
os.path.join(self.cache_base, self.TARBALL_CACHE))
self.misc_cache = cache.DiskCache(
os.path.join(self.cache_base, self.MISC_CACHE))
self.board = board
self.config = site_config.FindCanonicalConfigForBoard(
board, allow_internal=not use_external_config)
self.gs_base = archive_lib.GetBaseUploadURI(self.config)
self.clear_cache = clear_cache
self.chrome_src = chrome_src
self.sdk_path = sdk_path
self.toolchain_path = toolchain_path
self.silent = silent
# For external configs, there is no need to run 'gsutil config', because
# the necessary files are all accessible to anonymous users.
internal = self.config['internal']
self.gs_ctx = gs.GSContext(cache_dir=cache_dir, init_boto=internal)
if self.sdk_path is None:
self.sdk_path = os.environ.get(self.SDK_PATH_ENV)
if self.toolchain_path is None:
self.toolchain_path = 'gs://%s' % constants.SDK_GS_BUCKET
def GetBinhostCache(options):
"""Get and optionally clear the binhost cache."""
cache_dir = os.path.join(path_util.FindCacheDir(),
'cros_install_debug_syms-v' + CACHE_VERSION)
if options.clearcache:
osutils.RmDir(cache_dir, ignore_missing=True)
binhost_cache = None
if options.cachebinhost:
binhost_cache = cache.DiskCache(cache_dir)
return binhost_cache
def __init__(self, cache_dir, board):
"""Initialize the class.
Arguments:
cache_dir: The toplevel cache dir to use.
board: The board to manage the SDK for.
"""
self.gs_ctx = gs.GSContext.Cached(cache_dir, init_boto=True)
self.cache_base = os.path.join(cache_dir, COMMAND_NAME)
self.tarball_cache = cache.TarballCache(
os.path.join(self.cache_base, self.TARBALL_CACHE))
self.misc_cache = cache.DiskCache(
os.path.join(self.cache_base, self.MISC_CACHE))
self.board = board
self.gs_base = self._GetGSBaseForBoard(board)
def __init__(self, options, bots, local_patches):
"""Construct the object.
Args:
options: The parsed options passed into cbuildbot.
bots: A list of configs to run tryjobs for.
local_patches: A list of LocalPatch objects.
"""
self.options = options
self.use_buildbucket = options.use_buildbucket
self.user = getpass.getuser()
self.repo_cache = cache.DiskCache(self.options.cache_dir)
cwd = os.path.dirname(os.path.realpath(__file__))
self.user_email = git.GetProjectUserEmail(cwd)
logging.info('Using email:%s', self.user_email)
# Name of the job that appears on the waterfall.
patch_list = options.gerrit_patches + options.local_patches
self.name = options.remote_description
if self.name is None:
self.name = ''
if options.branch != 'master':
self.name = '[%s] ' % options.branch
self.name += ','.join(patch_list[:self.MAX_PATCHES_IN_DESCRIPTION])
if len(patch_list) > self.MAX_PATCHES_IN_DESCRIPTION:
remaining_patches = len(
patch_list) - self.MAX_PATCHES_IN_DESCRIPTION
self.name += '... (%d more CLs)' % (remaining_patches, )
self.bots = bots[:]
self.slaves_request = options.slaves
self.description = ('name: %s\n patches: %s\nbots: %s' %
(self.name, patch_list, self.bots))
self.extra_args = options.pass_through_args
if '--buildbot' not in self.extra_args:
self.extra_args.append('--remote-trybot')
self.extra_args.append('--remote-version=%s' %
(self.TRYJOB_FORMAT_VERSION, ))
self.local_patches = local_patches
self.repo_url = self.EXTERNAL_URL
self.cache_key = ('trybot', )
self.manifest = None
if repository.IsARepoRoot(options.sourceroot):
self.manifest = git.ManifestCheckout.Cached(options.sourceroot)
if repository.IsInternalRepoCheckout(options.sourceroot):
self.repo_url = self.INTERNAL_URL
self.cache_key = ('trybot-internal', )
def _FetchGoma(self):
"""Fetch, install, and start Goma, using cached version if it exists.
Returns:
A tuple (dir, port) containing the path to the cached goma/ dir and the
Goma port.
"""
common_path = os.path.join(self.options.cache_dir,
constants.COMMON_CACHE)
common_cache = cache.DiskCache(common_path)
goma_dir = self.options.gomadir
if not goma_dir:
ref = common_cache.Lookup(('goma', '2'))
if not ref.Exists():
Log('Installing Goma.', silent=self.silent)
with osutils.TempDir() as tempdir:
goma_dir = os.path.join(tempdir, 'goma')
os.mkdir(goma_dir)
result = cros_build_lib.DebugRunCommand(
self.FETCH_GOMA_CMD, cwd=goma_dir, error_code_ok=True)
if result.returncode:
raise GomaError('Failed to fetch Goma')
# Update to latest version of goma. We choose the outside-chroot
# version ('goobuntu') over the chroot version ('chromeos') by
# supplying input='1' to the following prompt:
#
# What is your platform?
# 1. Goobuntu 2. Precise (32bit) 3. Lucid (32bit) 4. Debian
# 5. Chrome OS 6. MacOS ? -->
cros_build_lib.DebugRunCommand(
['python2', 'goma_ctl.py', 'update'],
cwd=goma_dir,
input='1\n')
ref.SetDefault(goma_dir)
goma_dir = ref.path
port = None
if self.options.start_goma:
Log('Starting Goma.', silent=self.silent)
cros_build_lib.DebugRunCommand(
['python2', 'goma_ctl.py', 'ensure_start'], cwd=goma_dir)
port = self._GomaPort(goma_dir)
Log('Goma is started on port %s', port, silent=self.silent)
if not port:
raise GomaError('No Goma port detected')
return goma_dir, port
def __init__(self, cache_dir, board, silent=False):
"""Initialize the class.
Arguments:
cache_dir: The toplevel cache dir to use.
board: The board to manage the SDK for.
"""
self.gs_ctx = gs.GSContext.Cached(cache_dir, init_boto=True)
self.cache_base = os.path.join(cache_dir, COMMAND_NAME)
self.tarball_cache = cache.TarballCache(
os.path.join(self.cache_base, self.TARBALL_CACHE))
self.misc_cache = cache.DiskCache(
os.path.join(self.cache_base, self.MISC_CACHE))
self.board = board
self.config = cbuildbot_config.FindCanonicalConfigForBoard(board)
self.gs_base = '%s/%s' % (constants.DEFAULT_ARCHIVE_BUCKET,
self.config['name'])
self.silent = silent
def _SDKVersion(self):
"""Determine SDK version.
Check the environment if we're in the SDK shell, and failing that, look at
the misc cache.
Returns:
SDK version.
"""
sdk_version = os.environ.get(
cros_chrome_sdk.SDKFetcher.SDK_VERSION_ENV)
if not sdk_version and self.board:
misc_cache = cache.DiskCache(
self._GetCachePath(cros_chrome_sdk.SDKFetcher.MISC_CACHE))
with misc_cache.Lookup((self.board, 'latest')) as ref:
if ref.Exists(lock=True):
sdk_version = osutils.ReadFile(ref.path).strip()
return sdk_version
def _FetchGoma(self):
"""Fetch, install, and start Goma, using cached version if it exists.
Returns:
A tuple (dir, port) containing the path to the cached goma/ dir and the
Goma port.
"""
common_path = os.path.join(self.options.cache_dir,
constants.COMMON_CACHE)
common_cache = cache.DiskCache(common_path)
ref = common_cache.Lookup(('goma', ))
if not ref.Exists():
Log('Installing Goma.', silent=self.silent)
with osutils.TempDir() as tempdir:
goma_dir = os.path.join(tempdir, 'goma')
os.mkdir(goma_dir)
result = cros_build_lib.DebugRunCommand(self.FETCH_GOMA_CMD,
cwd=goma_dir,
error_code_ok=True)
if result.returncode:
raise GomaError('Failed to fetch Goma')
cros_build_lib.DebugRunCommand(
['bash', 'goma_ctl.sh', 'update'],
cwd=goma_dir,
input='2\n')
ref.SetDefault(goma_dir)
goma_dir = os.path.join(ref.path, 'goma')
port = self._GomaPort(goma_dir)
if not port:
Log('Starting Goma.', silent=self.silent)
cros_build_lib.DebugRunCommand(['./goma_ctl.sh', 'ensure_start'],
cwd=goma_dir)
port = self._GomaPort(goma_dir)
Log('Goma is started on port %s', port, silent=self.silent)
if not port:
raise GomaError('No Goma port detected')
return goma_dir, port
def __init__(self,
cache_dir,
board,
clear_cache=False,
chrome_src=None,
sdk_path=None,
silent=False):
"""Initialize the class.
Arguments:
cache_dir: The toplevel cache dir to use.
board: The board to manage the SDK for.
clear_cache: Clears the sdk cache during __init__.
chrome_src: The location of the chrome checkout. If unspecified, the
cwd is presumed to be within a chrome checkout.
sdk_path: The path (whether a local directory or a gs:// path) to fetch
SDK components from.
silent: If set, the fetcher prints less output.
"""
self.gs_ctx = gs.GSContext.Cached(cache_dir, init_boto=True)
self.cache_base = os.path.join(cache_dir, COMMAND_NAME)
if clear_cache:
logging.warning('Clearing the SDK cache.')
osutils.RmDir(self.cache_base, ignore_missing=True)
self.tarball_cache = cache.TarballCache(
os.path.join(self.cache_base, self.TARBALL_CACHE))
self.misc_cache = cache.DiskCache(
os.path.join(self.cache_base, self.MISC_CACHE))
self.board = board
self.config = cbuildbot_config.FindCanonicalConfigForBoard(board)
self.gs_base = '%s/%s' % (constants.DEFAULT_ARCHIVE_BUCKET,
self.config['name'])
self.clear_cache = clear_cache
self.chrome_src = chrome_src
self.sdk_path = sdk_path
self.silent = silent
if self.sdk_path is None:
self.sdk_path = os.environ.get(self.SDK_PATH_ENV)
def setUp(self):
self.cache = cache.DiskCache(self.tempdir)
