def main(argv): options = ParseArgs(argv) if not cros_build_lib.IsInsideChroot(): raise commandline.ChrootRequiredError() if os.geteuid() != 0: cros_build_lib.SudoRunCommand(sys.argv) return # sysroot must have a trailing / as the tree dictionary produced by # create_trees in indexed with a trailing /. sysroot = cros_build_lib.GetSysroot(options.board) + '/' trees = create_trees(target_root=sysroot, config_root=sysroot) vartree = trees[sysroot]['vartree'] cache_dir = os.path.join(path_util.FindCacheDir(), 'cros_install_debug_syms-v' + CACHE_VERSION) if options.clearcache: osutils.RmDir(cache_dir, ignore_missing=True) binhost_cache = None if options.cachebinhost: binhost_cache = cache.DiskCache(cache_dir) boto_file = vartree.settings['BOTO_CONFIG'] if boto_file: os.environ['BOTO_CONFIG'] = boto_file gs_context = gs.GSContext() symbols_mapping = RemoteSymbols(vartree, binhost_cache) if options.all: to_install = vartree.dbapi.cpv_all() else: to_install = [GetMatchingCPV(p, vartree.dbapi) for p in options.packages] to_install = [p for p in to_install if ShouldGetSymbols(p, vartree.dbapi, symbols_mapping)] if not to_install: logging.info('nothing to do, exit') return with DebugSymbolsInstaller(vartree, gs_context, sysroot, not options.debug) as installer: args = [(p, symbols_mapping[p]) for p in to_install] parallel.RunTasksInProcessPool(installer.Install, args, processes=options.jobs) logging.debug('installation done, updating packages index file') packages_dir = os.path.join(sysroot, 'packages') packages_file = os.path.join(packages_dir, 'Packages') # binpkg will set DEBUG_SYMBOLS automatically if it detects the debug symbols # in the packages dir. pkgindex = binpkg.GrabLocalPackageIndex(packages_dir) with open(packages_file, 'w') as p: pkgindex.Write(p)
def __init__(self, cache_dir, board, clear_cache=False, chrome_src=None, sdk_path=None, toolchain_path=None, silent=False, use_external_config=None): """Initialize the class. Args: cache_dir: The toplevel cache dir to use. board: The board to manage the SDK for. clear_cache: Clears the sdk cache during __init__. chrome_src: The location of the chrome checkout. If unspecified, the cwd is presumed to be within a chrome checkout. sdk_path: The path (whether a local directory or a gs:// path) to fetch SDK components from. toolchain_path: The path (whether a local directory or a gs:// path) to fetch toolchain components from. silent: If set, the fetcher prints less output. use_external_config: When identifying the configuration for a board, force usage of the external configuration if both external and internal are available. """ site_config = config_lib.GetConfig() self.cache_base = os.path.join(cache_dir, COMMAND_NAME) if clear_cache: logging.warning('Clearing the SDK cache.') osutils.RmDir(self.cache_base, ignore_missing=True) self.tarball_cache = cache.TarballCache( os.path.join(self.cache_base, self.TARBALL_CACHE)) self.misc_cache = cache.DiskCache( os.path.join(self.cache_base, self.MISC_CACHE)) self.board = board self.config = site_config.FindCanonicalConfigForBoard( board, allow_internal=not use_external_config) self.gs_base = archive_lib.GetBaseUploadURI(self.config) self.clear_cache = clear_cache self.chrome_src = chrome_src self.sdk_path = sdk_path self.toolchain_path = toolchain_path self.silent = silent # For external configs, there is no need to run 'gsutil config', because # the necessary files are all accessible to anonymous users. internal = self.config['internal'] self.gs_ctx = gs.GSContext(cache_dir=cache_dir, init_boto=internal) if self.sdk_path is None: self.sdk_path = os.environ.get(self.SDK_PATH_ENV) if self.toolchain_path is None: self.toolchain_path = 'gs://%s' % constants.SDK_GS_BUCKET
def GetBinhostCache(options): """Get and optionally clear the binhost cache.""" cache_dir = os.path.join(path_util.FindCacheDir(), 'cros_install_debug_syms-v' + CACHE_VERSION) if options.clearcache: osutils.RmDir(cache_dir, ignore_missing=True) binhost_cache = None if options.cachebinhost: binhost_cache = cache.DiskCache(cache_dir) return binhost_cache
def __init__(self, cache_dir, board): """Initialize the class. Arguments: cache_dir: The toplevel cache dir to use. board: The board to manage the SDK for. """ self.gs_ctx = gs.GSContext.Cached(cache_dir, init_boto=True) self.cache_base = os.path.join(cache_dir, COMMAND_NAME) self.tarball_cache = cache.TarballCache( os.path.join(self.cache_base, self.TARBALL_CACHE)) self.misc_cache = cache.DiskCache( os.path.join(self.cache_base, self.MISC_CACHE)) self.board = board self.gs_base = self._GetGSBaseForBoard(board)
def __init__(self, options, bots, local_patches): """Construct the object. Args: options: The parsed options passed into cbuildbot. bots: A list of configs to run tryjobs for. local_patches: A list of LocalPatch objects. """ self.options = options self.use_buildbucket = options.use_buildbucket self.user = getpass.getuser() self.repo_cache = cache.DiskCache(self.options.cache_dir) cwd = os.path.dirname(os.path.realpath(__file__)) self.user_email = git.GetProjectUserEmail(cwd) logging.info('Using email:%s', self.user_email) # Name of the job that appears on the waterfall. patch_list = options.gerrit_patches + options.local_patches self.name = options.remote_description if self.name is None: self.name = '' if options.branch != 'master': self.name = '[%s] ' % options.branch self.name += ','.join(patch_list[:self.MAX_PATCHES_IN_DESCRIPTION]) if len(patch_list) > self.MAX_PATCHES_IN_DESCRIPTION: remaining_patches = len( patch_list) - self.MAX_PATCHES_IN_DESCRIPTION self.name += '... (%d more CLs)' % (remaining_patches, ) self.bots = bots[:] self.slaves_request = options.slaves self.description = ('name: %s\n patches: %s\nbots: %s' % (self.name, patch_list, self.bots)) self.extra_args = options.pass_through_args if '--buildbot' not in self.extra_args: self.extra_args.append('--remote-trybot') self.extra_args.append('--remote-version=%s' % (self.TRYJOB_FORMAT_VERSION, )) self.local_patches = local_patches self.repo_url = self.EXTERNAL_URL self.cache_key = ('trybot', ) self.manifest = None if repository.IsARepoRoot(options.sourceroot): self.manifest = git.ManifestCheckout.Cached(options.sourceroot) if repository.IsInternalRepoCheckout(options.sourceroot): self.repo_url = self.INTERNAL_URL self.cache_key = ('trybot-internal', )
def _FetchGoma(self): """Fetch, install, and start Goma, using cached version if it exists. Returns: A tuple (dir, port) containing the path to the cached goma/ dir and the Goma port. """ common_path = os.path.join(self.options.cache_dir, constants.COMMON_CACHE) common_cache = cache.DiskCache(common_path) goma_dir = self.options.gomadir if not goma_dir: ref = common_cache.Lookup(('goma', '2')) if not ref.Exists(): Log('Installing Goma.', silent=self.silent) with osutils.TempDir() as tempdir: goma_dir = os.path.join(tempdir, 'goma') os.mkdir(goma_dir) result = cros_build_lib.DebugRunCommand( self.FETCH_GOMA_CMD, cwd=goma_dir, error_code_ok=True) if result.returncode: raise GomaError('Failed to fetch Goma') # Update to latest version of goma. We choose the outside-chroot # version ('goobuntu') over the chroot version ('chromeos') by # supplying input='1' to the following prompt: # # What is your platform? # 1. Goobuntu 2. Precise (32bit) 3. Lucid (32bit) 4. Debian # 5. Chrome OS 6. MacOS ? --> cros_build_lib.DebugRunCommand( ['python2', 'goma_ctl.py', 'update'], cwd=goma_dir, input='1\n') ref.SetDefault(goma_dir) goma_dir = ref.path port = None if self.options.start_goma: Log('Starting Goma.', silent=self.silent) cros_build_lib.DebugRunCommand( ['python2', 'goma_ctl.py', 'ensure_start'], cwd=goma_dir) port = self._GomaPort(goma_dir) Log('Goma is started on port %s', port, silent=self.silent) if not port: raise GomaError('No Goma port detected') return goma_dir, port
def __init__(self, cache_dir, board, silent=False): """Initialize the class. Arguments: cache_dir: The toplevel cache dir to use. board: The board to manage the SDK for. """ self.gs_ctx = gs.GSContext.Cached(cache_dir, init_boto=True) self.cache_base = os.path.join(cache_dir, COMMAND_NAME) self.tarball_cache = cache.TarballCache( os.path.join(self.cache_base, self.TARBALL_CACHE)) self.misc_cache = cache.DiskCache( os.path.join(self.cache_base, self.MISC_CACHE)) self.board = board self.config = cbuildbot_config.FindCanonicalConfigForBoard(board) self.gs_base = '%s/%s' % (constants.DEFAULT_ARCHIVE_BUCKET, self.config['name']) self.silent = silent
def _SDKVersion(self): """Determine SDK version. Check the environment if we're in the SDK shell, and failing that, look at the misc cache. Returns: SDK version. """ sdk_version = os.environ.get( cros_chrome_sdk.SDKFetcher.SDK_VERSION_ENV) if not sdk_version and self.board: misc_cache = cache.DiskCache( self._GetCachePath(cros_chrome_sdk.SDKFetcher.MISC_CACHE)) with misc_cache.Lookup((self.board, 'latest')) as ref: if ref.Exists(lock=True): sdk_version = osutils.ReadFile(ref.path).strip() return sdk_version
def _FetchGoma(self): """Fetch, install, and start Goma, using cached version if it exists. Returns: A tuple (dir, port) containing the path to the cached goma/ dir and the Goma port. """ common_path = os.path.join(self.options.cache_dir, constants.COMMON_CACHE) common_cache = cache.DiskCache(common_path) ref = common_cache.Lookup(('goma', )) if not ref.Exists(): Log('Installing Goma.', silent=self.silent) with osutils.TempDir() as tempdir: goma_dir = os.path.join(tempdir, 'goma') os.mkdir(goma_dir) result = cros_build_lib.DebugRunCommand(self.FETCH_GOMA_CMD, cwd=goma_dir, error_code_ok=True) if result.returncode: raise GomaError('Failed to fetch Goma') cros_build_lib.DebugRunCommand( ['bash', 'goma_ctl.sh', 'update'], cwd=goma_dir, input='2\n') ref.SetDefault(goma_dir) goma_dir = os.path.join(ref.path, 'goma') port = self._GomaPort(goma_dir) if not port: Log('Starting Goma.', silent=self.silent) cros_build_lib.DebugRunCommand(['./goma_ctl.sh', 'ensure_start'], cwd=goma_dir) port = self._GomaPort(goma_dir) Log('Goma is started on port %s', port, silent=self.silent) if not port: raise GomaError('No Goma port detected') return goma_dir, port
def __init__(self, cache_dir, board, clear_cache=False, chrome_src=None, sdk_path=None, silent=False): """Initialize the class. Arguments: cache_dir: The toplevel cache dir to use. board: The board to manage the SDK for. clear_cache: Clears the sdk cache during __init__. chrome_src: The location of the chrome checkout. If unspecified, the cwd is presumed to be within a chrome checkout. sdk_path: The path (whether a local directory or a gs:// path) to fetch SDK components from. silent: If set, the fetcher prints less output. """ self.gs_ctx = gs.GSContext.Cached(cache_dir, init_boto=True) self.cache_base = os.path.join(cache_dir, COMMAND_NAME) if clear_cache: logging.warning('Clearing the SDK cache.') osutils.RmDir(self.cache_base, ignore_missing=True) self.tarball_cache = cache.TarballCache( os.path.join(self.cache_base, self.TARBALL_CACHE)) self.misc_cache = cache.DiskCache( os.path.join(self.cache_base, self.MISC_CACHE)) self.board = board self.config = cbuildbot_config.FindCanonicalConfigForBoard(board) self.gs_base = '%s/%s' % (constants.DEFAULT_ARCHIVE_BUCKET, self.config['name']) self.clear_cache = clear_cache self.chrome_src = chrome_src self.sdk_path = sdk_path self.silent = silent if self.sdk_path is None: self.sdk_path = os.environ.get(self.SDK_PATH_ENV)
def setUp(self): self.cache = cache.DiskCache(self.tempdir)