def Cached(cls, cache_dir, *args, **kwargs):
"""Reuses previously fetched GSUtil, performing the fetch if necessary.
Arguments:
cache_dir: The toplevel cache dir.
*args, **kwargs: Arguments that are passed through to the GSContext()
constructor.
Returns:
An initialized GSContext() object.
"""
common_path = os.path.join(cache_dir, constants.COMMON_CACHE)
tar_cache = cache.TarballCache(common_path)
key = (cls.GSUTIL_TAR, )
# The common cache will not be LRU, removing the need to hold a read
# lock on the cached gsutil.
ref = tar_cache.Lookup(key)
if ref.Exists():
logging.debug('Reusing cached gsutil.')
else:
logging.debug('Fetching gsutil.')
with osutils.TempDir(base_dir=tar_cache.staging_dir) as tempdir:
gsutil_tar = os.path.join(tempdir, cls.GSUTIL_TAR)
cros_build_lib.RunCurl([cls.GSUTIL_URL, '-o', gsutil_tar],
debug_level=logging.DEBUG)
ref.SetDefault(gsutil_tar)
gsutil_bin = os.path.join(ref.path, 'gsutil', 'gsutil')
return cls(*args, gsutil_bin=gsutil_bin, **kwargs)
def FindSymbolFiles(tempdir, paths):
"""Locate symbol files in |paths|
This returns SymbolFile objects that contain file references which are valid
after this exits. Those files may exist externally, or be created in the
tempdir (say, when expanding tarballs). The caller must not consider
SymbolFile's valid after tempdir is cleaned up.
Args:
tempdir: Path to use for temporary files.
paths: A list of input paths to walk. Files are returned w/out any checks.
Dirs are searched for files that end in ".sym". Urls are fetched and then
processed. Tarballs are unpacked and walked.
Yields:
A SymbolFile for every symbol file found in paths.
"""
cache_dir = path_util.GetCacheDir()
common_path = os.path.join(cache_dir, constants.COMMON_CACHE)
tar_cache = cache.TarballCache(common_path)
for p in paths:
o = urllib.parse.urlparse(p)
if o.scheme:
# Support globs of filenames.
ctx = gs.GSContext()
for gspath in ctx.LS(p):
logging.info('processing files inside %s', gspath)
o = urllib.parse.urlparse(gspath)
key = ('%s%s' % (o.netloc, o.path)).split('/')
# The common cache will not be LRU, removing the need to hold a read
# lock on the cached gsutil.
ref = tar_cache.Lookup(key)
try:
ref.SetDefault(gspath)
except cros_build_lib.RunCommandError as e:
logging.warning('ignoring %s\n%s', gspath, e)
continue
for sym in FindSymbolFiles(tempdir, [ref.path]):
yield sym
elif os.path.isdir(p):
for root, _, files in os.walk(p):
for f in files:
if f.endswith('.sym'):
# If p is '/tmp/foo' and filename is '/tmp/foo/bar/bar.sym',
# display_path = 'bar/bar.sym'
filename = os.path.join(root, f)
yield SymbolFile(display_path=filename[len(p):].lstrip('/'),
file_name=filename)
elif IsTarball(p):
logging.info('processing files inside %s', p)
tardir = tempfile.mkdtemp(dir=tempdir)
cache.Untar(os.path.realpath(p), tardir)
for sym in FindSymbolFiles(tardir, [tardir]):
yield sym
else:
yield SymbolFile(display_path=p, file_name=p)
def GetDefaultGSUtilBin(cls, cache_dir=None):
if cls.DEFAULT_GSUTIL_BIN is None:
if cache_dir is None:
cache_dir = path_util.GetCacheDir()
if cache_dir is not None:
common_path = os.path.join(cache_dir, constants.COMMON_CACHE)
tar_cache = cache.TarballCache(common_path)
key = (cls.GSUTIL_TAR, )
# The common cache will not be LRU, removing the need to hold a read
# lock on the cached gsutil.
ref = tar_cache.Lookup(key)
ref.SetDefault(cls.GSUTIL_URL)
cls.DEFAULT_GSUTIL_BIN = os.path.join(ref.path, 'gsutil',
'gsutil')
else:
# Check if the default gsutil path for builders exists. If
# not, try locating gsutil. If none exists, simply use 'gsutil'.
gsutil_bin = cls.DEFAULT_GSUTIL_BUILDER_BIN
if not os.path.exists(gsutil_bin):
gsutil_bin = osutils.Which('gsutil')
if gsutil_bin is None:
gsutil_bin = 'gsutil'
cls.DEFAULT_GSUTIL_BIN = gsutil_bin
return cls.DEFAULT_GSUTIL_BIN
def __init__(self,
cache_dir,
board,
clear_cache=False,
chrome_src=None,
sdk_path=None,
toolchain_path=None,
silent=False,
use_external_config=None):
"""Initialize the class.
Args:
cache_dir: The toplevel cache dir to use.
board: The board to manage the SDK for.
clear_cache: Clears the sdk cache during __init__.
chrome_src: The location of the chrome checkout. If unspecified, the
cwd is presumed to be within a chrome checkout.
sdk_path: The path (whether a local directory or a gs:// path) to fetch
SDK components from.
toolchain_path: The path (whether a local directory or a gs:// path) to
fetch toolchain components from.
silent: If set, the fetcher prints less output.
use_external_config: When identifying the configuration for a board,
force usage of the external configuration if both external and internal
are available.
"""
site_config = config_lib.GetConfig()
self.cache_base = os.path.join(cache_dir, COMMAND_NAME)
if clear_cache:
logging.warning('Clearing the SDK cache.')
osutils.RmDir(self.cache_base, ignore_missing=True)
self.tarball_cache = cache.TarballCache(
os.path.join(self.cache_base, self.TARBALL_CACHE))
self.misc_cache = cache.DiskCache(
os.path.join(self.cache_base, self.MISC_CACHE))
self.board = board
self.config = site_config.FindCanonicalConfigForBoard(
board, allow_internal=not use_external_config)
self.gs_base = archive_lib.GetBaseUploadURI(self.config)
self.clear_cache = clear_cache
self.chrome_src = chrome_src
self.sdk_path = sdk_path
self.toolchain_path = toolchain_path
self.silent = silent
# For external configs, there is no need to run 'gsutil config', because
# the necessary files are all accessible to anonymous users.
internal = self.config['internal']
self.gs_ctx = gs.GSContext(cache_dir=cache_dir, init_boto=internal)
if self.sdk_path is None:
self.sdk_path = os.environ.get(self.SDK_PATH_ENV)
if self.toolchain_path is None:
self.toolchain_path = 'gs://%s' % constants.SDK_GS_BUCKET
def SymbolFinder(tempdir, paths):
"""Locate symbol files in |paths|
Args:
tempdir: Path to use for temporary files (caller will clean up).
paths: A list of input paths to walk. Files are returned w/out any checks.
Dirs are searched for files that end in ".sym". Urls are fetched and then
processed. Tarballs are unpacked and walked.
Returns:
Yield every viable sym file.
"""
cache_dir = path_util.GetCacheDir()
common_path = os.path.join(cache_dir, constants.COMMON_CACHE)
tar_cache = cache.TarballCache(common_path)
for p in paths:
# Pylint is confused about members of ParseResult.
o = urlparse.urlparse(p)
if o.scheme: # pylint: disable=E1101
# Support globs of filenames.
ctx = gs.GSContext()
for p in ctx.LS(p):
logging.info('processing files inside %s', p)
o = urlparse.urlparse(p)
key = ('%s%s' % (o.netloc, o.path)).split('/') # pylint: disable=E1101
# The common cache will not be LRU, removing the need to hold a read
# lock on the cached gsutil.
ref = tar_cache.Lookup(key)
try:
ref.SetDefault(p)
except cros_build_lib.RunCommandError as e:
logging.warning('ignoring %s\n%s', p, e)
continue
for p in SymbolFinder(tempdir, [ref.path]):
yield p
elif os.path.isdir(p):
for root, _, files in os.walk(p):
for f in files:
if f.endswith('.sym'):
yield os.path.join(root, f)
elif IsTarball(p):
logging.info('processing files inside %s', p)
tardir = tempfile.mkdtemp(dir=tempdir)
cache.Untar(os.path.realpath(p), tardir)
for p in SymbolFinder(tardir, [tardir]):
yield p
else:
yield p
def _CachePathForKey(self, key):
"""Get cache path for key.
Args:
key: cache key.
"""
tarball_cache = cache.TarballCache(
self._GetCachePath(cros_chrome_sdk.SDKFetcher.TARBALL_CACHE))
if self.board and self._SDKVersion():
cache_key = (self.board, self._SDKVersion(), key)
with tarball_cache.Lookup(cache_key) as ref:
if ref.Exists():
return ref.path
return None
def __init__(self, cache_dir, board):
"""Initialize the class.
Arguments:
cache_dir: The toplevel cache dir to use.
board: The board to manage the SDK for.
"""
self.gs_ctx = gs.GSContext.Cached(cache_dir, init_boto=True)
self.cache_base = os.path.join(cache_dir, COMMAND_NAME)
self.tarball_cache = cache.TarballCache(
os.path.join(self.cache_base, self.TARBALL_CACHE))
self.misc_cache = cache.DiskCache(
os.path.join(self.cache_base, self.MISC_CACHE))
self.board = board
self.gs_base = self._GetGSBaseForBoard(board)
def __init__(self, cache_dir, board, silent=False):
"""Initialize the class.
Arguments:
cache_dir: The toplevel cache dir to use.
board: The board to manage the SDK for.
"""
self.gs_ctx = gs.GSContext.Cached(cache_dir, init_boto=True)
self.cache_base = os.path.join(cache_dir, COMMAND_NAME)
self.tarball_cache = cache.TarballCache(
os.path.join(self.cache_base, self.TARBALL_CACHE))
self.misc_cache = cache.DiskCache(
os.path.join(self.cache_base, self.MISC_CACHE))
self.board = board
self.config = cbuildbot_config.FindCanonicalConfigForBoard(board)
self.gs_base = '%s/%s' % (constants.DEFAULT_ARCHIVE_BUCKET,
self.config['name'])
self.silent = silent
def __init__(self,
cache_dir,
board,
clear_cache=False,
chrome_src=None,
sdk_path=None,
silent=False):
"""Initialize the class.
Arguments:
cache_dir: The toplevel cache dir to use.
board: The board to manage the SDK for.
clear_cache: Clears the sdk cache during __init__.
chrome_src: The location of the chrome checkout. If unspecified, the
cwd is presumed to be within a chrome checkout.
sdk_path: The path (whether a local directory or a gs:// path) to fetch
SDK components from.
silent: If set, the fetcher prints less output.
"""
self.gs_ctx = gs.GSContext.Cached(cache_dir, init_boto=True)
self.cache_base = os.path.join(cache_dir, COMMAND_NAME)
if clear_cache:
logging.warning('Clearing the SDK cache.')
osutils.RmDir(self.cache_base, ignore_missing=True)
self.tarball_cache = cache.TarballCache(
os.path.join(self.cache_base, self.TARBALL_CACHE))
self.misc_cache = cache.DiskCache(
os.path.join(self.cache_base, self.MISC_CACHE))
self.board = board
self.config = cbuildbot_config.FindCanonicalConfigForBoard(board)
self.gs_base = '%s/%s' % (constants.DEFAULT_ARCHIVE_BUCKET,
self.config['name'])
self.clear_cache = clear_cache
self.chrome_src = chrome_src
self.sdk_path = sdk_path
self.silent = silent
if self.sdk_path is None:
self.sdk_path = os.environ.get(self.SDK_PATH_ENV)