def FindSymbolFiles(tempdir, paths):
"""Locate symbol files in |paths|
This returns SymbolFile objects that contain file references which are valid
after this exits. Those files may exist externally, or be created in the
tempdir (say, when expanding tarballs). The caller must not consider
SymbolFile's valid after tempdir is cleaned up.
Args:
tempdir: Path to use for temporary files.
paths: A list of input paths to walk. Files are returned w/out any checks.
Dirs are searched for files that end in ".sym". Urls are fetched and then
processed. Tarballs are unpacked and walked.
Yields:
A SymbolFile for every symbol file found in paths.
"""
cache_dir = path_util.GetCacheDir()
common_path = os.path.join(cache_dir, constants.COMMON_CACHE)
tar_cache = cache.TarballCache(common_path)
for p in paths:
o = urllib.parse.urlparse(p)
if o.scheme:
# Support globs of filenames.
ctx = gs.GSContext()
for gspath in ctx.LS(p):
logging.info('processing files inside %s', gspath)
o = urllib.parse.urlparse(gspath)
key = ('%s%s' % (o.netloc, o.path)).split('/')
# The common cache will not be LRU, removing the need to hold a read
# lock on the cached gsutil.
ref = tar_cache.Lookup(key)
try:
ref.SetDefault(gspath)
except cros_build_lib.RunCommandError as e:
logging.warning('ignoring %s\n%s', gspath, e)
continue
for sym in FindSymbolFiles(tempdir, [ref.path]):
yield sym
elif os.path.isdir(p):
for root, _, files in os.walk(p):
for f in files:
if f.endswith('.sym'):
# If p is '/tmp/foo' and filename is '/tmp/foo/bar/bar.sym',
# display_path = 'bar/bar.sym'
filename = os.path.join(root, f)
yield SymbolFile(display_path=filename[len(p):].lstrip('/'),
file_name=filename)
elif IsTarball(p):
logging.info('processing files inside %s', p)
tardir = tempfile.mkdtemp(dir=tempdir)
cache.Untar(os.path.realpath(p), tardir)
for sym in FindSymbolFiles(tardir, [tardir]):
yield sym
else:
yield SymbolFile(display_path=p, file_name=p)
def testCompression(self, mock_find_compressor, mock_compression_type):
"""Tests Untar with a compressed tarball."""
mock_compression_type.return_value = 'some-compression'
mock_find_compressor.return_value = '/bin/custom/xz'
cache.Untar('/some/tarball.tar.xz', '/')
self.assertCommandContains(
['tar', '-I', '/bin/custom/xz', '-xpf', '/some/tarball.tar.xz'])
def testPbzip2Compression(self, mock_find_compressor,
mock_compression_type):
"""Tests decompressing a tarball using pbzip2."""
mock_compression_type.return_value = 'some-compression'
mock_find_compressor.return_value = '/bin/custom/pbzip2'
cache.Untar('/some/tarball.tbz2', '/')
self.assertCommandContains([
'tar', '-I', '/bin/custom/pbzip2 --ignore-trailing-garbage=1',
'-xpf', '/some/tarball.tbz2'
])
def SymbolFinder(tempdir, paths):
"""Locate symbol files in |paths|
Args:
tempdir: Path to use for temporary files (caller will clean up).
paths: A list of input paths to walk. Files are returned w/out any checks.
Dirs are searched for files that end in ".sym". Urls are fetched and then
processed. Tarballs are unpacked and walked.
Returns:
Yield every viable sym file.
"""
cache_dir = path_util.GetCacheDir()
common_path = os.path.join(cache_dir, constants.COMMON_CACHE)
tar_cache = cache.TarballCache(common_path)
for p in paths:
# Pylint is confused about members of ParseResult.
o = urlparse.urlparse(p)
if o.scheme: # pylint: disable=E1101
# Support globs of filenames.
ctx = gs.GSContext()
for p in ctx.LS(p):
logging.info('processing files inside %s', p)
o = urlparse.urlparse(p)
key = ('%s%s' % (o.netloc, o.path)).split('/') # pylint: disable=E1101
# The common cache will not be LRU, removing the need to hold a read
# lock on the cached gsutil.
ref = tar_cache.Lookup(key)
try:
ref.SetDefault(p)
except cros_build_lib.RunCommandError as e:
logging.warning('ignoring %s\n%s', p, e)
continue
for p in SymbolFinder(tempdir, [ref.path]):
yield p
elif os.path.isdir(p):
for root, _, files in os.walk(p):
for f in files:
if f.endswith('.sym'):
yield os.path.join(root, f)
elif IsTarball(p):
logging.info('processing files inside %s', p)
tardir = tempfile.mkdtemp(dir=tempdir)
cache.Untar(os.path.realpath(p), tardir)
for p in SymbolFinder(tardir, [tardir]):
yield p
else:
yield p
def testNoneCompression(self, mock_compression_type):
"""Tests Untar with an uncompressed tarball."""
mock_compression_type.return_value = cros_build_lib.COMP_NONE
cache.Untar('/some/tarball.tar.gz', '/')
self.assertCommandContains(['tar', '-xpf', '/some/tarball.tar.gz'])