def BundleEBuildLogsTarball(chroot, sysroot, archive_dir):
"""Builds a tarball containing ebuild logs.
Args:
chroot (chroot_lib.Chroot): The chroot to be used.
sysroot (sysroot_lib.Sysroot): Sysroot whose images are being fetched.
archive_dir: The directory to drop the tarball in.
Returns:
The file name of the output tarball, None if no package found.
"""
tarball_paths = []
logs_path = chroot.full_path(sysroot.path, 'tmp/portage')
if not os.path.isdir(logs_path):
return None
if not os.path.exists(os.path.join(logs_path, 'logs')):
return None
tarball_paths.append('logs')
tarball_output = os.path.join(archive_dir, 'ebuild_logs.tar.xz')
try:
cros_build_lib.CreateTarball(tarball_output,
cwd=logs_path,
chroot=chroot.path,
inputs=tarball_paths)
except cros_build_lib.CreateTarballError:
logging.warning('Unable to create logs tarball; ignoring until '
'https://crbug.com/999933 is sorted out.')
return None
return os.path.basename(tarball_output)
def BundleFpmcuUnittests(chroot, sysroot, output_directory):
"""Create artifact tarball for fingerprint MCU on-device unittests.
Args:
chroot (chroot_lib.Chroot): The chroot containing the sysroot.
sysroot (sysroot_lib.Sysroot): The sysroot whose artifacts are being
archived.
output_directory (str): The path were the completed archives should be put.
Returns:
str|None - The archive file path if created, None otherwise.
"""
fpmcu_unittests_root = os.path.join(chroot.path,
sysroot.path.lstrip(os.sep),
'firmware', 'chromeos-fpmcu-unittests')
files = [
os.path.relpath(f, fpmcu_unittests_root)
for f in glob.iglob(os.path.join(fpmcu_unittests_root, '*'))
]
if not files:
return None
archive_file = os.path.join(output_directory,
constants.FPMCU_UNITTESTS_ARCHIVE_NAME)
cros_build_lib.CreateTarball(archive_file,
fpmcu_unittests_root,
compression=cros_build_lib.COMP_BZIP2,
chroot=chroot.path,
inputs=files)
return archive_file
def BundleTastFiles(chroot, sysroot, output_dir):
"""Tar up the Tast private test bundles.
Args:
chroot (chroot_lib.Chroot): Chroot containing the sysroot.
sysroot (sysroot_lib.Sysroot): Sysroot whose files are being archived.
output_dir: Location for storing the result tarball.
Returns:
Path of the generated tarball, or None if there is no private test bundles.
"""
cwd = os.path.join(chroot.path, sysroot.path.lstrip(os.sep), 'build')
dirs = []
for d in ('libexec/tast', 'share/tast'):
if os.path.exists(os.path.join(cwd, d)):
dirs.append(d)
if not dirs:
return None
tarball = os.path.join(output_dir, TAST_BUNDLE_NAME)
cros_build_lib.CreateTarball(tarball,
cwd,
compression=TAST_COMPRESSOR,
chroot=chroot.path,
inputs=dirs)
return tarball
def CreateTarball(source_root, tarball_path, exclude_paths=None):
"""Packs |source_root| into |tarball_path|.
Args:
source_root: Path to the directory we want to package.
tarball_path: Path of the tarball that should be created.
exclude_paths: Subdirectories to exclude.
"""
# TODO(zbehan): We cannot use xz from the chroot unless it's
# statically linked.
extra_args = None
if exclude_paths is not None:
extra_args = ['--anchored']
extra_args.extend('--exclude=./%s/*' % x for x in exclude_paths)
# Options for maximum compression.
extra_env = {'XZ_OPT': '-e9'}
cros_build_lib.CreateTarball(
tarball_path,
source_root,
sudo=True,
extra_args=extra_args,
debug_level=logging.INFO,
extra_env=extra_env)
# Make sure the regular user has the permission to read.
cmd = ['chmod', 'a+r', tarball_path]
cros_build_lib.sudo_run(cmd)
def _ArchiveGomaccInfoFiles(self):
"""Archives gomacc INFO files, with gzip'ing.
Returns:
Archived file path. If failed, None.
"""
# Since the number of gomacc logs can be large, we'd like to compress them.
# Otherwise, archive will take long (> 10 mins).
# Each gomacc logs file size must be small (around 4KB).
# Find files matched with the pattern in |log_dir|.
# The paths were themselves used as the inputs for the create
# tarball, but there can be too many of them. As long as we have
# files we'll just tar up the entire directory.
gomacc_paths = glob.glob(os.path.join(self._log_dir,
'gomacc.*.INFO.*'))
if not gomacc_paths:
# gomacc logs won't be made every time.
# Only when goma compiler_proxy has
# crashed. So it's usual gomacc logs are not found.
logging.info('No gomacc logs found')
return None
tgz_name = os.path.basename(min(gomacc_paths)) + '.tar.gz'
tgz_path = os.path.join(self._dest_dir, tgz_name)
cros_build_lib.CreateTarball(target=tgz_path,
cwd=self._log_dir,
compression=cros_build_lib.COMP_GZIP)
return tgz_name
def BuildFirmwareArchive(chroot, sysroot, output_directory):
"""Build firmware_from_source.tar.bz2 in chroot's sysroot firmware directory.
Args:
chroot (chroot_lib.Chroot): The chroot to be used.
sysroot (sysroot_lib.Sysroot): The sysroot whose artifacts are being
archived.
output_directory (str): The path were the completed archives should be put.
Returns:
str|None - The archive file path if created, None otherwise.
"""
firmware_root = os.path.join(chroot.path, sysroot.path.lstrip(os.sep),
'firmware')
source_list = [
os.path.relpath(f, firmware_root)
for f in glob.iglob(os.path.join(firmware_root, '*'))
]
if not source_list:
return None
archive_file = os.path.join(output_directory,
constants.FIRMWARE_ARCHIVE_NAME)
cros_build_lib.CreateTarball(archive_file,
firmware_root,
compression=cros_build_lib.COMP_BZIP2,
chroot=chroot.path,
inputs=source_list)
return archive_file
def _UploadGomaccInfoFiles(self):
"""Uploads gomacc INFO files, with gzip'ing.
Returns:
Uploaded file path. If failed, None.
"""
# Since the number of gomacc logs can be large, we'd like to compress them.
# Otherwise, upload will take long (> 10 mins).
# Each gomacc logs file size must be small (around 4KB).
# Find files matched with the pattern in |goma_log_dir|. Sort for
# stabilization.
gomacc_paths = sorted(glob.glob(
os.path.join(self._goma_log_dir, 'gomacc.*.INFO.*')))
if not gomacc_paths:
# gomacc logs won't be made every time.
# Only when goma compiler_proxy has
# crashed. So it's usual gomacc logs are not found.
logging.info('No gomacc logs found')
return None
# Taking the first name as uploaded_filename.
tgz_name = os.path.basename(gomacc_paths[0]) + '.tar.gz'
tgz_path = os.path.join(self._goma_log_dir, tgz_name)
cros_build_lib.CreateTarball(target=tgz_path,
cwd=self._goma_log_dir,
compression=cros_build_lib.COMP_GZIP,
inputs=gomacc_paths)
self._gs_context.CopyInto(tgz_path, self._remote_dir,
filename=tgz_name,
headers=self._headers)
return tgz_name
def ArchiveImageScripts(): """Archive tarball of generated image manipulation scripts.""" target = os.path.join(archive_path, constants.IMAGE_SCRIPTS_TAR) files = glob.glob(os.path.join(image_dir, '*.sh')) files = [os.path.basename(f) for f in files] cros_build_lib.CreateTarball(target, image_dir, inputs=files) self._upload_queue.put([constants.IMAGE_SCRIPTS_TAR])
def main(argv):
"""Generate the delta sysroot
Create a tarball containing a sysroot that can be patched over extracted
prebuilt package contents to create a complete sysroot.
1. Unpack all packages for a board into an unpack_only sysroot directory.
2. Emerge all packages for a board into a build sysroot directory.
3. Create a batch file using:
rsync -rplgoDc --delete --write-batch=<batch> <build_sys> <unpackonly_sys>
4. Put the batch file inside a tarball.
"""
options = _ParseCommandLine(argv)
FinishParsing(options)
cros_build_lib.AssertInsideChroot()
with osutils.TempDir(set_global=False, sudo_rm=True) as tmp_dir:
build_sysroot = os.path.join(tmp_dir, 'build-sys')
unpackonly_sysroot = os.path.join(tmp_dir, 'tmp-sys')
batch_filename = options.out_batch
GenerateSysroot(unpackonly_sysroot, options.board, options.build_tests,
unpack_only=True)
GenerateSysroot(build_sysroot, options.board, options.build_tests,
unpack_only=False)
# Finally create batch file.
CreateBatchFile(build_sysroot, unpackonly_sysroot,
os.path.join(tmp_dir, batch_filename))
cros_build_lib.CreateTarball(
os.path.join(options.out_dir, options.out_file), tmp_dir, sudo=True,
inputs=[batch_filename])
def testFailedOnceHard(self):
"""Test unrecoverable error."""
self.tarResults = [2]
with self.assertRaises(cros_build_lib.RunCommandError) as cm:
cros_build_lib.CreateTarball('foo', 'bar', inputs=['a', 'b'])
self.assertEqual(self.mockRun.call_count, 1)
self.assertEqual(cm.exception.args[1].returncode, 2)
def testUploadLocalTarball(self):
"""Test uploading symbols contains in a local tarball"""
tarball = os.path.join(self.tempdir, 'syms.tar.gz')
cros_build_lib.CreateTarball('syms.tar.gz',
self.tempdir,
compression=cros_build_lib.COMP_GZIP,
inputs=('foo', 'bar', 'some'))
self._testUpload([tarball], sym_paths=self.sym_paths)
def _SetGSUtilUrl(self):
tempfile = os.path.join(self.tempdir, 'tempfile')
osutils.WriteFile(tempfile, 'some content')
gsutil_path = os.path.join(self.tempdir, gs.GSContext.GSUTIL_TAR)
cros_build_lib.CreateTarball(gsutil_path,
self.tempdir,
inputs=[tempfile])
self.GSUTIL_URL = 'file://%s' % gsutil_path
def testSuccessWithTooManyFiles(self):
"""Test a tarfile creation with -T /dev/stdin."""
# pylint: disable=protected-access
num_inputs = cros_build_lib._THRESHOLD_TO_USE_T_FOR_TAR + 1
inputs = ['input%s' % x for x in range(num_inputs)]
largeInputDir = os.path.join(self.tempdir, 'largeinputs')
for i in inputs:
osutils.WriteFile(os.path.join(largeInputDir, i), i, makedirs=True)
cros_build_lib.CreateTarball(self.target, largeInputDir, inputs=inputs)
def _CollectPGOProfiles(self):
"""Collect and upload PGO profiles for the board."""
assert self.archive_path.startswith(self._build_root)
# Look for profiles generated by instrumented LLVM
out_chroot = os.path.abspath(os.path.join(self._build_root, 'chroot'))
cov_data_location = 'build/%s/build/coverage_data' % self._current_board
out_chroot_cov_data = os.path.join(out_chroot, cov_data_location)
try:
profiles_dirs = [
root for root, _, _ in os.walk(out_chroot_cov_data)
if os.path.basename(root) == 'raw_profiles'
]
if not profiles_dirs:
raise Exception('No profile directories found.')
# Get out of chroot profile paths, and convert to in chroot paths
profraws = [
path_util.ToChrootPath(os.path.join(profiles_dir, f))
for profiles_dir in profiles_dirs
for f in os.listdir(profiles_dir)
]
if not profraws:
raise Exception(
'No profraw files found in profiles directory.')
except:
logging.info('Error: Not able to collect correct profiles.')
raise
# Create profdata file and make tarball
in_chroot_path = path_util.ToChrootPath(self.archive_path)
profdata_loc = os.path.join(in_chroot_path, self.PROFDATA)
out_chroot_path = os.path.join(out_chroot, self.archive_path)
out_profdata_loc = os.path.join(out_chroot_path, self.PROFDATA)
# There can bee too many profraws to merge, put them as a list in the file
# so that bash will not complain about arguments getting too long.
profraw_list = os.path.join(in_chroot_path, 'profraw_list')
out_profraw_list = os.path.join(out_chroot_path, 'profraw_list')
with open(out_profraw_list, 'w') as f:
f.write('\n'.join(profraws))
self._merge_cmd = [
'llvm-profdata', 'merge', '-output', profdata_loc, '-f',
profraw_list
]
cros_build_lib.run(self._merge_cmd,
cwd=self._build_root,
enter_chroot=True)
cros_build_lib.CreateTarball(self.PROFDATA_TAR,
cwd=out_chroot_path,
inputs=[out_profdata_loc])
# Upload profdata tarball
self._upload_queue.put([self.PROFDATA_TAR])
def testFailedOnceSoft(self):
"""Force a single retry for CreateTarball."""
self.tarResults = [1, 0]
cros_build_lib.CreateTarball('foo',
'bar',
inputs=['a', 'b'],
timeout=0)
self.assertEqual(self.mockRun.call_count, 2)
def CreateCacheTarball(extensions, outputdir, identifier, tarball):
"""Cache |extensions| in |outputdir| and pack them in |tarball|."""
crxdir = os.path.join(outputdir, 'crx')
jsondir = os.path.join(outputdir, 'json')
validationdir = os.path.join(outputdir, 'validation')
osutils.SafeMakedirs(os.path.join(crxdir, 'extensions', 'managed_users'))
osutils.SafeMakedirs(os.path.join(jsondir, 'extensions', 'managed_users'))
osutils.SafeMakedirs(os.path.join(jsondir, 'extensions', 'child_users'))
was_errors = False
for ext in extensions:
managed_users = extensions[ext].get('managed_users', 'no')
cache_crx = extensions[ext].get('cache_crx', 'yes')
child_users = extensions[ext].get('child_users', 'no')
# Remove fields that shouldn't be in the output file.
for key in ('cache_crx', 'managed_users'):
extensions[ext].pop(key, None)
if cache_crx == 'yes':
if not DownloadCrx(ext, extensions[ext], crxdir):
was_errors = True
elif cache_crx == 'no':
pass
else:
cros_build_lib.Die('Unknown value for "cache_crx" %s for %s',
cache_crx, ext)
if managed_users == 'yes':
json_file = os.path.join(jsondir,
'extensions/managed_users/%s.json' % ext)
json.dump(extensions[ext],
open(json_file, 'w'),
sort_keys=True,
indent=2,
separators=(',', ': '))
if managed_users != 'only':
target_json_dir = 'extensions'
if child_users == 'yes':
target_json_dir = 'extensions/child_users'
json_file = os.path.join(jsondir, target_json_dir, '%s.json' % ext)
json.dump(extensions[ext],
open(json_file, 'w'),
sort_keys=True,
indent=2,
separators=(',', ': '))
if was_errors:
cros_build_lib.Die('FAIL to download some extensions')
CreateValidationFiles(validationdir, crxdir, identifier)
cros_build_lib.CreateTarball(tarball, outputdir)
logging.info('Tarball created %s', tarball)
def testFailedThriceSoft(self):
"""Exhaust retries for recoverable errors."""
self.tarResults = [1, 1, 1]
with self.assertRaises(cros_build_lib.RunCommandError) as cm:
cros_build_lib.CreateTarball('foo',
'bar',
inputs=['a', 'b'],
timeout=0)
self.assertEqual(self.mockRun.call_count, 3)
self.assertEqual(cm.exception.args[1].returncode, 1)
def _ArchiveMoblabVMWorkspace(self, workspace):
"""Try to find the VM files used during testing and archive them.
Args:
workspace: Path to a directory used as moblabvm workspace.
"""
tarball_relpath = 'workspace.tar.bz2'
tarball_path = os.path.join(self.archive_path, tarball_relpath)
cros_build_lib.CreateTarball(tarball_path,
workspace,
compression=cros_build_lib.COMP_BZIP2)
self._Upload(tarball_relpath, 'moblabvm workspace')
def _CreateStatefulUpdate(self):
"""Creates a stateful update tar file so we can test it."""
self._payload = os.path.join(self.tempdir, 'stateful.tgz')
tmp_stateful = os.path.join(self.tempdir, 'temp-stateful')
stateful_dirs = ('var_new', 'dev_image_new')
for d in stateful_dirs:
osutils.SafeMakedirs(os.path.join(tmp_stateful, d))
cros_build_lib.CreateTarball(self._payload,
tmp_stateful,
compression=cros_build_lib.COMP_GZIP,
inputs=stateful_dirs)
self.assertExists(self._payload)
def ArchiveVMFiles(buildroot, test_results_dir, archive_path):
"""Archives the VM memory and disk images into tarballs.
There may be multiple tests (e.g. SimpleTestUpdate and
SimpleTestUpdateAndVerify), and multiple files for each test (one
for the VM disk, and one for the VM memory). We create a separate
tar file for each of these files, so that each can be downloaded
independently.
Args:
buildroot: Build root directory.
test_results_dir: Path from buildroot/chroot to find test results.
This must a subdir of /tmp.
archive_path: Directory the tarballs should be written to.
Returns:
The paths to the tarballs.
"""
images_dir = os.path.join(buildroot, 'chroot',
test_results_dir.lstrip('/'))
images = []
for path, _, filenames in os.walk(images_dir):
images.extend([
os.path.join(path, filename) for filename in fnmatch.filter(
filenames, constants.VM_DISK_PREFIX + '*')
])
images.extend([
os.path.join(path, filename)
for filename in fnmatch.filter(filenames, constants.VM_MEM_PREFIX +
'*')
])
tar_files = []
for image_path in images:
image_rel_path = os.path.relpath(image_path, images_dir)
image_parent_dir = os.path.dirname(image_path)
image_file = os.path.basename(image_path)
tarball_path = os.path.join(
archive_path, "%s.tar" % image_rel_path.replace('/', '_'))
# Note that tar will chdir to |image_parent_dir|, so that |image_file|
# is at the top-level of the tar file.
cros_build_lib.CreateTarball(tarball_path,
image_parent_dir,
compression=cros_build_lib.COMP_BZIP2,
inputs=[image_file])
tar_files.append(tarball_path)
return tar_files
def CreateTestImageTar(self):
"""Create and upload chromiumos_test_image.tar.xz.
This depends on the WorkspaceBuildImage stage having previously created
chromiumos_test_image.bin.
"""
with osutils.TempDir(prefix='test_image_dir') as tempdir:
target = os.path.join(tempdir, constants.TEST_IMAGE_TAR)
cros_build_lib.CreateTarball(target,
inputs=[constants.TEST_IMAGE_BIN],
cwd=self.GetImageDirSymlink(
pointer='latest',
buildroot=self._build_root),
compression=cros_build_lib.COMP_GZIP)
self.UploadDummyArtifact(target)
def BuildTargetUnitTestTarball(chroot, sysroot, result_path):
"""Build the unittest tarball.
Args:
chroot (chroot_lib.Chroot): Chroot where the tests were run.
sysroot (sysroot_lib.Sysroot): The sysroot where the tests were run.
result_path (str): The directory where the archive should be created.
"""
tarball = 'unit_tests.tar'
tarball_path = os.path.join(result_path, tarball)
cwd = os.path.join(chroot.path, sysroot.path.lstrip(os.sep),
constants.UNITTEST_PKG_PATH)
result = cros_build_lib.CreateTarball(tarball_path, cwd, chroot=chroot.path,
compression=cros_build_lib.COMP_NONE,
error_code_ok=True)
return tarball_path if result.returncode == 0 else None
def BuildFirmwareArchive(chroot, sysroot, output_directory):
"""Build firmware_from_source.tar.bz2 in chroot's sysroot firmware directory.
Args:
chroot (chroot_lib.Chroot): The chroot to be used.
sysroot (sysroot_lib.Sysroot): The sysroot whose artifacts are being
archived.
output_directory (str): The path were the completed archives should be put.
Returns:
str|None - The archive file path if created, None otherwise.
"""
firmware_root = os.path.join(chroot.path, sysroot.path.lstrip(os.sep),
'firmware')
if not os.path.exists(firmware_root):
return None
# Private fingerprint libraries should not be uploaded.
private_fingerprint_dirs = glob.glob(os.path.join(
firmware_root, '**/ec-private/fingerprint'),
recursive=True)
source_list = []
for directory, _, filenames in os.walk(firmware_root):
if any(directory.startswith(e) for e in private_fingerprint_dirs):
continue
for filename in filenames:
source_list.append(
os.path.relpath(os.path.join(directory, filename),
firmware_root))
if not source_list:
return None
archive_file = os.path.join(output_directory,
constants.FIRMWARE_ARCHIVE_NAME)
cros_build_lib.CreateTarball(archive_file,
firmware_root,
compression=cros_build_lib.COMP_BZIP2,
chroot=chroot.path,
inputs=source_list)
return archive_file
def _UploadGomaccInfoFiles(self):
"""Uploads gomacc INFO files, with gzip'ing.
Returns:
Uploaded file path. If failed, None.
"""
# Since the number of gomacc logs can be large, we'd like to compress them.
# Otherwise, upload will take long (> 10 mins).
# Each gomacc logs file size must be small (around 4KB).
# Find files matched with the pattern in |goma_log_dir|.
# The paths were themselves used as the inputs for the create
# tarball, but there can be too many of them. As long as we have
# files we'll just tar up the entire directory.
gomacc_paths = glob.glob(
os.path.join(self._goma_log_dir, 'gomacc.*.INFO.*'))
if not gomacc_paths:
# gomacc logs won't be made every time.
# Only when goma compiler_proxy has
# crashed. So it's usual gomacc logs are not found.
logging.info('No gomacc logs found')
return None
# Taking the alphabetically first name as uploaded_filename.
tgz_name = os.path.basename(min(gomacc_paths)) + '.tar.gz'
# When using the pigz compressor (what we use for gzip) to create an
# archive in a folder that is also a source for contents, there is a race
# condition involving the created archive itself that can cause it to fail
# creating the archive. To avoid this, make the archive in a tempdir.
with osutils.TempDir() as tempdir:
tgz_path = os.path.join(tempdir, tgz_name)
cros_build_lib.CreateTarball(target=tgz_path,
cwd=self._goma_log_dir,
compression=cros_build_lib.COMP_GZIP)
self._gs_context.CopyInto(tgz_path,
self._remote_dir,
filename=tgz_name,
headers=self._headers)
return tgz_name
def _BuildTarball(self,
input_list,
tarball_output,
compressed=True,
**kwargs):
"""Tars and zips files and directories from input_list to tarball_output.
Args:
input_list: A list of files and directories to be archived.
tarball_output: Path of output tar archive file.
compressed: Whether or not the tarball should be compressed with pbzip2.
**kwargs: Keyword arguments to pass to CreateTarball.
Returns:
Return value of cros_build_lib.CreateTarball.
"""
for pathname in input_list:
if os.path.exists(os.path.join(self.archive_basedir, pathname)):
break
else:
# If any of them exist we can create an archive, but if none
# do then we need to stop. For now, since we either pass in a
# handful of directories we don't necessarily check, or actually
# search the filesystem for lots of files, this is far more
# efficient than building out a list of files that do exist.
return None
compressor = cros_build_lib.COMP_NONE
chroot = None
if compressed:
compressor = cros_build_lib.COMP_BZIP2
if not cros_build_lib.IsInsideChroot():
chroot = path_util.FromChrootPath('/')
return cros_build_lib.CreateTarball(tarball_output,
self.archive_basedir,
compression=compressor,
chroot=chroot,
inputs=input_list,
**kwargs)
def _ArchiveChromeEbuildEnv(self):
"""Generate and upload Chrome ebuild environment."""
files = glob.glob(os.path.join(self._pkg_dir, constants.CHROME_CP) + '-*')
if not files:
raise artifact_stages.NothingToArchiveException(
'Failed to find package %s' % constants.CHROME_CP)
if len(files) > 1:
logging.PrintBuildbotStepWarnings()
logging.warning('Expected one package for %s, found %d',
constants.CHROME_CP, len(files))
chrome_dir = sorted(files)[-1]
env_bzip = os.path.join(chrome_dir, 'environment.bz2')
with osutils.TempDir(prefix='chrome-sdk-stage') as tempdir:
# Convert from bzip2 to tar format.
bzip2 = cros_build_lib.FindCompressor(cros_build_lib.COMP_BZIP2)
cros_build_lib.RunCommand(
[bzip2, '-d', env_bzip, '-c'],
log_stdout_to_file=os.path.join(tempdir, constants.CHROME_ENV_FILE))
env_tar = os.path.join(self.archive_path, constants.CHROME_ENV_TAR)
cros_build_lib.CreateTarball(env_tar, tempdir)
self._upload_queue.put([os.path.basename(env_tar)])
def _BuildTarball(self, input_list, tarball_output, compressed=True,
**kwargs):
"""Tars and zips files and directories from input_list to tarball_output.
Args:
input_list: A list of files and directories to be archived.
tarball_output: Path of output tar archive file.
compressed: Whether or not the tarball should be compressed with pbzip2.
**kwargs: Keyword arguments to pass to CreateTarball.
Returns:
Return value of cros_build_lib.CreateTarball.
"""
compressor = cros_build_lib.COMP_NONE
chroot = None
if compressed:
compressor = cros_build_lib.COMP_BZIP2
if not cros_build_lib.IsInsideChroot():
chroot = path_util.FromChrootPath('/')
return cros_build_lib.CreateTarball(
tarball_output, self.archive_basedir, compression=compressor,
chroot=chroot, inputs=input_list, **kwargs)
def collect_logs():
remove_old_tarballs()
tempdir = tempfile.mkdtemp(prefix=TMPDIR_PREFIX, dir=TMPDIR)
os.chmod(tempdir, 0o777)
try:
for name, path in LOG_DIRS.iteritems():
if not os.path.exists(path):
continue
if os.path.isdir(path):
shutil.copytree(path, os.path.join(tempdir, name))
else:
shutil.copyfile(path, os.path.join(tempdir, name))
cmd = ['mobmoncli', 'GetStatus']
cros_build_lib.RunCommand(cmd,
log_stdout_to_file=os.path.join(
tempdir, 'mobmonitor_getstatus'))
finally:
tarball = '%s.tgz' % tempdir
cros_build_lib.CreateTarball(tarball, tempdir)
osutils.RmDir(tempdir, ignore_missing=True)
return tarball
def ArchiveFilesFromImageDir(images_dir, archive_path):
"""Archives the files into tarballs if they match a prefix from prefix_list.
Create and return a list of tarballs from the images_dir of files that match
VM disk and memory prefixes.
Args:
images_dir (str): The directory containing the images to archive.
archive_path (str): The directory where the archives should be created.
Returns:
list[str] - The paths to the tarballs.
"""
images = []
for prefix in [constants.VM_DISK_PREFIX, constants.VM_MEM_PREFIX]:
for path, _, filenames in os.walk(images_dir):
images.extend([
os.path.join(path, filename)
for filename in fnmatch.filter(filenames, prefix + '*')
])
tar_files = []
for image_path in images:
image_rel_path = os.path.relpath(image_path, images_dir)
image_parent_dir = os.path.dirname(image_path)
image_file = os.path.basename(image_path)
tarball_path = os.path.join(
archive_path, '%s.tar' % image_rel_path.replace('/', '_'))
# Note that tar will chdir to |image_parent_dir|, so that |image_file|
# is at the top-level of the tar file.
cros_build_lib.CreateTarball(tarball_path,
image_parent_dir,
compression=cros_build_lib.COMP_BZIP2,
inputs=[image_file])
tar_files.append(tarball_path)
return tar_files
def ArchiveImages(image_dir, output_dir):
"""Create a .tar.xz archive for each image that has been created.
Args:
image_dir (str): The directory where the images are located.
output_dir (str): The location where the archives should be created.
Returns:
list[str]: The list of created file names.
"""
files = os.listdir(image_dir)
archives = []
# Filter down to the ones that exist first.
images = {img: tar for img, tar in IMAGE_TARS.items() if img in files}
for img, tar in images.items():
target = os.path.join(output_dir, tar)
cros_build_lib.CreateTarball(target,
image_dir,
inputs=(img, ),
print_cmd=False)
archives.append(tar)
return archives
