def _CreateVMImage(src_dir, dest_dir):
"""Creates a VM image from a given chromiumos image.
Args:
src_dir: Path to the directory containing (non-VM) image. Defaults to None
to use the latest image for the board.
dest_dir: Path to the directory where the VM image should be written.
Returns:
The path of the created VM image.
"""
# image_to_vm.sh only runs in chroot, but src_dir / dest_dir may not be
# reachable from chroot. Also, image_to_vm.sh needs all the contents of
# src_dir to work correctly (it silently does the wrong thing if some files
# are missing).
# So, create a tempdir reachable from chroot, copy everything to that path,
# create vm image there and finally move it all to dest_dir.
with chroot_util.TempDirInChroot() as tempdir:
logging.debug('Copying images from %s to %s.', src_dir, tempdir)
osutils.CopyDirContents(src_dir, tempdir)
# image_to_vm.sh doesn't let us provide arbitrary names for the input image.
# Instead, it picks the name based on whether we pass in --test_image or not
# (and doesn't use that flag for anything else).
cmd = [
path_util.ToChrootPath(
os.path.join(constants.CROSUTILS_DIR, 'image_to_vm.sh')),
'--from=%s' % path_util.ToChrootPath(tempdir),
'--disk_layout=16gb-rootfs',
'--test_image',
]
try:
cros_build_lib.run(cmd,
enter_chroot=True,
cwd=constants.SOURCE_ROOT)
except cros_build_lib.RunCommandError as e:
raise SetupError('Failed to create VM image for %s: %s' %
(src_dir, e))
# Preserve most content, although we should need only the generated VM
# image. Other files like boot.desc might be needed elsewhere, but the
# source images should no longer be needed.
osutils.SafeUnlink(os.path.join(tempdir, constants.BASE_IMAGE_BIN),
sudo=True)
osutils.SafeUnlink(os.path.join(tempdir, constants.TEST_IMAGE_BIN),
sudo=True)
osutils.CopyDirContents(tempdir, dest_dir)
# The exact name of the output image is hard-coded in image_to_vm.sh
return os.path.join(dest_dir, constants.VM_IMAGE_BIN)
def PrepareMoblabVmImageCache(vms, builder, payload_dirs):
"""Preload the given payloads into the moblab VM image cache.
Args:
vms (MoblabVm): The Moblab VM.
builder (str): The builder path, used to name the cache dir.
payload_dirs (list[str]): List of payload directories to load.
Returns:
str: Absolute path to the image cache path.
"""
with vms.MountedMoblabDiskContext() as disk_dir:
image_cache_root = os.path.join(disk_dir, 'static/prefetched')
# If by any chance this path exists, the permission bits are surely
# nonsense, since 'moblab' user doesn't exist on the host system.
osutils.RmDir(image_cache_root, ignore_missing=True, sudo=True)
image_cache_dir = os.path.join(image_cache_root, builder)
osutils.SafeMakedirsNonRoot(image_cache_dir)
for payload_dir in payload_dirs:
osutils.CopyDirContents(payload_dir,
image_cache_dir,
allow_nonempty=True)
image_cache_rel_dir = image_cache_dir[len(disk_dir):].strip('/')
return os.path.join('/', 'mnt/moblab', image_cache_rel_dir)
def testCopyingSymlinks(self):
in_dir = os.path.join(self.tempdir, 'input')
in_dir_link = os.path.join(in_dir, 'link')
in_dir_symlinks_dir = os.path.join(in_dir, 'holding_symlink')
in_dir_symlinks_dir_link = os.path.join(in_dir_symlinks_dir, 'link')
out_dir = os.path.join(self.tempdir, 'output')
out_dir_link = os.path.join(out_dir, 'link')
out_dir_symlinks_dir = os.path.join(out_dir, 'holding_symlink')
out_dir_symlinks_dir_link = os.path.join(out_dir_symlinks_dir, 'link')
# Create directories and symlinks appropriately.
osutils.SafeMakedirsNonRoot(in_dir)
osutils.SafeMakedirsNonRoot(in_dir_symlinks_dir)
os.symlink(self.tempdir, in_dir_link)
os.symlink(self.tempdir, in_dir_symlinks_dir_link)
osutils.SafeMakedirsNonRoot(out_dir)
# Actual execution.
osutils.CopyDirContents(in_dir, out_dir, symlinks=True)
# Assertions.
self.assertTrue(os.path.islink(out_dir_link))
self.assertTrue(os.path.islink(out_dir_symlinks_dir_link))
def testDestinationDirDoesNotExistRaises(self):
"""Coping to a non-existent destination dir raises."""
in_dir = os.path.join(self.tempdir, 'input')
out_dir = os.path.join(self.tempdir, 'output')
osutils.SafeMakedirsNonRoot(in_dir)
with self.assertRaises(osutils.BadPathsException):
osutils.CopyDirContents(in_dir, out_dir)
def testCopyEmptyDir(self): """Copy "empty" contents from a dir.""" in_dir = os.path.join(self.tempdir, 'input') out_dir = os.path.join(self.tempdir, 'output') osutils.SafeMakedirsNonRoot(in_dir) osutils.SafeMakedirsNonRoot(out_dir) osutils.CopyDirContents(in_dir, out_dir)
def CopyAllDlcs(sysroot, install_root_dir):
"""Copies all DLC image files into the images directory.
Copies the DLC image files in the given build directory into the given DLC
image directory. If the DLC build directory does not exist, or there is no DLC
for that board, this function does nothing.
Args:
sysroot: Path to directory containing DLC images, e.g /build/<board>.
install_root_dir: Path to DLC output directory,
e.g. src/build/images/<board>/<version>.
"""
build_dir = os.path.join(sysroot, DLC_IMAGE_DIR)
output_dir = os.path.join(install_root_dir, 'dlc')
if not os.path.exists(build_dir):
logging.info('DLC build directory (%s) does not exist, ignoring.',
build_dir)
return
if not os.listdir(build_dir):
logging.info('There are no DLC(s) to copy to output, ignoring.')
return
logging.info('Copying all DLC images from %s to %s.', build_dir,
output_dir)
logging.info('Detected the following DLCs: %s',
', '.join(os.listdir(build_dir)))
osutils.SafeMakedirs(output_dir)
osutils.CopyDirContents(build_dir, output_dir)
logging.info('Done copying the DLCs to their destination.')
def testDestinationDirNonEmptyAllowNonEmptySet(self): """Copying to a non-empty destination with allow_nonempty does not raise.""" in_dir = os.path.join(self.tempdir, 'input') out_dir = os.path.join(self.tempdir, 'output') osutils.SafeMakedirsNonRoot(in_dir) osutils.SafeMakedirsNonRoot(out_dir) osutils.SafeMakedirsNonRoot(os.path.join(out_dir, 'blah')) osutils.CopyDirContents(in_dir, out_dir, allow_nonempty=True)
def testCopyTree(self):
"""Copy from a dir that contains files."""
in_dir = os.path.join(self.tempdir, 'input')
out_dir = os.path.join(self.tempdir, 'output')
osutils.SafeMakedirsNonRoot(in_dir)
osutils.SafeMakedirsNonRoot(os.path.join(in_dir, 'a'))
osutils.WriteFile(os.path.join(in_dir, 'a', 'b.txt'), 'bbb')
osutils.SafeMakedirsNonRoot(out_dir)
osutils.CopyDirContents(in_dir, out_dir)
self.assertEqual(
osutils.ReadFile(os.path.join(out_dir, 'a', 'b.txt')).strip(), 'bbb')
def SetupDlcImageFiles(self, dlc_dir):
"""Prepares the directory dlc_dir with all the files a DLC needs.
Args:
dlc_dir: (str) The path to where to setup files inside the DLC.
"""
dlc_root_dir = os.path.join(dlc_dir, self._DLC_ROOT_DIR)
osutils.SafeMakedirs(dlc_root_dir)
osutils.CopyDirContents(self.src_dir, dlc_root_dir, symlinks=True)
self.PrepareLsbRelease(dlc_dir)
self.CollectExtraResources(dlc_dir)
self.SquashOwnerships(dlc_dir)
def testCopyingSubDirWithPathArgs(self):
"""Copying given |Path| arguments works properly for subdirectories."""
in_dir = Path(self.tempdir) / 'input'
osutils.SafeMakedirs(in_dir)
tmp_file = in_dir / 'subdir' / 'a.txt'
osutils.SafeMakedirs(tmp_file.parent)
tmp_file.write_text('aaa', encoding='utf-8')
out_dir = Path(self.tempdir) / 'output'
osutils.SafeMakedirs(out_dir)
osutils.CopyDirContents(in_dir, out_dir, symlinks=True)
out_tmp_file = out_dir / 'subdir' / tmp_file.name
self.assertEqual(out_tmp_file.read_text(encoding='utf-8'), 'aaa')
def testCopyingSymlinksAndFilesWithPathArgs(self):
"""Copying given |Path| arguments works properly for symlinks+files."""
in_dir = Path(self.tempdir) / 'input'
osutils.SafeMakedirs(in_dir)
tmp_file = in_dir / 'a.txt'
tmp_file.write_text('aaa', encoding='utf-8')
tmp_file_link = tmp_file.with_suffix('.link')
tmp_file_link.symlink_to(tmp_file)
out_dir = Path(self.tempdir) / 'output'
osutils.SafeMakedirs(out_dir)
osutils.CopyDirContents(in_dir, out_dir, symlinks=True)
out_tmp_file = out_dir / tmp_file.name
self.assertEqual(out_tmp_file.read_text(encoding='utf-8'), 'aaa')
out_tmp_file_link = out_dir / tmp_file_link.name
self.assertEqual(Path(os.readlink(out_tmp_file_link)), tmp_file)
def BundleAFDOGenerationArtifacts(is_orderfile, chroot, chrome_root,
build_target, output_dir):
"""Generate artifacts for toolchain-related AFDO artifacts.
Args:
is_orderfile (boolean): The generation is for orderfile (True) or
for AFDO (False).
chroot (chroot_lib.Chroot): The chroot in which the sysroot should be built.
chrome_root (str): Path to Chrome root.
build_target (build_target_lib.BuildTarget): The build target.
output_dir (str): The location outside the chroot where the files should be
stored.
Returns:
list[str]: The list of tarballs of artifacts.
"""
chroot_args = chroot.get_enter_args()
with chroot.tempdir() as tempdir:
if is_orderfile:
generate_orderfile = toolchain_util.GenerateChromeOrderfile(
board=build_target.name,
output_dir=tempdir,
chrome_root=chrome_root,
chroot_path=chroot.path,
chroot_args=chroot_args)
generate_orderfile.Perform()
else:
generate_afdo = toolchain_util.GenerateBenchmarkAFDOProfile(
board=build_target.name,
output_dir=tempdir,
chroot_path=chroot.path,
chroot_args=chroot_args)
generate_afdo.Perform()
files = []
for path in osutils.DirectoryIterator(tempdir):
if os.path.isfile(path):
rel_path = os.path.relpath(path, tempdir)
files.append(os.path.join(output_dir, rel_path))
osutils.CopyDirContents(tempdir, output_dir, allow_nonempty=True)
return files
def CreateChromeRoot(chroot, build_target, output_dir):
"""Create the chrome sysroot.
Args:
chroot (chroot_lib.Chroot): The chroot in which the sysroot should be built.
build_target (build_target_lib.BuildTarget): The build target.
output_dir (str): The location outside the chroot where the files should be
stored.
Returns:
list[str]: The list of created files.
Raises:
CrosGenerateSysrootError: When cros_generate_sysroot does not complete
successfully.
"""
chroot_args = chroot.get_enter_args()
extra_env = {'USE': 'chrome_internal'}
with chroot.tempdir() as tempdir:
in_chroot_path = os.path.relpath(tempdir, chroot.path)
cmd = [
'cros_generate_sysroot', '--out-dir', in_chroot_path, '--board',
build_target.name, '--deps-only', '--package', constants.CHROME_CP
]
try:
cros_build_lib.run(cmd,
enter_chroot=True,
extra_env=extra_env,
chroot_args=chroot_args)
except cros_build_lib.RunCommandError as e:
raise CrosGenerateSysrootError(
'Error encountered when running cros_generate_sysroot: %s' % e,
e)
files = []
for path in osutils.DirectoryIterator(tempdir):
if os.path.isfile(path):
rel_path = os.path.relpath(path, tempdir)
files.append(os.path.join(output_dir, rel_path))
osutils.CopyDirContents(tempdir, output_dir, allow_nonempty=True)
return files
def generate_lints(board: str, ebuild_path: str) -> Path:
"""Collects the lints for a given package on a given board.
Args:
board: the board to collect lints for.
ebuild_path: the path to the ebuild to collect lints for.
Returns:
The path to a tmpdir that all of the lint YAML files (if any) will be in.
This will also be populated by JSON files containing InvocationMetadata.
The generation of this is handled by our compiler wrapper.
"""
logging.info('Running lints for %r on board %r', ebuild_path, board)
osutils.RmDir(LINT_BASE, ignore_missing=True, sudo=True)
osutils.SafeMakedirs(LINT_BASE, 0o777, sudo=True)
# FIXME(gbiv): |test| might be better here?
result = cros_build_lib.run(
[f'ebuild-{board}', ebuild_path, 'clean', 'compile'],
check=False,
print_cmd=True,
extra_env={'WITH_TIDY': 'tricium'},
capture_output=True,
encoding='utf-8',
errors='replace',
)
if result.returncode:
status = f'failed with code {result.returncode}; output:\n{result.stdout}'
log_fn = logging.warning
else:
status = 'succeeded'
log_fn = logging.info
log_fn('Running |ebuild| on %s %s', ebuild_path, status)
lint_tmpdir = tempfile.mkdtemp(prefix='tricium_tidy')
osutils.CopyDirContents(LINT_BASE, lint_tmpdir)
return Path(lint_tmpdir)
def testNotCopyingSymlinks(self):
# Create temporary to symlink against.
tmp_file = os.path.join(self.tempdir, 'a.txt')
osutils.WriteFile(tmp_file, 'aaa')
tmp_subdir = os.path.join(self.tempdir, 'tmp')
osutils.SafeMakedirsNonRoot(tmp_subdir)
tmp_subdir_file = os.path.join(tmp_subdir, 'b.txt')
osutils.WriteFile(tmp_subdir_file, 'bbb')
in_dir = os.path.join(self.tempdir, 'input')
in_dir_link = os.path.join(in_dir, 'link')
in_dir_symlinks_dir = os.path.join(in_dir, 'holding_symlink')
in_dir_symlinks_dir_link = os.path.join(in_dir_symlinks_dir, 'link')
out_dir = os.path.join(self.tempdir, 'output')
out_dir_file = os.path.join(out_dir, 'link')
out_dir_symlinks_dir = os.path.join(out_dir, 'holding_symlink')
out_dir_symlinks_dir_subdir = os.path.join(out_dir_symlinks_dir,
'link')
# Create directories and symlinks appropriately.
osutils.SafeMakedirsNonRoot(in_dir)
osutils.SafeMakedirsNonRoot(in_dir_symlinks_dir)
os.symlink(tmp_file, in_dir_link)
os.symlink(tmp_subdir, in_dir_symlinks_dir_link)
osutils.SafeMakedirsNonRoot(out_dir)
# Actual execution.
osutils.CopyDirContents(in_dir, out_dir, symlinks=False)
# Assertions.
self.assertFalse(os.path.islink(out_dir_file))
self.assertFalse(os.path.islink(out_dir_symlinks_dir_subdir))
self.assertTrue(filecmp.cmp(out_dir_file, tmp_file))
self.assertTrue(
filecmp.cmp(os.path.join(out_dir_symlinks_dir_subdir, 'b.txt'),
tmp_subdir_file))
def _sync(self, src, dest):
logging.info('Syncing %s to %s', src, dest)
# TODO: This would probably be more efficient with rsync.
osutils.EmptyDir(dest)
osutils.CopyDirContents(src, dest)
