def GetImagePathWithXbuddy(path, board, version=None,
static_dir=DEFAULT_STATIC_DIR,
lookup_only=False, silent=False):
"""Gets image path and resolved XBuddy path using xbuddy.
Ask xbuddy to translate |path|, and if necessary, download and stage the
image, then return a translated path to the image. Also returns the resolved
XBuddy path, which may be useful for subsequent calls in case the argument is
an alias.
Args:
path: The xbuddy path.
board: The default board to use if board is not specified in |path|.
version: The default version to use if one is not specified in |path|.
static_dir: Static directory to stage the image in.
lookup_only: Caller only wants to translate the path not download the
artifact.
silent: Suppress error messages.
Returns:
A tuple consisting of a translated path to the image
(build-id/version/image_name) as well as the fully resolved XBuddy path (in
the case where |path| is an XBuddy alias).
"""
# Since xbuddy often wants to use gsutil from $PATH, make sure our local copy
# shows up first.
upath = os.environ['PATH'].split(os.pathsep)
upath.insert(0, os.path.dirname(gs.GSContext.GetDefaultGSUtilBin()))
os.environ['PATH'] = os.pathsep.join(upath)
# If we are using the progress bar, quiet the logging output of cherrypy.
if cherrypy and command.UseProgressBar():
if (hasattr(cherrypy.log, 'access_log') and
hasattr(cherrypy.log, 'error_log')):
cherrypy.log.access_log.setLevel(logging.NOTICE)
cherrypy.log.error_log.setLevel(logging.NOTICE)
else:
cherrypy.config.update({'server.log_to_screen': False})
xb = xbuddy.XBuddy(static_dir=static_dir, board=board, version=version,
log_screen=False)
path_list = GetXbuddyPath(path).rsplit(os.path.sep)
try:
if lookup_only:
build_id, file_name = xb.Translate(path_list)
else:
build_id, file_name = xb.Get(path_list)
resolved_path, _ = xb.LookupAlias(os.path.sep.join(path_list))
return os.path.join(build_id, file_name), resolved_path
except xbuddy.XBuddyException as e:
if not silent:
logging.error('Locating image "%s" failed. The path might not be valid '
'or the image might not exist.', path)
raise ImagePathError('Cannot locate image %s: %s' % (path, e))
except build_artifact.ArtifactDownloadError as e:
if not silent:
logging.error('Downloading image "%s" failed.', path)
raise ArtifactDownloadError('Cannot download image %s: %s' % (path, e))
def Run(self):
"""Run cros build."""
self.options.Freeze()
if not self.host:
if not (self.board or self.brick):
cros_build_lib.Die('You did not specify a board/brick to build for. '
'You need to be in a brick directory or set '
'--board/--brick/--host')
if self.brick and self.brick.legacy:
cros_build_lib.Die('--brick should not be used with board names. Use '
'--board=%s instead.' % self.brick.config['name'])
if self.board:
chroot_args = ['--board', self.board]
else:
chroot_args = None
commandline.RunInsideChroot(self, chroot_args=chroot_args)
if not (self.build_pkgs or self.options.init_only):
cros_build_lib.Die('No packages found, nothing to build.')
# Set up the sysroots if not building for host.
if self.brick or self.board:
chroot_util.SetupBoard(
brick=self.brick, board=self.board,
update_chroot=self.chroot_update,
update_host_packages=self.options.host_packages_update,
use_binary=self.options.binary)
if not self.options.init_only:
# Preliminary: enable all packages that only have a live ebuild.
if self.options.enable_only_latest:
workon = workon_helper.WorkonHelper(self.sysroot)
workon.StartWorkingOnPackages([], use_workon_only=True)
if command.UseProgressBar():
op = BrilloBuildOperation()
op.Run(
parallel.RunParallelSteps, [self._CheckDependencies, self._Build],
log_level=logging.DEBUG)
if self.options.test:
self._Test()
else:
parallel.RunParallelSteps([self._CheckDependencies, self._Build])
if self.options.test:
self._Test()
logging.notice('Build completed successfully.')
def UpdateRootfs(self):
"""Update the rootfs partition of the device (utilizing nebraska)."""
logging.notice('Updating rootfs partition...')
nebraska_bin = os.path.join(self.device_dev_dir,
self.REMOTE_NEBRASKA_FILENAME)
nebraska = nebraska_wrapper.RemoteNebraskaWrapper(
self.device,
nebraska_bin=nebraska_bin,
update_payloads_address='file://' + self.device_payload_dir,
update_metadata_dir=self.device_payload_dir)
try:
nebraska.Start()
# Use the localhost IP address (default) to ensure that update engine
# client can connect to the nebraska.
nebraska_url = nebraska.GetURL(critical_update=True)
cmd = [
self.REMOTE_UPDATE_ENGINE_BIN_FILENAME, '--check_for_update',
'--omaha_url="%s"' % nebraska_url
]
self.device.run(cmd, **self._cmd_kwargs)
# If we are using a progress bar, update it every 0.5s instead of 10s.
if command.UseProgressBar():
update_check_interval = self.UPDATE_CHECK_INTERVAL_PROGRESSBAR
oper = operation.ProgressBarOperation()
else:
update_check_interval = self.UPDATE_CHECK_INTERVAL_NORMAL
oper = None
end_message_not_printed = True
# Loop until update is complete.
while True:
# Number of times to retry `update_engine_client --status`. See
# crbug.com/744212.
update_engine_status_retry = 30
op, progress = retry_util.RetryException(
cros_build_lib.RunCommandError,
update_engine_status_retry,
self.GetUpdateStatus,
self.device, ['CURRENT_OP', 'PROGRESS'],
delay_sec=DELAY_SEC_FOR_RETRY)[0:2]
logging.info('Waiting for update...status: %s at progress %s',
op, progress)
if op == UPDATE_STATUS_UPDATED_NEED_REBOOT:
logging.info('Update completed.')
break
if op == UPDATE_STATUS_IDLE:
# Something went wrong. Try to get last error code.
cmd = ['cat', self.REMOTE_UPDATE_ENGINE_LOGFILE_PATH]
log = self.device.run(cmd).stdout.strip().splitlines()
err_str = 'Updating payload state for error code: '
targets = [line for line in log if err_str in line]
logging.debug('Error lines found: %s', targets)
if not targets:
raise RootfsUpdateError(
'Update failed with unexpected update status: %s' %
op)
else:
# e.g 20 (ErrorCode::kDownloadStateInitializationError)
raise RootfsUpdateError(
targets[-1].rpartition(err_str)[2])
if oper is not None:
if op == UPDATE_STATUS_DOWNLOADING:
oper.ProgressBar(float(progress))
elif end_message_not_printed and op == UPDATE_STATUS_FINALIZING:
oper.Cleanup()
logging.info('Finalizing image.')
end_message_not_printed = False
time.sleep(update_check_interval)
# TODO(ahassani): Scope the Exception to finer levels. For example we don't
# need to revert the boot partition if the Nebraska fails to start, etc.
except Exception as e:
logging.error('Rootfs update failed %s', e)
self.RevertBootPartition()
logging.warning(nebraska.PrintLog()
or 'No nebraska log is available.')
raise RootfsUpdateError('Failed to perform rootfs update: %r' % e)
finally:
nebraska.Stop()
nebraska.CollectLogs(
os.path.join(self.tempdir, self.LOCAL_NEBRASKA_LOG_FILENAME))
self.device.CopyFromDevice(
self.REMOTE_UPDATE_ENGINE_LOGFILE_PATH,
os.path.join(
self.tempdir,
os.path.basename(self.REMOTE_UPDATE_ENGINE_LOGFILE_PATH)),
follow_symlinks=True,
**self._cmd_kwargs_omit_error)
def Deploy(device, packages, board=None, emerge=True, update=False, deep=False,
deep_rev=False, clean_binpkg=True, root='/', strip=True,
emerge_args=None, ssh_private_key=None, ping=True, force=False,
dry_run=False):
"""Deploys packages to a device.
Args:
device: commandline.Device object; None to use the default device.
packages: List of packages (strings) to deploy to device.
board: Board to use; None to automatically detect.
emerge: True to emerge package, False to unmerge.
update: Check installed version on device.
deep: Install dependencies also. Implies |update|.
deep_rev: Install reverse dependencies. Implies |deep|.
clean_binpkg: Clean outdated binary packages.
root: Package installation root path.
strip: Run strip_package to filter out preset paths in the package.
emerge_args: Extra arguments to pass to emerge.
ssh_private_key: Path to an SSH private key file; None to use test keys.
ping: True to ping the device before trying to connect.
force: Ignore sanity checks and prompts.
dry_run: Print deployment plan but do not deploy anything.
Raises:
ValueError: Invalid parameter or parameter combination.
DeployError: Unrecoverable failure during deploy.
"""
if deep_rev:
deep = True
if deep:
update = True
if not packages:
raise DeployError('No packages provided, nothing to deploy.')
if update and not emerge:
raise ValueError('Cannot update and unmerge.')
if device:
hostname, username, port = device.hostname, device.username, device.port
else:
hostname, username, port = None, None, None
lsb_release = None
sysroot = None
try:
with remote_access.ChromiumOSDeviceHandler(
hostname, port=port, username=username, private_key=ssh_private_key,
base_dir=_DEVICE_BASE_DIR, ping=ping) as device:
lsb_release = device.lsb_release
board = cros_build_lib.GetBoard(device_board=device.board,
override_board=board)
if not force and board != device.board:
raise DeployError('Device (%s) is incompatible with board %s. Use '
'--force to deploy anyway.' % (device.board, board))
sysroot = cros_build_lib.GetSysroot(board=board)
if clean_binpkg:
logging.notice('Cleaning outdated binary packages from %s', sysroot)
portage_util.CleanOutdatedBinaryPackages(sysroot)
if not device.IsDirWritable(root):
# Only remounts rootfs if the given root is not writable.
if not device.MountRootfsReadWrite():
raise DeployError('Cannot remount rootfs as read-write. Exiting.')
# Obtain list of packages to upgrade/remove.
pkg_scanner = _InstallPackageScanner(sysroot)
pkgs, listed, num_updates = pkg_scanner.Run(
device, root, packages, update, deep, deep_rev)
if emerge:
action_str = 'emerge'
else:
pkgs.reverse()
action_str = 'unmerge'
if not pkgs:
logging.notice('No packages to %s', action_str)
return
logging.notice('These are the packages to %s:', action_str)
for i, pkg in enumerate(pkgs):
logging.notice('%s %d) %s', '*' if pkg in listed else ' ', i + 1, pkg)
if dry_run or not _ConfirmDeploy(num_updates):
return
# Select function (emerge or unmerge) and bind args.
if emerge:
func = functools.partial(_EmergePackages, pkgs, device, strip,
sysroot, root, emerge_args)
else:
func = functools.partial(_UnmergePackages, pkgs, device, root)
# Call the function with the progress bar or with normal output.
if command.UseProgressBar():
op = BrilloDeployOperation(len(pkgs), emerge)
op.Run(func, log_level=logging.DEBUG)
else:
func()
logging.warning('Please restart any updated services on the device, '
'or just reboot it.')
except Exception:
if lsb_release:
lsb_entries = sorted(lsb_release.items())
logging.info('Following are the LSB version details of the device:\n%s',
'\n'.join('%s=%s' % (k, v) for k, v in lsb_entries))
raise
def UpdateRootfs(self):
"""Update the rootfs partition of the device."""
devserver_bin = os.path.join(self.device_dev_dir,
self.DEVSERVER_FILENAME)
ds = ds_wrapper.RemoteDevServerWrapper(
self.device,
devserver_bin,
static_dir=self.device_static_dir,
log_dir=self.device.work_dir)
logging.info('Updating rootfs partition')
try:
ds.Start()
# Use the localhost IP address to ensure that update engine
# client can connect to the devserver.
omaha_url = ds.GetDevServerURL(ip='127.0.0.1',
port=ds.port,
sub_dir='update/pregenerated')
cmd = [
self.UPDATE_ENGINE_BIN, '-check_for_update',
'-omaha_url=%s' % omaha_url
]
self.device.RunCommand(cmd)
# If we are using a progress bar, update it every 0.5s instead of 10s.
if command.UseProgressBar():
update_check_interval = self.UPDATE_CHECK_INTERVAL_PROGRESSBAR
oper = operation.ProgressBarOperation()
else:
update_check_interval = self.UPDATE_CHECK_INTERVAL_NORMAL
oper = None
end_message_not_printed = True
# Loop until update is complete.
while True:
op, progress = self.GetUpdateStatus(self.device,
['CURRENT_OP', 'PROGRESS'])
logging.info('Waiting for update...status: %s at progress %s',
op, progress)
if op == UPDATE_STATUS_UPDATED_NEED_REBOOT:
logging.notice('Update completed.')
break
if op == UPDATE_STATUS_IDLE:
raise RootfsUpdateError(
'Update failed with unexpected update status: %s' % op)
if oper is not None:
if op == UPDATE_STATUS_DOWNLOADING:
oper.ProgressBar(float(progress))
elif end_message_not_printed and op == UPDATE_STATUS_FINALIZING:
oper.Cleanup()
logging.notice('Finalizing image.')
end_message_not_printed = False
time.sleep(update_check_interval)
ds.Stop()
except Exception as e:
logging.error('Rootfs update failed.')
logging.warning(ds.TailLog() or 'No devserver log is available.')
error_msg = 'Failed to perform rootfs update: %r'
raise RootfsUpdateError(error_msg % e)
finally:
ds.Stop()
self.device.CopyFromDevice(ds.log_file,
os.path.join(self.tempdir,
self.DEVSERVER_LOG),
error_code_ok=True)
self.device.CopyFromDevice(self.UPDATE_ENGINE_LOG,
self.tempdir,
follow_symlinks=True,
error_code_ok=True)
def Deploy(device,
packages,
board=None,
emerge=True,
update=False,
deep=False,
deep_rev=False,
clean_binpkg=True,
root='/',
strip=True,
emerge_args=None,
ssh_private_key=None,
ping=True,
force=False,
dry_run=False):
"""Deploys packages to a device.
Args:
device: commandline.Device object; None to use the default device.
packages: List of packages (strings) to deploy to device.
board: Board to use; None to automatically detect.
emerge: True to emerge package, False to unmerge.
update: Check installed version on device.
deep: Install dependencies also. Implies |update|.
deep_rev: Install reverse dependencies. Implies |deep|.
clean_binpkg: Clean outdated binary packages.
root: Package installation root path.
strip: Run strip_package to filter out preset paths in the package.
emerge_args: Extra arguments to pass to emerge.
ssh_private_key: Path to an SSH private key file; None to use test keys.
ping: True to ping the device before trying to connect.
force: Ignore sanity checks and prompts.
dry_run: Print deployment plan but do not deploy anything.
Raises:
ValueError: Invalid parameter or parameter combination.
DeployError: Unrecoverable failure during deploy.
"""
if deep_rev:
deep = True
if deep:
update = True
if not packages:
raise DeployError('No packages provided, nothing to deploy.')
if update and not emerge:
raise ValueError('Cannot update and unmerge.')
if device:
hostname, username, port = device.hostname, device.username, device.port
else:
hostname, username, port = None, None, None
lsb_release = None
sysroot = None
try:
# Somewhat confusing to clobber, but here we are.
# pylint: disable=redefined-argument-from-local
with remote_access.ChromiumOSDeviceHandler(hostname,
port=port,
username=username,
private_key=ssh_private_key,
base_dir=_DEVICE_BASE_DIR,
ping=ping) as device:
lsb_release = device.lsb_release
board = cros_build_lib.GetBoard(device_board=device.board,
override_board=board)
if not force and board != device.board:
raise DeployError(
'Device (%s) is incompatible with board %s. Use '
'--force to deploy anyway.' % (device.board, board))
sysroot = cros_build_lib.GetSysroot(board=board)
if clean_binpkg:
logging.notice('Cleaning outdated binary packages from %s',
sysroot)
portage_util.CleanOutdatedBinaryPackages(sysroot)
# Remount rootfs as writable if necessary.
if not device.MountRootfsReadWrite():
raise DeployError(
'Cannot remount rootfs as read-write. Exiting.')
# Obtain list of packages to upgrade/remove.
pkg_scanner = _InstallPackageScanner(sysroot)
pkgs, listed, num_updates, pkgs_attrs = pkg_scanner.Run(
device, root, packages, update, deep, deep_rev)
if emerge:
action_str = 'emerge'
else:
pkgs.reverse()
action_str = 'unmerge'
if not pkgs:
logging.notice('No packages to %s', action_str)
return
# Warn when the user seems to forget `cros workon start`.
worked_on_cps = workon_helper.WorkonHelper(sysroot).ListAtoms()
for package in listed:
cp = package_info.SplitCPV(package).cp
if cp not in worked_on_cps:
logging.warning(
'Are you intentionally deploying unmodified packages, or did '
'you forget to run `cros workon --board=$BOARD start %s`?',
cp)
logging.notice('These are the packages to %s:', action_str)
for i, pkg in enumerate(pkgs):
logging.notice('%s %d) %s', '*' if pkg in listed else ' ',
i + 1, pkg)
if dry_run or not _ConfirmDeploy(num_updates):
return
# Select function (emerge or unmerge) and bind args.
if emerge:
func = functools.partial(_EmergePackages, pkgs, device, strip,
sysroot, root, board, emerge_args)
else:
func = functools.partial(_UnmergePackages, pkgs, device, root,
pkgs_attrs)
# Call the function with the progress bar or with normal output.
if command.UseProgressBar():
op = BrilloDeployOperation(len(pkgs), emerge)
op.Run(func, log_level=logging.DEBUG)
else:
func()
if device.IsSELinuxAvailable():
if sum(x.count('selinux-policy') for x in pkgs):
logging.warning(
'Deploying SELinux policy will not take effect until reboot. '
'SELinux policy is loaded by init. Also, changing the security '
'contexts (labels) of a file will require building a new image '
'and flashing the image onto the device.')
logging.warning(
'Please restart any updated services on the device, '
'or just reboot it.')
except Exception:
if lsb_release:
lsb_entries = sorted(lsb_release.items())
logging.info(
'Following are the LSB version details of the device:\n%s',
'\n'.join('%s=%s' % (k, v) for k, v in lsb_entries))
raise
