def GetImagePathWithXbuddy(path, board, version=None, static_dir=DEFAULT_STATIC_DIR, lookup_only=False, silent=False): """Gets image path and resolved XBuddy path using xbuddy. Ask xbuddy to translate |path|, and if necessary, download and stage the image, then return a translated path to the image. Also returns the resolved XBuddy path, which may be useful for subsequent calls in case the argument is an alias. Args: path: The xbuddy path. board: The default board to use if board is not specified in |path|. version: The default version to use if one is not specified in |path|. static_dir: Static directory to stage the image in. lookup_only: Caller only wants to translate the path not download the artifact. silent: Suppress error messages. Returns: A tuple consisting of a translated path to the image (build-id/version/image_name) as well as the fully resolved XBuddy path (in the case where |path| is an XBuddy alias). """ # Since xbuddy often wants to use gsutil from $PATH, make sure our local copy # shows up first. upath = os.environ['PATH'].split(os.pathsep) upath.insert(0, os.path.dirname(gs.GSContext.GetDefaultGSUtilBin())) os.environ['PATH'] = os.pathsep.join(upath) # If we are using the progress bar, quiet the logging output of cherrypy. if cherrypy and command.UseProgressBar(): if (hasattr(cherrypy.log, 'access_log') and hasattr(cherrypy.log, 'error_log')): cherrypy.log.access_log.setLevel(logging.NOTICE) cherrypy.log.error_log.setLevel(logging.NOTICE) else: cherrypy.config.update({'server.log_to_screen': False}) xb = xbuddy.XBuddy(static_dir=static_dir, board=board, version=version, log_screen=False) path_list = GetXbuddyPath(path).rsplit(os.path.sep) try: if lookup_only: build_id, file_name = xb.Translate(path_list) else: build_id, file_name = xb.Get(path_list) resolved_path, _ = xb.LookupAlias(os.path.sep.join(path_list)) return os.path.join(build_id, file_name), resolved_path except xbuddy.XBuddyException as e: if not silent: logging.error('Locating image "%s" failed. The path might not be valid ' 'or the image might not exist.', path) raise ImagePathError('Cannot locate image %s: %s' % (path, e)) except build_artifact.ArtifactDownloadError as e: if not silent: logging.error('Downloading image "%s" failed.', path) raise ArtifactDownloadError('Cannot download image %s: %s' % (path, e))
def Run(self): """Run cros build.""" self.options.Freeze() if not self.host: if not (self.board or self.brick): cros_build_lib.Die('You did not specify a board/brick to build for. ' 'You need to be in a brick directory or set ' '--board/--brick/--host') if self.brick and self.brick.legacy: cros_build_lib.Die('--brick should not be used with board names. Use ' '--board=%s instead.' % self.brick.config['name']) if self.board: chroot_args = ['--board', self.board] else: chroot_args = None commandline.RunInsideChroot(self, chroot_args=chroot_args) if not (self.build_pkgs or self.options.init_only): cros_build_lib.Die('No packages found, nothing to build.') # Set up the sysroots if not building for host. if self.brick or self.board: chroot_util.SetupBoard( brick=self.brick, board=self.board, update_chroot=self.chroot_update, update_host_packages=self.options.host_packages_update, use_binary=self.options.binary) if not self.options.init_only: # Preliminary: enable all packages that only have a live ebuild. if self.options.enable_only_latest: workon = workon_helper.WorkonHelper(self.sysroot) workon.StartWorkingOnPackages([], use_workon_only=True) if command.UseProgressBar(): op = BrilloBuildOperation() op.Run( parallel.RunParallelSteps, [self._CheckDependencies, self._Build], log_level=logging.DEBUG) if self.options.test: self._Test() else: parallel.RunParallelSteps([self._CheckDependencies, self._Build]) if self.options.test: self._Test() logging.notice('Build completed successfully.')
def UpdateRootfs(self): """Update the rootfs partition of the device (utilizing nebraska).""" logging.notice('Updating rootfs partition...') nebraska_bin = os.path.join(self.device_dev_dir, self.REMOTE_NEBRASKA_FILENAME) nebraska = nebraska_wrapper.RemoteNebraskaWrapper( self.device, nebraska_bin=nebraska_bin, update_payloads_address='file://' + self.device_payload_dir, update_metadata_dir=self.device_payload_dir) try: nebraska.Start() # Use the localhost IP address (default) to ensure that update engine # client can connect to the nebraska. nebraska_url = nebraska.GetURL(critical_update=True) cmd = [ self.REMOTE_UPDATE_ENGINE_BIN_FILENAME, '--check_for_update', '--omaha_url="%s"' % nebraska_url ] self.device.run(cmd, **self._cmd_kwargs) # If we are using a progress bar, update it every 0.5s instead of 10s. if command.UseProgressBar(): update_check_interval = self.UPDATE_CHECK_INTERVAL_PROGRESSBAR oper = operation.ProgressBarOperation() else: update_check_interval = self.UPDATE_CHECK_INTERVAL_NORMAL oper = None end_message_not_printed = True # Loop until update is complete. while True: # Number of times to retry `update_engine_client --status`. See # crbug.com/744212. update_engine_status_retry = 30 op, progress = retry_util.RetryException( cros_build_lib.RunCommandError, update_engine_status_retry, self.GetUpdateStatus, self.device, ['CURRENT_OP', 'PROGRESS'], delay_sec=DELAY_SEC_FOR_RETRY)[0:2] logging.info('Waiting for update...status: %s at progress %s', op, progress) if op == UPDATE_STATUS_UPDATED_NEED_REBOOT: logging.info('Update completed.') break if op == UPDATE_STATUS_IDLE: # Something went wrong. Try to get last error code. cmd = ['cat', self.REMOTE_UPDATE_ENGINE_LOGFILE_PATH] log = self.device.run(cmd).stdout.strip().splitlines() err_str = 'Updating payload state for error code: ' targets = [line for line in log if err_str in line] logging.debug('Error lines found: %s', targets) if not targets: raise RootfsUpdateError( 'Update failed with unexpected update status: %s' % op) else: # e.g 20 (ErrorCode::kDownloadStateInitializationError) raise RootfsUpdateError( targets[-1].rpartition(err_str)[2]) if oper is not None: if op == UPDATE_STATUS_DOWNLOADING: oper.ProgressBar(float(progress)) elif end_message_not_printed and op == UPDATE_STATUS_FINALIZING: oper.Cleanup() logging.info('Finalizing image.') end_message_not_printed = False time.sleep(update_check_interval) # TODO(ahassani): Scope the Exception to finer levels. For example we don't # need to revert the boot partition if the Nebraska fails to start, etc. except Exception as e: logging.error('Rootfs update failed %s', e) self.RevertBootPartition() logging.warning(nebraska.PrintLog() or 'No nebraska log is available.') raise RootfsUpdateError('Failed to perform rootfs update: %r' % e) finally: nebraska.Stop() nebraska.CollectLogs( os.path.join(self.tempdir, self.LOCAL_NEBRASKA_LOG_FILENAME)) self.device.CopyFromDevice( self.REMOTE_UPDATE_ENGINE_LOGFILE_PATH, os.path.join( self.tempdir, os.path.basename(self.REMOTE_UPDATE_ENGINE_LOGFILE_PATH)), follow_symlinks=True, **self._cmd_kwargs_omit_error)
def Deploy(device, packages, board=None, emerge=True, update=False, deep=False, deep_rev=False, clean_binpkg=True, root='/', strip=True, emerge_args=None, ssh_private_key=None, ping=True, force=False, dry_run=False): """Deploys packages to a device. Args: device: commandline.Device object; None to use the default device. packages: List of packages (strings) to deploy to device. board: Board to use; None to automatically detect. emerge: True to emerge package, False to unmerge. update: Check installed version on device. deep: Install dependencies also. Implies |update|. deep_rev: Install reverse dependencies. Implies |deep|. clean_binpkg: Clean outdated binary packages. root: Package installation root path. strip: Run strip_package to filter out preset paths in the package. emerge_args: Extra arguments to pass to emerge. ssh_private_key: Path to an SSH private key file; None to use test keys. ping: True to ping the device before trying to connect. force: Ignore sanity checks and prompts. dry_run: Print deployment plan but do not deploy anything. Raises: ValueError: Invalid parameter or parameter combination. DeployError: Unrecoverable failure during deploy. """ if deep_rev: deep = True if deep: update = True if not packages: raise DeployError('No packages provided, nothing to deploy.') if update and not emerge: raise ValueError('Cannot update and unmerge.') if device: hostname, username, port = device.hostname, device.username, device.port else: hostname, username, port = None, None, None lsb_release = None sysroot = None try: with remote_access.ChromiumOSDeviceHandler( hostname, port=port, username=username, private_key=ssh_private_key, base_dir=_DEVICE_BASE_DIR, ping=ping) as device: lsb_release = device.lsb_release board = cros_build_lib.GetBoard(device_board=device.board, override_board=board) if not force and board != device.board: raise DeployError('Device (%s) is incompatible with board %s. Use ' '--force to deploy anyway.' % (device.board, board)) sysroot = cros_build_lib.GetSysroot(board=board) if clean_binpkg: logging.notice('Cleaning outdated binary packages from %s', sysroot) portage_util.CleanOutdatedBinaryPackages(sysroot) if not device.IsDirWritable(root): # Only remounts rootfs if the given root is not writable. if not device.MountRootfsReadWrite(): raise DeployError('Cannot remount rootfs as read-write. Exiting.') # Obtain list of packages to upgrade/remove. pkg_scanner = _InstallPackageScanner(sysroot) pkgs, listed, num_updates = pkg_scanner.Run( device, root, packages, update, deep, deep_rev) if emerge: action_str = 'emerge' else: pkgs.reverse() action_str = 'unmerge' if not pkgs: logging.notice('No packages to %s', action_str) return logging.notice('These are the packages to %s:', action_str) for i, pkg in enumerate(pkgs): logging.notice('%s %d) %s', '*' if pkg in listed else ' ', i + 1, pkg) if dry_run or not _ConfirmDeploy(num_updates): return # Select function (emerge or unmerge) and bind args. if emerge: func = functools.partial(_EmergePackages, pkgs, device, strip, sysroot, root, emerge_args) else: func = functools.partial(_UnmergePackages, pkgs, device, root) # Call the function with the progress bar or with normal output. if command.UseProgressBar(): op = BrilloDeployOperation(len(pkgs), emerge) op.Run(func, log_level=logging.DEBUG) else: func() logging.warning('Please restart any updated services on the device, ' 'or just reboot it.') except Exception: if lsb_release: lsb_entries = sorted(lsb_release.items()) logging.info('Following are the LSB version details of the device:\n%s', '\n'.join('%s=%s' % (k, v) for k, v in lsb_entries)) raise
def UpdateRootfs(self): """Update the rootfs partition of the device.""" devserver_bin = os.path.join(self.device_dev_dir, self.DEVSERVER_FILENAME) ds = ds_wrapper.RemoteDevServerWrapper( self.device, devserver_bin, static_dir=self.device_static_dir, log_dir=self.device.work_dir) logging.info('Updating rootfs partition') try: ds.Start() # Use the localhost IP address to ensure that update engine # client can connect to the devserver. omaha_url = ds.GetDevServerURL(ip='127.0.0.1', port=ds.port, sub_dir='update/pregenerated') cmd = [ self.UPDATE_ENGINE_BIN, '-check_for_update', '-omaha_url=%s' % omaha_url ] self.device.RunCommand(cmd) # If we are using a progress bar, update it every 0.5s instead of 10s. if command.UseProgressBar(): update_check_interval = self.UPDATE_CHECK_INTERVAL_PROGRESSBAR oper = operation.ProgressBarOperation() else: update_check_interval = self.UPDATE_CHECK_INTERVAL_NORMAL oper = None end_message_not_printed = True # Loop until update is complete. while True: op, progress = self.GetUpdateStatus(self.device, ['CURRENT_OP', 'PROGRESS']) logging.info('Waiting for update...status: %s at progress %s', op, progress) if op == UPDATE_STATUS_UPDATED_NEED_REBOOT: logging.notice('Update completed.') break if op == UPDATE_STATUS_IDLE: raise RootfsUpdateError( 'Update failed with unexpected update status: %s' % op) if oper is not None: if op == UPDATE_STATUS_DOWNLOADING: oper.ProgressBar(float(progress)) elif end_message_not_printed and op == UPDATE_STATUS_FINALIZING: oper.Cleanup() logging.notice('Finalizing image.') end_message_not_printed = False time.sleep(update_check_interval) ds.Stop() except Exception as e: logging.error('Rootfs update failed.') logging.warning(ds.TailLog() or 'No devserver log is available.') error_msg = 'Failed to perform rootfs update: %r' raise RootfsUpdateError(error_msg % e) finally: ds.Stop() self.device.CopyFromDevice(ds.log_file, os.path.join(self.tempdir, self.DEVSERVER_LOG), error_code_ok=True) self.device.CopyFromDevice(self.UPDATE_ENGINE_LOG, self.tempdir, follow_symlinks=True, error_code_ok=True)
def Deploy(device, packages, board=None, emerge=True, update=False, deep=False, deep_rev=False, clean_binpkg=True, root='/', strip=True, emerge_args=None, ssh_private_key=None, ping=True, force=False, dry_run=False): """Deploys packages to a device. Args: device: commandline.Device object; None to use the default device. packages: List of packages (strings) to deploy to device. board: Board to use; None to automatically detect. emerge: True to emerge package, False to unmerge. update: Check installed version on device. deep: Install dependencies also. Implies |update|. deep_rev: Install reverse dependencies. Implies |deep|. clean_binpkg: Clean outdated binary packages. root: Package installation root path. strip: Run strip_package to filter out preset paths in the package. emerge_args: Extra arguments to pass to emerge. ssh_private_key: Path to an SSH private key file; None to use test keys. ping: True to ping the device before trying to connect. force: Ignore sanity checks and prompts. dry_run: Print deployment plan but do not deploy anything. Raises: ValueError: Invalid parameter or parameter combination. DeployError: Unrecoverable failure during deploy. """ if deep_rev: deep = True if deep: update = True if not packages: raise DeployError('No packages provided, nothing to deploy.') if update and not emerge: raise ValueError('Cannot update and unmerge.') if device: hostname, username, port = device.hostname, device.username, device.port else: hostname, username, port = None, None, None lsb_release = None sysroot = None try: # Somewhat confusing to clobber, but here we are. # pylint: disable=redefined-argument-from-local with remote_access.ChromiumOSDeviceHandler(hostname, port=port, username=username, private_key=ssh_private_key, base_dir=_DEVICE_BASE_DIR, ping=ping) as device: lsb_release = device.lsb_release board = cros_build_lib.GetBoard(device_board=device.board, override_board=board) if not force and board != device.board: raise DeployError( 'Device (%s) is incompatible with board %s. Use ' '--force to deploy anyway.' % (device.board, board)) sysroot = cros_build_lib.GetSysroot(board=board) if clean_binpkg: logging.notice('Cleaning outdated binary packages from %s', sysroot) portage_util.CleanOutdatedBinaryPackages(sysroot) # Remount rootfs as writable if necessary. if not device.MountRootfsReadWrite(): raise DeployError( 'Cannot remount rootfs as read-write. Exiting.') # Obtain list of packages to upgrade/remove. pkg_scanner = _InstallPackageScanner(sysroot) pkgs, listed, num_updates, pkgs_attrs = pkg_scanner.Run( device, root, packages, update, deep, deep_rev) if emerge: action_str = 'emerge' else: pkgs.reverse() action_str = 'unmerge' if not pkgs: logging.notice('No packages to %s', action_str) return # Warn when the user seems to forget `cros workon start`. worked_on_cps = workon_helper.WorkonHelper(sysroot).ListAtoms() for package in listed: cp = package_info.SplitCPV(package).cp if cp not in worked_on_cps: logging.warning( 'Are you intentionally deploying unmodified packages, or did ' 'you forget to run `cros workon --board=$BOARD start %s`?', cp) logging.notice('These are the packages to %s:', action_str) for i, pkg in enumerate(pkgs): logging.notice('%s %d) %s', '*' if pkg in listed else ' ', i + 1, pkg) if dry_run or not _ConfirmDeploy(num_updates): return # Select function (emerge or unmerge) and bind args. if emerge: func = functools.partial(_EmergePackages, pkgs, device, strip, sysroot, root, board, emerge_args) else: func = functools.partial(_UnmergePackages, pkgs, device, root, pkgs_attrs) # Call the function with the progress bar or with normal output. if command.UseProgressBar(): op = BrilloDeployOperation(len(pkgs), emerge) op.Run(func, log_level=logging.DEBUG) else: func() if device.IsSELinuxAvailable(): if sum(x.count('selinux-policy') for x in pkgs): logging.warning( 'Deploying SELinux policy will not take effect until reboot. ' 'SELinux policy is loaded by init. Also, changing the security ' 'contexts (labels) of a file will require building a new image ' 'and flashing the image onto the device.') logging.warning( 'Please restart any updated services on the device, ' 'or just reboot it.') except Exception: if lsb_release: lsb_entries = sorted(lsb_release.items()) logging.info( 'Following are the LSB version details of the device:\n%s', '\n'.join('%s=%s' % (k, v) for k, v in lsb_entries)) raise