def testShouldDetectBoardNotSetUp(self):
"""Check that we complain if a board has not been previously setup."""
with self.assertRaises(workon_helper.WorkonError):
workon_helper.WorkonHelper(os.path.join(self.tempdir,
'nonexistent'),
'this-board-is-not-setup.',
src_root=self._mock_srcdir)
def main(argv):
parser = GetParser()
options = parser.parse_args(argv)
options.Freeze()
if options.command == 'list-all':
board_to_packages = workon_helper.ListAllWorkedOnAtoms()
color = terminal.Color()
for board in sorted(board_to_packages):
print(color.Start(color.GREEN) + board + ':' + color.Stop())
for package in board_to_packages[board]:
print(' ' + package)
print('')
return 0
# TODO(wiley): Assert that we're not running as root.
cros_build_lib.AssertInsideChroot()
if options.host:
friendly_name = 'host'
sysroot = '/'
elif options.board:
friendly_name = options.board
sysroot = cros_build_lib.GetSysroot(board=options.board)
else:
cros_build_lib.Die('You must specify either --host, --board')
helper = workon_helper.WorkonHelper(sysroot, friendly_name)
try:
if options.command == 'start':
helper.StartWorkingOnPackages(options.packages,
use_all=options.all,
use_workon_only=options.workon_only)
elif options.command == 'stop':
helper.StopWorkingOnPackages(options.packages,
use_all=options.all,
use_workon_only=options.workon_only)
elif options.command == 'info':
triples = helper.GetPackageInfo(
options.packages,
use_all=options.all,
use_workon_only=options.workon_only)
for package, repos, paths in triples:
print(package, ','.join(repos), ','.join(paths))
elif options.command == 'list':
packages = helper.ListAtoms(use_all=options.all,
use_workon_only=options.workon_only)
if packages:
print('\n'.join(packages))
elif options.command == 'iterate':
helper.RunCommandInPackages(options.packages,
options.iterate_command,
use_all=options.all,
use_workon_only=options.workon_only)
except workon_helper.WorkonError as e:
cros_build_lib.Die(e)
return 0
def ListWorkonPackages(sysroot, all_opt=False):
"""List the packages that are currently being worked on.
Args:
sysroot: sysroot_lib.Sysroot object.
all_opt: Pass --all to cros_workon. For testing purposes.
"""
helper = workon_helper.WorkonHelper(sysroot.path)
return helper.ListAtoms(use_all=all_opt)
def main(argv):
opts = ParseArgs(argv)
cros_build_lib.AssertInsideChroot()
sysroot = opts.sysroot or cros_build_lib.GetSysroot(opts.board)
package_blacklist = portage_util.UNITTEST_PACKAGE_BLACKLIST
if opts.package_blacklist:
package_blacklist |= set(opts.package_blacklist.split())
packages = set()
# The list of packages to test can be passed as a file containing a
# space-separated list of package names.
# This is used by the builder to test only the packages that were upreved.
if opts.package_file and os.path.exists(opts.package_file):
packages = set(osutils.ReadFile(opts.package_file).split())
if opts.packages:
packages |= set(opts.packages.split())
# If no packages were specified, use all testable packages.
if not (opts.packages or opts.package_file):
workon = workon_helper.WorkonHelper(sysroot)
packages = (workon.InstalledWorkonAtoms()
if opts.installed else workon.ListAtoms(use_all=True))
for cp in packages & package_blacklist:
logging.info('Skipping blacklisted package %s.', cp)
packages = packages - package_blacklist
pkg_with_test = portage_util.PackagesWithTest(sysroot, packages)
if packages - pkg_with_test:
logging.warning('The following packages do not have tests:')
logging.warning('\n'.join(sorted(packages - pkg_with_test)))
if opts.pretend:
print('\n'.join(sorted(pkg_with_test)))
return
env = None
if opts.nowithdebug:
use_flags = os.environ.get('USE', '')
use_flags += ' -cros-debug'
env = {'USE': use_flags}
try:
chroot_util.RunUnittests(sysroot,
pkg_with_test,
extra_env=env,
jobs=min(10, multiprocessing.cpu_count()))
except cros_build_lib.RunCommandError:
logging.error('Unittests failed.')
raise
def Run(self):
"""Run cros build."""
self.options.Freeze()
if not self.host:
if not (self.board or self.brick):
cros_build_lib.Die('You did not specify a board/brick to build for. '
'You need to be in a brick directory or set '
'--board/--brick/--host')
if self.brick and self.brick.legacy:
cros_build_lib.Die('--brick should not be used with board names. Use '
'--board=%s instead.' % self.brick.config['name'])
if self.board:
chroot_args = ['--board', self.board]
else:
chroot_args = None
commandline.RunInsideChroot(self, chroot_args=chroot_args)
if not (self.build_pkgs or self.options.init_only):
cros_build_lib.Die('No packages found, nothing to build.')
# Set up the sysroots if not building for host.
if self.brick or self.board:
chroot_util.SetupBoard(
brick=self.brick, board=self.board,
update_chroot=self.chroot_update,
update_host_packages=self.options.host_packages_update,
use_binary=self.options.binary)
if not self.options.init_only:
# Preliminary: enable all packages that only have a live ebuild.
if self.options.enable_only_latest:
workon = workon_helper.WorkonHelper(self.sysroot)
workon.StartWorkingOnPackages([], use_workon_only=True)
if command.UseProgressBar():
op = BrilloBuildOperation()
op.Run(
parallel.RunParallelSteps, [self._CheckDependencies, self._Build],
log_level=logging.DEBUG)
if self.options.test:
self._Test()
else:
parallel.RunParallelSteps([self._CheckDependencies, self._Build])
if self.options.test:
self._Test()
logging.notice('Build completed successfully.')
def _RefreshWorkonSymlinks(target, sysroot):
"""Force refresh the workon symlinks.
Create an instance of the WorkonHelper, which will recreate all symlinks
to masked/unmasked packages currently worked on in case the sysroot was
recreated (crbug.com/679831).
This was done with a call to `cros_workon list` in the bash version of
the script, but all we actually need is for the WorkonHelper to be
instantiated since it refreshes the symlinks in its __init__.
Args:
target (str): The build target name.
sysroot (sysroot_lib.Sysroot): The board's sysroot.
"""
workon_helper.WorkonHelper(sysroot.path, friendly_name=target)
def CreateHelper(self, host=False):
"""Creates and returns a WorkonHelper object.
Args:
host: If True, create the WorkonHelper for the host.
"""
if host:
overlay = os.path.join(self._overlay_root, HOST_OVERLAY_DIR)
name = 'host'
else:
overlay = os.path.join(self._overlay_root, BOARD_OVERLAY_DIR)
name = BOARD
# Setup the sysroots.
sysroot_lib.Sysroot(self._sysroot).WriteConfig(
'ARCH="amd64"\nPORTDIR_OVERLAY="%s"' % overlay)
# Create helpers for the host or board.
return workon_helper.WorkonHelper(self._sysroot,
name,
src_root=self._mock_srcdir)
def setup_tidy(board: str, ebuild_list: List[portage_util.EBuild]):
"""Sets up to run clang-tidy on the given ebuilds for the given board."""
packages = [x.package for x in ebuild_list]
logging.info('Setting up to lint %r', packages)
workon = workon_helper.WorkonHelper(cros_build_lib.GetSysroot(board))
workon.StopWorkingOnPackages(packages=[], use_all=True)
workon.StartWorkingOnPackages(packages)
# We're going to be hacking with |ebuild| later on, so having all
# dependencies in place is necessary so one |ebuild| won't stomp on another.
cmd = [
f'emerge-{board}',
'--onlydeps',
# Since each `emerge` may eat up to `ncpu` cores, limit the maximum
# concurrency we can get here to (arbitrarily) 8 jobs. Having
# `configure`s and such run in parallel is nice.
f'-j{min(8, multiprocessing.cpu_count())}',
]
cmd += packages
result = cros_build_lib.run(cmd, print_cmd=True, check=False)
if result.returncode:
logging.error('Setup failed with exit code %d; some lints may fail.',
result.returncode)
def main(argv):
shared = commandline.SharedParser()
shared.add_argument('--board',
default=cros_build_lib.GetDefaultBoard(),
help='The board to set package keywords for.')
shared.add_argument('--host',
default=False,
action='store_true',
help='Uses the host instead of board')
shared.add_argument('--remote',
default='',
help='For non-workon projects, the git remote to use.')
shared.add_argument('--revision',
default='',
help='Use to override the manifest defined default '
'revision used for a project')
shared.add_argument('--command',
default='git status',
dest='iterate_command',
help='The command to be run by forall.')
shared.add_argument(
'--workon_only',
default=False,
action='store_true',
help='Apply to packages that have a workon ebuild only')
shared.add_argument('--all',
default=False,
action='store_true',
help='Apply to all possible packages for the '
'given command (overrides workon_only)')
parser = commandline.ArgumentParser(description=__doc__,
parents=[
shared,
])
# Add the shared 'packages' argument after creating the main parser so that
# it is only bound/shared with the subcommands and doesn't confuse argparse.
shared.add_argument('packages',
nargs='*',
help='The packages to run command against.')
commands = [
('start', 'Moves an ebuild to live (intended to support development)'),
('stop', 'Moves an ebuild to stable (use last known good)'),
('info', 'Print package name, repo name, and source directory.'),
('list', 'List of live ebuilds (workon ebuilds if --all)'),
('list-all', 'List all of the live ebuilds for all setup boards'),
('iterate', 'For each ebuild, cd to the source dir and run a command'),
]
command_parsers = parser.add_subparsers(dest='command', title='commands')
for command, description in commands:
command_parsers.add_parser(command,
parents=(shared, ),
help=description,
description=description)
options = parser.parse_args(argv)
options.Freeze()
if options.command == 'list-all':
board_to_packages = workon_helper.ListAllWorkedOnAtoms()
color = terminal.Color()
for board in sorted(board_to_packages):
print(color.Start(color.GREEN) + board + ':' + color.Stop())
for package in board_to_packages[board]:
print(' ' + package)
print('')
return 0
# TODO(wiley): Assert that we're not running as root.
cros_build_lib.AssertInsideChroot()
if options.host:
friendly_name = 'host'
sysroot = '/'
elif options.board:
friendly_name = options.board
sysroot = cros_build_lib.GetSysroot(board=options.board)
else:
cros_build_lib.Die('You must specify either --host, --board')
helper = workon_helper.WorkonHelper(sysroot, friendly_name)
try:
if options.command == 'start':
helper.StartWorkingOnPackages(options.packages,
use_all=options.all,
use_workon_only=options.workon_only)
elif options.command == 'stop':
helper.StopWorkingOnPackages(options.packages,
use_all=options.all,
use_workon_only=options.workon_only)
elif options.command == 'info':
triples = helper.GetPackageInfo(
options.packages,
use_all=options.all,
use_workon_only=options.workon_only)
for package, repos, paths in triples:
print(package, ','.join(repos), ','.join(paths))
elif options.command == 'list':
packages = helper.ListAtoms(use_all=options.all,
use_workon_only=options.workon_only)
if packages:
print('\n'.join(packages))
elif options.command == 'iterate':
helper.RunCommandInPackages(options.packages,
options.iterate_command,
use_all=options.all,
use_workon_only=options.workon_only)
except workon_helper.WorkonError as e:
cros_build_lib.Die(e.message)
return 0
def Run(self):
if self.options.build_target_name:
self.options.build_target = build_target_lib.BuildTarget(
self.options.build_target_name)
else:
self.options.build_target = None
self.options.Freeze()
has_target = self.options.host or self.options.build_target
needs_target = self.options.action != 'list-all'
if needs_target and not has_target:
cros_build_lib.Die(
f'{self.options.action} requires a build target or '
'specifying the host.')
commandline.RunInsideChroot(self)
if self.options.action == 'list-all':
build_target_to_packages = workon_helper.ListAllWorkedOnAtoms()
color = terminal.Color()
for build_target_name in sorted(build_target_to_packages):
print(
color.Start(color.GREEN) + build_target_name + ':' +
color.Stop())
for package in build_target_to_packages[build_target_name]:
print(' ' + package)
print('')
return 0
if self.options.build_target:
target = self.options.build_target.name
sysroot = self.options.build_target.root
else:
target = 'host'
sysroot = '/'
helper = workon_helper.WorkonHelper(sysroot, target)
try:
if self.options.action == 'start':
helper.StartWorkingOnPackages(
self.options.packages,
use_all=self.options.all,
use_workon_only=self.options.workon_only)
elif self.options.action == 'stop':
helper.StopWorkingOnPackages(
self.options.packages,
use_all=self.options.all,
use_workon_only=self.options.workon_only)
elif self.options.action == 'info':
triples = helper.GetPackageInfo(
self.options.packages,
use_all=self.options.all,
use_workon_only=self.options.workon_only)
for package, repos, paths in triples:
print(package, ','.join(repos), ','.join(paths))
elif self.options.action == 'list':
packages = helper.ListAtoms(
use_all=self.options.all,
use_workon_only=self.options.workon_only)
if packages:
print('\n'.join(packages))
elif self.options.action == 'iterate':
helper.RunCommandInPackages(
self.options.packages,
self.options.iterate_command,
use_all=self.options.all,
use_workon_only=self.options.workon_only)
else:
cros_build_lib.Die(
f'No implementation for {self.options.action}')
except workon_helper.WorkonError as e:
cros_build_lib.Die(e)
return 0
def main(argv):
opts = ParseArgs(argv)
cros_build_lib.AssertInsideChroot()
sysroot = opts.sysroot or cros_build_lib.GetSysroot(opts.board)
package_blacklist = set()
if opts.package_blacklist:
package_blacklist |= set(opts.package_blacklist.split())
packages = set()
# The list of packages to test can be passed as a file containing a
# space-separated list of package names.
# This is used by the builder to test only the packages that were upreved.
if opts.package_file and os.path.exists(opts.package_file):
packages = set(osutils.ReadFile(opts.package_file).split())
if opts.packages:
packages |= set(opts.packages.split())
# If no packages were specified, use all testable packages.
if not (opts.packages or opts.package_file) and not opts.empty_sysroot:
workon = workon_helper.WorkonHelper(sysroot)
packages = (workon.InstalledWorkonAtoms() if opts.installed else set(
workon.ListAtoms(use_all=True)))
if opts.empty_sysroot:
packages |= determine_board_packages(sysroot, BOARD_VIRTUAL_PACKAGES)
workon = workon_helper.WorkonHelper(sysroot)
workon_packages = set(workon.ListAtoms(use_all=True))
packages &= workon_packages
for cp in packages & package_blacklist:
logging.info('Skipping blacklisted package %s.', cp)
packages = packages - package_blacklist
pkg_with_test = portage_util.PackagesWithTest(sysroot, packages)
if packages - pkg_with_test:
logging.warning('The following packages do not have tests:\n %s',
'\n '.join(sorted(packages - pkg_with_test)))
if not pkg_with_test:
if opts.testable_packages_optional:
logging.warning('No testable packages found!')
return 0
logging.error('No testable packages found!')
return 1
if opts.pretend:
print('\n'.join(sorted(pkg_with_test)))
return 0
env = None
if opts.nowithdebug:
use_flags = os.environ.get('USE', '')
use_flags += ' -cros-debug'
env = {'USE': use_flags}
if opts.empty_sysroot:
try:
chroot_util.Emerge(list(IMPLICIT_TEST_DEPS),
sysroot,
rebuild_deps=False,
use_binary=False)
chroot_util.Emerge(list(pkg_with_test),
sysroot,
rebuild_deps=False,
use_binary=False)
except cros_build_lib.RunCommandError:
logging.error('Failed building dependencies for unittests.')
return 1
try:
chroot_util.RunUnittests(sysroot,
pkg_with_test,
extra_env=env,
jobs=opts.jobs)
except cros_build_lib.RunCommandError:
logging.error('Unittests failed.')
return 1
def Deploy(device,
packages,
board=None,
emerge=True,
update=False,
deep=False,
deep_rev=False,
clean_binpkg=True,
root='/',
strip=True,
emerge_args=None,
ssh_private_key=None,
ping=True,
force=False,
dry_run=False):
"""Deploys packages to a device.
Args:
device: commandline.Device object; None to use the default device.
packages: List of packages (strings) to deploy to device.
board: Board to use; None to automatically detect.
emerge: True to emerge package, False to unmerge.
update: Check installed version on device.
deep: Install dependencies also. Implies |update|.
deep_rev: Install reverse dependencies. Implies |deep|.
clean_binpkg: Clean outdated binary packages.
root: Package installation root path.
strip: Run strip_package to filter out preset paths in the package.
emerge_args: Extra arguments to pass to emerge.
ssh_private_key: Path to an SSH private key file; None to use test keys.
ping: True to ping the device before trying to connect.
force: Ignore sanity checks and prompts.
dry_run: Print deployment plan but do not deploy anything.
Raises:
ValueError: Invalid parameter or parameter combination.
DeployError: Unrecoverable failure during deploy.
"""
if deep_rev:
deep = True
if deep:
update = True
if not packages:
raise DeployError('No packages provided, nothing to deploy.')
if update and not emerge:
raise ValueError('Cannot update and unmerge.')
if device:
hostname, username, port = device.hostname, device.username, device.port
else:
hostname, username, port = None, None, None
lsb_release = None
sysroot = None
try:
# Somewhat confusing to clobber, but here we are.
# pylint: disable=redefined-argument-from-local
with remote_access.ChromiumOSDeviceHandler(hostname,
port=port,
username=username,
private_key=ssh_private_key,
base_dir=_DEVICE_BASE_DIR,
ping=ping) as device:
lsb_release = device.lsb_release
board = cros_build_lib.GetBoard(device_board=device.board,
override_board=board)
if not force and board != device.board:
raise DeployError(
'Device (%s) is incompatible with board %s. Use '
'--force to deploy anyway.' % (device.board, board))
sysroot = cros_build_lib.GetSysroot(board=board)
if clean_binpkg:
logging.notice('Cleaning outdated binary packages from %s',
sysroot)
portage_util.CleanOutdatedBinaryPackages(sysroot)
# Remount rootfs as writable if necessary.
if not device.MountRootfsReadWrite():
raise DeployError(
'Cannot remount rootfs as read-write. Exiting.')
# Obtain list of packages to upgrade/remove.
pkg_scanner = _InstallPackageScanner(sysroot)
pkgs, listed, num_updates, pkgs_attrs = pkg_scanner.Run(
device, root, packages, update, deep, deep_rev)
if emerge:
action_str = 'emerge'
else:
pkgs.reverse()
action_str = 'unmerge'
if not pkgs:
logging.notice('No packages to %s', action_str)
return
# Warn when the user seems to forget `cros workon start`.
worked_on_cps = workon_helper.WorkonHelper(sysroot).ListAtoms()
for package in listed:
cp = package_info.SplitCPV(package).cp
if cp not in worked_on_cps:
logging.warning(
'Are you intentionally deploying unmodified packages, or did '
'you forget to run `cros workon --board=$BOARD start %s`?',
cp)
logging.notice('These are the packages to %s:', action_str)
for i, pkg in enumerate(pkgs):
logging.notice('%s %d) %s', '*' if pkg in listed else ' ',
i + 1, pkg)
if dry_run or not _ConfirmDeploy(num_updates):
return
# Select function (emerge or unmerge) and bind args.
if emerge:
func = functools.partial(_EmergePackages, pkgs, device, strip,
sysroot, root, board, emerge_args)
else:
func = functools.partial(_UnmergePackages, pkgs, device, root,
pkgs_attrs)
# Call the function with the progress bar or with normal output.
if command.UseProgressBar():
op = BrilloDeployOperation(len(pkgs), emerge)
op.Run(func, log_level=logging.DEBUG)
else:
func()
if device.IsSELinuxAvailable():
if sum(x.count('selinux-policy') for x in pkgs):
logging.warning(
'Deploying SELinux policy will not take effect until reboot. '
'SELinux policy is loaded by init. Also, changing the security '
'contexts (labels) of a file will require building a new image '
'and flashing the image onto the device.')
logging.warning(
'Please restart any updated services on the device, '
'or just reboot it.')
except Exception:
if lsb_release:
lsb_entries = sorted(lsb_release.items())
logging.info(
'Following are the LSB version details of the device:\n%s',
'\n'.join('%s=%s' % (k, v) for k, v in lsb_entries))
raise
def build(build_target, fw_name=None, dry_run=False):
"""Build the AP Firmware.
Args:
build_target (BuildTarget): The build target (board) being built.
fw_name (str|None): Optionally set the FW_NAME envvar to allow building
the firmware for only a specific variant.
dry_run (bool): Whether to perform a dry run.
"""
logging.notice('Building AP Firmware.')
workon = workon_helper.WorkonHelper(build_target.root, build_target.name)
config = _get_build_config(build_target)
# Workon the required packages. Also calculate the list of packages that need
# to be stopped to clean up after ourselves when we're done.
if config.workon:
before_workon = workon.ListAtoms()
logging.info('cros_workon starting packages.')
logging.debug('cros_workon-%s start %s', build_target.name,
' '.join(config.workon))
if dry_run:
# Pretend it worked: after = before U workon.
after_workon = set(before_workon) & set(config.workon)
else:
workon.StartWorkingOnPackages(config.workon)
after_workon = workon.ListAtoms()
# Stop = the set we actually started. Preserves workon started status for
# any in the config.workon packages that were already worked on.
stop_packages = list(set(after_workon) - set(before_workon))
else:
stop_packages = []
extra_env = {'FW_NAME': fw_name} if fw_name else None
# Run the emerge command to build the packages. Don't raise an exception here
# if it fails so we can cros workon stop afterwords.
logging.info('Building the AP firmware packages.')
# Print command with --debug.
print_cmd = logging.getLogger(__name__).getEffectiveLevel() == logging.DEBUG
result = cros_build_lib.run(
[build_target.get_command('emerge')] + list(config.build),
print_cmd=print_cmd,
check=False,
debug_level=logging.DEBUG,
dryrun=dry_run,
extra_env=extra_env)
# Reset the environment.
logging.notice('Restoring cros_workon status.')
if stop_packages:
# Stop the packages we started.
logging.info('Stopping workon packages previously started.')
logging.debug('cros_workon-%s stop %s', build_target.name,
' '.join(stop_packages))
if not dry_run:
workon.StopWorkingOnPackages(stop_packages)
else:
logging.info('No packages needed to be stopped.')
if result.returncode:
# Now raise the emerge failure since we're done cleaning up.
raise BuildError('The emerge command failed. Run with --verbose or --debug '
'to see the emerge output for details.')
logging.notice('Done! AP firmware successfully built.')
