def PerformStage(self):
usepkg = False
if self.AfterLimit(BUILD_PACKAGES_PREBUILTS):
usepkg = self._run.config.usepkg_build_packages
packages = self.GetListOfPackagesToBuild()
cmd = ['./build_packages', '--board=%s' % self._current_board,
'--accept_licenses=@CHROMEOS', '--skip_chroot_upgrade']
if not self._run.options.tests:
cmd.append('--nowithautotest')
if self.AfterLimit(BUILD_PACKAGES_WITH_DEBUG_SYMBOLS):
cmd.append('--withdebugsymbols')
if not usepkg:
cmd.extend(commands.LOCAL_BUILD_FLAGS)
if self._run.config.nobuildretry:
cmd.append('--nobuildretry')
cmd.extend(packages)
commands.RunBuildScript(
self._build_root, cmd,
enter_chroot=True,
chroot_args=ChrootArgs(self._run.options),
extra_env=self._portage_extra_env)
def RunCrosVMTest(buildroot, board, image_dir):
"""Runs cros_vm_test script to verify cros commands work."""
image_path = os.path.join(image_dir, constants.TEST_IMAGE_BIN)
script = os.path.join(buildroot, 'chromite', 'cli', 'cros', 'tests',
'cros_vm_test')
commands.RunBuildScript(
buildroot, [script, '--board', board, '--image_path', image_path])
def PerformStage(self):
new_chroot_dir = 'new-sdk-chroot'
tarball_location = os.path.join(self._build_root, SDK_TARBALL_NAME)
new_chroot_args = ['--chroot', new_chroot_dir]
if self._run.options.chrome_root:
new_chroot_args += ['--chrome_root', self._run.options.chrome_root]
# Build a new SDK using the provided tarball.
chroot_args = new_chroot_args + [
'--download', '--replace', '--nousepkg', '--url',
'file://' + tarball_location
]
cros_build_lib.run(['true'],
cwd=self._build_root,
enter_chroot=True,
chroot_args=chroot_args,
extra_env=self._portage_extra_env)
# Inject the toolchain binpkgs from the previous sdk build. On end user
# systems, they'd be fetched from the binpkg mirror, but we don't have one
# set up for this local build.
pkgdir = os.path.join('var', 'lib', 'portage', 'pkgs')
old_pkgdir = os.path.join(self._build_root,
constants.DEFAULT_CHROOT_DIR, pkgdir)
new_pkgdir = os.path.join(self._build_root, new_chroot_dir, pkgdir)
osutils.SafeMakedirs(new_pkgdir, sudo=True)
cros_build_lib.sudo_run(['cp', '-r'] +
glob.glob(os.path.join(old_pkgdir, '*')) +
[new_pkgdir])
# Now install those toolchains in the new chroot. We skip the chroot
# upgrade below which means we need to install the toolchain manually.
cmd = [
'cros_setup_toolchains', '--targets=boards',
'--include-boards=%s' % ','.join(self._boards)
]
commands.RunBuildScript(self._build_root,
cmd,
chromite_cmd=True,
enter_chroot=True,
sudo=True,
chroot_args=new_chroot_args,
extra_env=self._portage_extra_env)
# Build all the boards with the new sdk.
for board in self._boards:
logging.PrintBuildbotStepText(board)
commands.SetupBoard(self._build_root,
board,
usepkg=True,
chroot_upgrade=False,
extra_env=self._portage_extra_env,
chroot_args=new_chroot_args)
commands.Build(self._build_root,
board,
build_autotest=True,
usepkg=False,
extra_env=self._portage_extra_env,
chroot_args=new_chroot_args,
disable_revdep_logic=True)
def CrosSetupToolchains(self, cmd_args, **kwargs):
"""Wrapper around cros_setup_toolchains to simplify things."""
commands.RunBuildScript(
self._build_root, ['cros_setup_toolchains'] + list(cmd_args),
chromite_cmd=True,
enter_chroot=True,
**kwargs)
def PerformStage(self):
"""Run untitests for infra venv projects."""
commands.RunBuildScript(
self._build_root,
['./bin/test_venv_packages'],
cwd=os.path.join(constants.SOURCE_ROOT, 'chromeos-admin'),
)
commands.RunBuildScript(
self._build_root,
['./bin/test_venv_packages'],
cwd=os.path.join(constants.SOURCE_ROOT, 'infra', 'skylab_inventory'),
)
commands.RunBuildScript(
self._build_root,
['./bin/run_tests'],
cwd=os.path.join(constants.SOURCE_ROOT, 'infra', 'ci_results_archiver'),
)
def PerformStage(self):
"""Build infra Go packages."""
commands.RunBuildScript(
self._build_root,
['bash', '-c',
'cd ../../chromeos-admin/puppet'
' && make -j -O check GEM=gem19'],
enter_chroot=True)
def PerformStage(self):
manager = self._run.attrs.manifest_manager
cmd = ['chrome_chromeos_lkgm', '--lkgm=%s' % manager.current_version]
# Always do a dryrun for now so that we can check the output and ensure it
# is doing the correct thing.
if self._run.options.debug:
cmd.append('--dryrun')
commands.RunBuildScript(self._build_root, cmd, chromite_cmd=True)
def PerformStage(self):
chroot_dir = os.path.join(self._build_root, constants.DEFAULT_CHROOT_DIR)
sdk_dir = os.path.join(chroot_dir, 'build/amd64-host')
tmp_dir = os.path.join(chroot_dir, 'tmp')
osutils.SafeMakedirs(tmp_dir, mode=0o777, sudo=True)
overlay_output_dir = os.path.join(chroot_dir,
constants.SDK_OVERLAYS_OUTPUT)
osutils.RmDir(overlay_output_dir, ignore_missing=True, sudo=True)
osutils.SafeMakedirs(overlay_output_dir, mode=0o777, sudo=True)
overlay_tarball_template = os.path.join(
overlay_output_dir, TOOLCHAINS_OVERLAY_TARBALL_TEMPLATE)
# Generate an overlay tarball for each unique toolchain combination. We
# restrict ourselves to (a) board configs that are available to the builder
# (naturally), and (b) toolchains that are part of the 'sdk' set.
sdk_toolchains = set(toolchain.GetToolchainsForBoard('sdk'))
generated = set()
for board in self._run.site_config.GetBoards():
try:
toolchains = set(toolchain.GetToolchainsForBoard(board).iterkeys())
except portage_util.MissingOverlayException:
# The board overlay may not exist, e.g. on external builders.
continue
toolchains_str = '-'.join(sorted(toolchains))
if not toolchains.issubset(sdk_toolchains) or toolchains_str in generated:
continue
with osutils.TempDir(prefix='toolchains-overlay-%s.' % toolchains_str,
base_dir=tmp_dir, sudo_rm=True) as overlay_dir:
# NOTE: We let MountOverlayContext remove the mount point created by
# the TempDir context below, because it has built-in retries for rmdir
# EBUSY errors that are due to unmount lag.
with osutils.TempDir(prefix='amd64-host-%s.' % toolchains_str,
base_dir=tmp_dir, delete=False) as merged_dir:
with osutils.MountOverlayContext(sdk_dir, overlay_dir, merged_dir,
cleanup=True):
sysroot = merged_dir[len(chroot_dir):]
cmd = ['cros_setup_toolchains', '--targets=boards',
'--include-boards=%s' % board,
'--sysroot=%s' % sysroot]
commands.RunBuildScript(self._build_root, cmd, chromite_cmd=True,
enter_chroot=True, sudo=True,
extra_env=self._portage_extra_env)
# NOTE: Make sure that the overlay directory is owned root:root and has
# 0o755 perms; apparently, these things are preserved through
# tarring/untarring and might cause havoc if overlooked.
os.chmod(overlay_dir, 0o755)
cros_build_lib.SudoRunCommand(['chown', 'root:root', overlay_dir])
CreateTarball(overlay_dir,
overlay_tarball_template % {'toolchains': toolchains_str})
generated.add(toolchains_str)
def PerformStage(self):
chroot_path = os.path.join(self._build_root,
constants.DEFAULT_CHROOT_DIR)
# Worksapce chroots are always wiped by cleanup stage, no need to update.
cmd = ['cros_sdk', '--create'] + ChrootArgs(self._run.options)
commands.RunBuildScript(self._build_root, cmd, chromite_cmd=True,
extra_env=self._portage_extra_env)
post_ver = cros_sdk_lib.GetChrootVersion(chroot_path)
logging.PrintBuildbotStepText(post_ver)
def PerformStage(self):
if not self._build_threshold_successful():
logging.info('Insufficient number of successful builders. '
'Skipping LKGM update.')
manager = self._run.attrs.manifest_manager
cmd = ['chrome_chromeos_lkgm', '--lkgm=%s' % manager.current_version]
# Always do a dryrun for now so that we can check the output and ensure it
# is doing the correct thing.
if self._run.options.debug:
cmd.append('--dryrun')
commands.RunBuildScript(self._build_root, cmd, chromite_cmd=True)
def _UploadPrebuilts(buildroot, board, extra_args):
"""Upload prebuilts.
Args:
buildroot: The root directory where the build occurs.
board: Board type that was built on this machine.
extra_args: Extra args to pass to prebuilts script.
"""
cmd = ['upload_prebuilts', '--build-path', buildroot]
if board:
cmd.extend(['--board', board])
cmd.extend(extra_args)
commands.RunBuildScript(buildroot, cmd, chromite_cmd=True)
def _GetPortagePackageFiles(self, package):
"""Gets paths of files owned by an installed Portage package.
Args:
package: Portage package name without category.
Returns:
A list of paths of files owned by the package.
"""
cmd = ['equery', '--no-color', '--quiet', 'f', '--filter=obj,cmd', package]
result = commands.RunBuildScript(self._build_root, cmd, enter_chroot=True,
stdout=True)
return result.output.splitlines()
def PerformStage(self):
# Collect build_image arguments.
version = self.GetWorkspaceReleaseTag()
rootfs_verification = self._run.config.rootfs_verification
disk_layout = self._run.config.disk_layout
builder_path = '/'.join([self._bot_id, version])
# We only build base, dev, and test images from this stage.
images_can_build = set(['base', 'dev', 'test'])
images_to_build = set(
self._run.config.images).intersection(images_can_build)
assert images_to_build
# Build up command line.
cmd = [
'./build_image', '--board', self._current_board, '--replace',
'--version', version
]
if self.AfterLimit(BUILD_IMAGE_ECLEAN_FLAG):
cmd += ['--noeclean']
if not rootfs_verification:
cmd += ['--noenable_rootfs_verification']
if disk_layout:
cmd += ['--disk_layout', disk_layout]
if self.AfterLimit(BUILD_IMAGE_BUILDER_PATH):
cmd += ['--builder_path', builder_path]
cmd += sorted(images_to_build)
# Run command.
commands.RunBuildScript(self._build_root,
cmd,
enter_chroot=True,
extra_env=self._portage_extra_env,
chroot_args=ChrootArgs(self._run.options))
def _assertRunBuildScript(self, in_chroot=False, error=None, raises=None,
**kwargs):
"""Test the RunBuildScript function.
Args:
in_chroot: Whether to enter the chroot or not.
error: error result message to simulate.
raises: If the command should fail, the exception to be raised.
kwargs: Extra kwargs passed to RunBuildScript.
"""
# Write specified error message to status file.
def WriteError(_cmd, extra_env=None, **_kwargs):
if extra_env is not None and error is not None:
status_file = extra_env[constants.PARALLEL_EMERGE_STATUS_FILE_ENVVAR]
osutils.WriteFile(status_file, error)
buildroot = self.tempdir
osutils.SafeMakedirs(os.path.join(buildroot, '.repo'))
if error is not None:
osutils.SafeMakedirs(os.path.join(buildroot, 'chroot', 'tmp'))
# Run the command, throwing an exception if it fails.
with cros_build_lib_unittest.RunCommandMock() as m:
cmd = ['example', 'command']
sudo_cmd = ['sudo', '--'] + cmd
returncode = 1 if raises else 0
m.AddCmdResult(cmd, returncode=returncode, side_effect=WriteError)
m.AddCmdResult(sudo_cmd, returncode=returncode, side_effect=WriteError)
with mock.patch.object(path_util, 'ToChrootPath',
side_effect=lambda x: x):
with cros_test_lib.LoggingCapturer():
# If the script failed, the exception should be raised and printed.
if raises:
self.assertRaises(raises, commands.RunBuildScript, buildroot,
cmd, enter_chroot=in_chroot, **kwargs)
else:
commands.RunBuildScript(buildroot, cmd, enter_chroot=in_chroot,
**kwargs)
def _EmergePackages(self):
cmd = ['emerge', '--deep']
cmd.extend(_GO_PACKAGES)
commands.RunBuildScript(self._build_root, cmd,
sudo=True, enter_chroot=True,
extra_env={'FEATURES': 'test'})
