def PerformStage(self):
chroot_dir = os.path.join(self._build_root, constants.DEFAULT_CHROOT_DIR)
sdk_dir = os.path.join(chroot_dir, 'build/amd64-host')
tmp_dir = os.path.join(chroot_dir, 'tmp')
osutils.SafeMakedirs(tmp_dir, mode=0o777, sudo=True)
overlay_output_dir = os.path.join(chroot_dir,
constants.SDK_OVERLAYS_OUTPUT)
osutils.RmDir(overlay_output_dir, ignore_missing=True, sudo=True)
osutils.SafeMakedirs(overlay_output_dir, mode=0o777, sudo=True)
overlay_tarball_template = os.path.join(
overlay_output_dir, TOOLCHAINS_OVERLAY_TARBALL_TEMPLATE)
# Generate an overlay tarball for each unique toolchain combination. We
# restrict ourselves to (a) board configs that are available to the builder
# (naturally), and (b) toolchains that are part of the 'sdk' set.
sdk_toolchains = set(toolchain.GetToolchainsForBoard('sdk'))
generated = set()
for board in self._run.site_config.GetBoards():
try:
toolchains = set(toolchain.GetToolchainsForBoard(board).iterkeys())
except portage_util.MissingOverlayException:
# The board overlay may not exist, e.g. on external builders.
continue
toolchains_str = '-'.join(sorted(toolchains))
if not toolchains.issubset(sdk_toolchains) or toolchains_str in generated:
continue
with osutils.TempDir(prefix='toolchains-overlay-%s.' % toolchains_str,
base_dir=tmp_dir, sudo_rm=True) as overlay_dir:
# NOTE: We let MountOverlayContext remove the mount point created by
# the TempDir context below, because it has built-in retries for rmdir
# EBUSY errors that are due to unmount lag.
with osutils.TempDir(prefix='amd64-host-%s.' % toolchains_str,
base_dir=tmp_dir, delete=False) as merged_dir:
with osutils.MountOverlayContext(sdk_dir, overlay_dir, merged_dir,
cleanup=True):
sysroot = merged_dir[len(chroot_dir):]
cmd = ['cros_setup_toolchains', '--targets=boards',
'--include-boards=%s' % board,
'--sysroot=%s' % sysroot]
commands.RunBuildScript(self._build_root, cmd, chromite_cmd=True,
enter_chroot=True, sudo=True,
extra_env=self._portage_extra_env)
# NOTE: Make sure that the overlay directory is owned root:root and has
# 0o755 perms; apparently, these things are preserved through
# tarring/untarring and might cause havoc if overlooked.
os.chmod(overlay_dir, 0o755)
cros_build_lib.SudoRunCommand(['chown', 'root:root', overlay_dir])
CreateTarball(overlay_dir,
overlay_tarball_template % {'toolchains': toolchains_str})
generated.add(toolchains_str)
def GetCrossGdb(self):
"""Find the appropriate cross-version of gdb for the board."""
toolchains = toolchain.GetToolchainsForBoard(self.board)
tc = toolchain.FilterToolchains(toolchains, 'default', True).keys()
cross_gdb = tc[0] + '-gdb'
if not osutils.Which(cross_gdb):
raise GdbMissingDebuggerError('Cannot find %s; do you need to run '
'setup_board?' % cross_gdb)
return cross_gdb
def UpdateToolchains(usepkg, deleteold, hostonly, reconfig,
targets_wanted, boards_wanted, root='/'):
"""Performs all steps to create a synchronized toolchain enviroment.
Args:
usepkg: Use prebuilt packages
deleteold: Unmerge deprecated packages
hostonly: Only setup the host toolchain
reconfig: Reload crossdev config and reselect toolchains
targets_wanted: All the targets to update
boards_wanted: Load targets from these boards
root: The root in which to install the toolchains.
"""
targets, crossdev_targets, reconfig_targets = {}, {}, {}
if not hostonly:
# For hostonly, we can skip most of the below logic, much of which won't
# work on bare systems where this is useful.
targets = ExpandTargets(targets_wanted)
# Now re-add any targets that might be from this board. This is to
# allow unofficial boards to declare their own toolchains.
for board in boards_wanted:
targets.update(toolchain.GetToolchainsForBoard(board))
# First check and initialize all cross targets that need to be.
for target in targets:
if TargetIsInitialized(target):
reconfig_targets[target] = targets[target]
else:
crossdev_targets[target] = targets[target]
if crossdev_targets:
logging.info('The following targets need to be re-initialized:')
logging.info('%s', crossdev_targets)
Crossdev.UpdateTargets(crossdev_targets, usepkg)
# Those that were not initialized may need a config update.
Crossdev.UpdateTargets(reconfig_targets, usepkg, config_only=True)
# If we're building a subset of toolchains for a board, we might not have
# all the tuples that the packages expect. We don't define the "full" set
# of tuples currently other than "whatever the full sdk has normally".
if usepkg or set(('all', 'sdk')) & targets_wanted:
# Since we have cross-compilers now, we can update these packages.
targets['host-post-cross'] = {}
# We want host updated.
targets['host'] = {}
# Now update all packages.
if UpdateTargets(targets, usepkg, root=root) or crossdev_targets or reconfig:
SelectActiveToolchains(targets, CONFIG_TARGET_SUFFIXES, root=root)
if deleteold:
CleanTargets(targets, root=root)
# Now that we've cleared out old versions, see if we need to rebuild
# anything. Can't do this earlier as it might not be broken.
RebuildLibtool(root=root)
def ShowConfig(name):
"""Show the toolchain tuples used by |name|
Args:
name: The board name to query.
"""
toolchains = toolchain.GetToolchainsForBoard(name)
# Make sure we display the default toolchain first.
print(','.join(
toolchain.FilterToolchains(toolchains, 'default', True).keys() +
toolchain.FilterToolchains(toolchains, 'default', False).keys()))
def ShowBoardConfig(board):
"""Show the toolchain tuples used by |board|
Args:
board: The board to query.
"""
toolchains = toolchain.GetToolchainsForBoard(board)
# Make sure we display the default toolchain first.
print ','.join(
toolchain.FilterToolchains(toolchains, 'default', True).keys() +
toolchain.FilterToolchains(toolchains, 'default', False).keys())
def testReadsBoardToolchains(self, find_overlays_mock):
"""Tests that we correctly parse toolchain configs for an overlay stack."""
# Create some fake overlays and put toolchain confs in a subset of them.
overlays = [os.path.join(self.tempdir, 'overlay%d' % i) for i in range(3)]
for overlay in overlays:
osutils.SafeMakedirs(overlay)
for overlay, contents in [(overlays[0], BASE_TOOLCHAIN_CONF),
(overlays[2], ADDITIONAL_TOOLCHAIN_CONF)]:
osutils.WriteFile(os.path.join(overlay, 'toolchain.conf'), contents)
find_overlays_mock.return_value = overlays
actual_targets = toolchain.GetToolchainsForBoard('board_value')
self.assertEqual(EXPECTED_TOOLCHAINS, actual_targets)
def ShowConfig(name):
"""Show the toolchain tuples used by |name|
Args:
name: The board name to query.
"""
toolchains = toolchain.GetToolchainsForBoard(name)
# Make sure we display the default toolchain first.
# Note: Do not use logging here as this is meant to be used by other tools.
print(','.join(
toolchain.FilterToolchains(toolchains, 'default', True).keys() +
toolchain.FilterToolchains(toolchains, 'default', False).keys()))
def testTarballCreation(self):
"""Tests that tarballs are created for all board toolchains."""
self._Prepare('chromiumos-sdk')
self.RunStage()
# Check that a tarball was created correctly for all toolchain sets.
sdk_toolchains = set(toolchain.GetToolchainsForBoard('sdk'))
all_toolchain_combos = set()
for board in self._run.site_config.GetBoards():
try:
toolchains = set(toolchain.GetToolchainsForBoard(board).keys())
if toolchains.issubset(sdk_toolchains):
all_toolchain_combos.add('-'.join(sorted(toolchains)))
except portage_util.MissingOverlayError:
pass
for toolchains in all_toolchain_combos:
overlay_tarball = os.path.join(
self.build_root, constants.DEFAULT_CHROOT_DIR,
constants.SDK_OVERLAYS_OUTPUT,
'built-sdk-overlay-toolchains-%s.tar.xz' % toolchains)
output = cros_build_lib.run(
['tar', '-I', 'xz', '-tf', overlay_tarball],
encoding='utf-8',
capture_output=True).stdout.splitlines()
# Check that the overlay tarball contains a marker file and that the
# board recorded by this marker file indeed uses the toolchains for which
# the tarball was built.
tmp_files = [
os.path.basename(x) for x in output if x.endswith('.tmp')
]
self.assertEqual(1, len(tmp_files))
board = tmp_files[0][:-len('.tmp')]
board_toolchains = '-'.join(
sorted(toolchain.GetToolchainsForBoard(board).keys()))
self.assertEqual(toolchains, board_toolchains)
def GenerateBoardConfig(self, board):
"""Generates the configuration for a given board.
Args:
board: board name to use to generate the configuration.
"""
toolchains = toolchain.GetToolchainsForBoard(board)
# Compute the overlay list.
portdir_overlays = portage_util.FindOverlays(constants.BOTH_OVERLAYS, board)
prefix = os.path.join(constants.SOURCE_ROOT, 'src', 'third_party')
board_overlays = [o for o in portdir_overlays if not o.startswith(prefix)]
header = "# Created by cros_sysroot_utils from --board=%s." % board
return self._GenerateConfig(toolchains, board_overlays, portdir_overlays,
header, BOARD_USE=board)
def UpdateToolchains(usepkg, getbinpkg, deleteold, hostonly, reconfig,
targets_wanted, boards_wanted):
"""Performs all steps to create a synchronized toolchain enviroment.
args:
arguments correspond to the given commandline flags
"""
targets, crossdev_targets, reconfig_targets = {}, {}, {}
if not hostonly:
# For hostonly, we can skip most of the below logic, much of which won't
# work on bare systems where this is useful.
targets = ExpandTargets(targets_wanted)
# Now re-add any targets that might be from this board. This is
# to allow unofficial boards to declare their own toolchains.
for board in boards_wanted:
targets.update(toolchain.GetToolchainsForBoard(board))
# First check and initialize all cross targets that need to be.
for target in targets:
if TargetIsInitialized(target):
reconfig_targets[target] = targets[target]
else:
crossdev_targets[target] = targets[target]
if crossdev_targets:
print 'The following targets need to be re-initialized:'
print crossdev_targets
Crossdev.UpdateTargets(crossdev_targets,
usepkg,
getbinpkg=getbinpkg)
# Those that were not initialized may need a config update.
Crossdev.UpdateTargets(reconfig_targets,
usepkg,
getbinpkg=getbinpkg,
config_only=True)
# We want host updated.
targets['host'] = {}
# Now update all packages.
if (UpdateTargets(targets, usepkg, getbinpkg=getbinpkg) or crossdev_targets
or reconfig):
SelectActiveToolchains(targets, CONFIG_TARGET_SUFFIXES)
if deleteold:
CleanTargets(targets)
def PerformStage(self):
config = self._run.config
build_root = self._build_root
logging.info('Build re-executions have finished. Chromite source '
'will not be modified for remainder of run.')
logging.info("config['important']=%s", config['important'])
logging.PrintBuildbotStepText("config['important']=%s" %
config['important'])
# Flat list of all child config boards. Since child configs
# are not allowed to have children, it is not necessary to search
# deeper than one generation.
child_configs = GetChildConfigListMetadata(
child_configs=config['child_configs'], config_status_map=None)
sdk_verinfo = cros_build_lib.LoadKeyValueFile(os.path.join(
build_root, constants.SDK_VERSION_FILE),
ignore_missing=True)
verinfo = self._run.GetVersionInfo()
platform_tag = getattr(self._run.attrs, 'release_tag')
if not platform_tag:
platform_tag = verinfo.VersionString()
version = {
'full': self._run.GetVersion(),
'milestone': verinfo.chrome_branch,
'platform': platform_tag,
}
metadata = {
# Version of the metadata format.
'metadata-version': '2',
'boards': config['boards'],
'child-configs': child_configs,
'build_type': config['build_type'],
'important': config['important'],
# Data for the toolchain used.
'sdk-version': sdk_verinfo.get('SDK_LATEST_VERSION', '<unknown>'),
'toolchain-url': sdk_verinfo.get('TC_PATH', '<unknown>'),
}
if len(config['boards']) == 1:
toolchains = toolchain.GetToolchainsForBoard(config['boards'][0],
buildroot=build_root)
metadata['toolchain-tuple'] = (
toolchain.FilterToolchains(toolchains, 'default', True).keys()
+ toolchain.FilterToolchains(toolchains, 'default',
False).keys())
logging.info('Metadata being written: %s', metadata)
self._run.attrs.metadata.UpdateWithDict(metadata)
# Update 'version' separately to avoid overwriting the existing
# entries in it (e.g. PFQ builders may have written the Chrome
# version to uprev).
logging.info("Metadata 'version' being written: %s", version)
self._run.attrs.metadata.UpdateKeyDictWithDict('version', version)
# Ensure that all boards and child config boards have a per-board
# metadata subdict.
for b in config['boards']:
self._run.attrs.metadata.UpdateBoardDictWithDict(b, {})
for cc in child_configs:
for b in cc['boards']:
self._run.attrs.metadata.UpdateBoardDictWithDict(b, {})
# Upload build metadata (and write it to database if necessary)
self.UploadMetadata(filename=constants.PARTIAL_METADATA_JSON)
# Write child-per-build and board-per-build rows to database
build_id, db = self._run.GetCIDBHandle()
if db:
# TODO(akeshet): replace this with a GetValue call once crbug.com/406522
# is resolved
per_board_dict = self._run.attrs.metadata.GetDict(
)['board-metadata']
for board, board_metadata in per_board_dict.items():
db.InsertBoardPerBuild(build_id, board)
if board_metadata:
db.UpdateBoardPerBuildMetadata(build_id, board,
board_metadata)
for child_config in self._run.attrs.metadata.GetValue(
'child-configs'):
db.InsertChildConfigPerBuild(build_id, child_config['name'])
# If this build has a master build, ensure that the master full_version
# is the same as this build's full_version. This is a sanity check to
# avoid bugs in master-slave logic.
master_id = self._run.attrs.metadata.GetDict().get(
'master_build_id')
if master_id is not None:
master_full_version = db.GetBuildStatus(
master_id)['full_version']
my_full_version = self._run.attrs.metadata.GetValue(
'version').get('full')
if master_full_version != my_full_version:
raise failures_lib.MasterSlaveVersionMismatchFailure(
'Master build id %s has full_version %s, while slave version is '
'%s.' %
(master_id, master_full_version, my_full_version))
# Abort previous hw test suites. This happens after reexecution as it
# requires chromite/third_party/swarming.client, which is not available
# untill after reexecution.
self._AbortPreviousHWTestSuites(version['milestone'])
def main(argv):
conf = cros_build_lib.LoadKeyValueFile(
os.path.join(constants.SOURCE_ROOT, constants.SDK_VERSION_FILE),
ignore_missing=True)
sdk_latest_version = conf.get('SDK_LATEST_VERSION', '<unknown>')
bootstrap_latest_version = conf.get('BOOTSTRAP_LATEST_VERSION', '<unknown>')
parser, commands = _CreateParser(sdk_latest_version, bootstrap_latest_version)
options = parser.parse_args(argv)
chroot_command = options.commands
# Some sanity checks first, before we ask for sudo credentials.
cros_build_lib.AssertOutsideChroot()
host = os.uname()[4]
if host != 'x86_64':
parser.error(
"cros_sdk is currently only supported on x86_64; you're running"
" %s. Please find a x86_64 machine." % (host,))
_ReportMissing(osutils.FindMissingBinaries(NEEDED_TOOLS))
if options.proxy_sim:
_ReportMissing(osutils.FindMissingBinaries(PROXY_NEEDED_TOOLS))
_ReExecuteIfNeeded([sys.argv[0]] + argv)
if options.ns_pid:
first_pid = namespaces.CreatePidNs()
else:
first_pid = None
# Expand out the aliases...
if options.replace:
options.delete = options.create = True
if options.bootstrap:
options.create = True
# If a command is not given, default to enter.
# pylint: disable=protected-access
# This _group_actions access sucks, but upstream decided to not include an
# alternative to optparse's option_list, and this is what they recommend.
options.enter |= not any(getattr(options, x.dest)
for x in commands._group_actions)
# pylint: enable=protected-access
options.enter |= bool(chroot_command)
if options.enter and options.delete and not options.create:
parser.error("Trying to enter the chroot when --delete "
"was specified makes no sense.")
# Finally, discern if we need to create the chroot.
chroot_exists = os.path.exists(options.chroot)
if options.create or options.enter:
# Only create if it's being wiped, or if it doesn't exist.
if not options.delete and chroot_exists:
options.create = False
else:
options.download = True
# Finally, flip create if necessary.
if options.enter:
options.create |= not chroot_exists
if not options.sdk_version:
sdk_version = (bootstrap_latest_version if options.bootstrap
else sdk_latest_version)
else:
sdk_version = options.sdk_version
if options.buildbot_log_version:
logging.PrintBuildbotStepText(sdk_version)
# Based on selections, determine the tarball to fetch.
if options.sdk_url:
urls = [options.sdk_url]
elif options.bootstrap:
urls = GetStage3Urls(sdk_version)
else:
urls = GetArchStageTarballs(sdk_version)
# Get URLs for the toolchains overlay, if one is to be used.
toolchains_overlay_urls = None
if not options.bootstrap:
toolchains = None
if options.toolchains:
toolchains = options.toolchains.split(',')
elif options.board:
toolchains = toolchain.GetToolchainsForBoard(options.board).keys()
if toolchains:
toolchains_overlay_urls = GetToolchainsOverlayUrls(sdk_version,
toolchains)
lock_path = os.path.dirname(options.chroot)
lock_path = os.path.join(
lock_path, '.%s_lock' % os.path.basename(options.chroot).lstrip('.'))
with cgroups.SimpleContainChildren('cros_sdk', pid=first_pid):
with locking.FileLock(lock_path, 'chroot lock') as lock:
toolchains_overlay_tarball = None
if options.proxy_sim:
_ProxySimSetup(options)
if options.delete and os.path.exists(options.chroot):
lock.write_lock()
DeleteChroot(options.chroot)
sdk_cache = os.path.join(options.cache_dir, 'sdks')
distfiles_cache = os.path.join(options.cache_dir, 'distfiles')
osutils.SafeMakedirsNonRoot(options.cache_dir)
for target in (sdk_cache, distfiles_cache):
src = os.path.join(constants.SOURCE_ROOT, os.path.basename(target))
if not os.path.exists(src):
osutils.SafeMakedirsNonRoot(target)
continue
lock.write_lock(
"Upgrade to %r needed but chroot is locked; please exit "
"all instances so this upgrade can finish." % src)
if not os.path.exists(src):
# Note that while waiting for the write lock, src may've vanished;
# it's a rare race during the upgrade process that's a byproduct
# of us avoiding taking a write lock to do the src check. If we
# took a write lock for that check, it would effectively limit
# all cros_sdk for a chroot to a single instance.
osutils.SafeMakedirsNonRoot(target)
elif not os.path.exists(target):
# Upgrade occurred, but a reversion, or something whacky
# occurred writing to the old location. Wipe and continue.
os.rename(src, target)
else:
# Upgrade occurred once already, but either a reversion or
# some before/after separate cros_sdk usage is at play.
# Wipe and continue.
osutils.RmDir(src)
if options.download:
lock.write_lock()
sdk_tarball = FetchRemoteTarballs(
sdk_cache, urls, 'stage3' if options.bootstrap else 'SDK')
if toolchains_overlay_urls:
toolchains_overlay_tarball = FetchRemoteTarballs(
sdk_cache, toolchains_overlay_urls, 'SDK toolchains overlay',
allow_none=True)
if options.create:
lock.write_lock()
CreateChroot(options.chroot, sdk_tarball, toolchains_overlay_tarball,
options.cache_dir,
nousepkg=(options.bootstrap or options.nousepkg))
if options.enter:
lock.read_lock()
EnterChroot(options.chroot, options.cache_dir, options.chrome_root,
options.chrome_root_mount, options.workspace,
chroot_command)
def PerformStage(self):
"""Perform the actual work for this stage.
This includes final metadata archival, and update CIDB with our final status
as well as producting a logged build result summary.
"""
build_identifier, _ = self._run.GetCIDBHandle()
build_id = build_identifier.cidb_id
buildbucket_id = build_identifier.buildbucket_id
if results_lib.Results.BuildSucceededSoFar(self.buildstore,
buildbucket_id, self.name):
final_status = constants.BUILDER_STATUS_PASSED
else:
final_status = constants.BUILDER_STATUS_FAILED
if not hasattr(self._run.attrs, 'release_tag'):
# If, for some reason, sync stage was not completed and
# release_tag was not set. Set it to None here because
# ArchiveResults() depends the existence of this attr.
self._run.attrs.release_tag = None
# Set up our report metadata.
self._run.attrs.metadata.UpdateWithDict(
self.GetReportMetadata(
final_status=final_status,
completion_instance=self._completion_instance))
src_root = self._build_root
# Workspace builders use a different buildroot for overlays.
if self._run.config.workspace_branch and self._run.options.workspace:
src_root = self._run.options.workspace
# Add tags for the arches and statuses of the build.
# arches requires crossdev which isn't available at the early part of the
# build.
arches = []
for board in self._run.config['boards']:
toolchains = toolchain.GetToolchainsForBoard(board,
buildroot=src_root)
default = list(
toolchain.FilterToolchains(toolchains, 'default', True))
if default:
try:
arches.append(toolchain.GetArchForTarget(default[0]))
except cros_build_lib.RunCommandError as e:
logging.warning(
'Unable to retrieve arch for board %s default toolchain %s: %s',
board, default, e)
tags = {
'arches': arches,
'status': final_status,
}
results = self._run.attrs.metadata.GetValue('results')
for stage in results:
tags['stage_status:%s' % stage['name']] = stage['status']
tags['stage_summary:%s' % stage['name']] = stage['summary']
self._run.attrs.metadata.UpdateKeyDictWithDict(constants.METADATA_TAGS,
tags)
# Some operations can only be performed if a valid version is available.
try:
self._run.GetVersionInfo()
self.ArchiveResults(final_status)
metadata_url = os.path.join(self.upload_url,
constants.METADATA_JSON)
except cbuildbot_run.VersionNotSetError:
logging.error('A valid version was never set for this run. '
'Can not archive results.')
metadata_url = ''
results_lib.Results.Report(sys.stdout,
current_version=(self._run.attrs.release_tag
or ''))
# Upload goma log if used for BuildPackage and TestSimpleChrome.
_UploadAndLinkGomaLogIfNecessary(
'BuildPackages', self._run.config.name, self._run.options.goma_dir,
self._run.options.goma_client_json,
self._run.attrs.metadata.GetValueWithDefault('goma_tmp_dir'))
_UploadAndLinkGomaLogIfNecessary(
'TestSimpleChromeWorkflow', self._run.config.name,
self._run.options.goma_dir, self._run.options.goma_client_json,
self._run.attrs.metadata.GetValueWithDefault(
'goma_tmp_dir_for_simple_chrome'))
if self.buildstore.AreClientsReady():
status_for_db = final_status
# TODO(pprabhu): After BuildData and CBuildbotMetdata are merged, remove
# this extra temporary object creation.
# XXX:HACK We're creating a BuildData with an empty URL. Don't try to
# MarkGathered this object.
build_data = metadata_lib.BuildData(
'', self._run.attrs.metadata.GetDict())
# TODO(akeshet): Find a clearer way to get the "primary upload url" for
# the metadata.json file. One alternative is _GetUploadUrls(...)[0].
# Today it seems that element 0 of its return list is the primary upload
# url, but there is no guarantee or unit test coverage of that.
self.buildstore.FinishBuild(build_id,
status=status_for_db,
summary=build_data.failure_message,
metadata_url=metadata_url)
duration = self._GetBuildDuration()
mon_fields = {
'status': status_for_db,
'build_config': self._run.config.name,
'important': self._run.config.important
}
metrics.Counter(
constants.MON_BUILD_COMP_COUNT).increment(fields=mon_fields)
metrics.CumulativeSecondsDistribution(
constants.MON_BUILD_DURATION).add(duration, fields=mon_fields)
if self._run.options.sanity_check_build:
metrics.Counter(
constants.MON_BUILD_SANITY_COMP_COUNT).increment(
fields=mon_fields)
metrics.Gauge(
constants.MON_BUILD_SANITY_ID,
description=
'The build number of the latest sanity build. Used '
'for recovering the link to the latest failing build '
'in the alert when a sanity build fails.',
field_spec=[
ts_mon.StringField('status'),
ts_mon.StringField('build_config'),
ts_mon.StringField('builder_name'),
ts_mon.BooleanField('important')
]).set(self._run.buildnumber,
fields=dict(
mon_fields,
builder_name=self._run.GetBuilderName()))
if config_lib.IsMasterCQ(self._run.config):
self_destructed = self._run.attrs.metadata.GetValueWithDefault(
constants.SELF_DESTRUCTED_BUILD, False)
mon_fields = {
'status': status_for_db,
'self_destructed': self_destructed
}
metrics.CumulativeSecondsDistribution(
constants.MON_CQ_BUILD_DURATION).add(duration,
fields=mon_fields)
annotator_link = uri_lib.ConstructAnnotatorUri(build_id)
logging.PrintBuildbotLink('Build annotator', annotator_link)
# From this point forward, treat all exceptions as warnings.
self._post_completion = True
# Dump report about things we retry.
retry_stats.ReportStats(sys.stdout)
def main(argv):
conf = cros_build_lib.LoadKeyValueFile(
os.path.join(constants.SOURCE_ROOT, constants.SDK_VERSION_FILE),
ignore_missing=True)
sdk_latest_version = conf.get('SDK_LATEST_VERSION', '<unknown>')
bootstrap_latest_version = conf.get('BOOTSTRAP_LATEST_VERSION', '<unknown>')
parser, commands = _CreateParser(sdk_latest_version, bootstrap_latest_version)
options = parser.parse_args(argv)
chroot_command = options.commands
# Some sanity checks first, before we ask for sudo credentials.
cros_build_lib.AssertOutsideChroot()
host = os.uname()[4]
if host != 'x86_64':
cros_build_lib.Die(
"cros_sdk is currently only supported on x86_64; you're running"
" %s. Please find a x86_64 machine." % (host,))
_ReportMissing(osutils.FindMissingBinaries(NEEDED_TOOLS))
if options.proxy_sim:
_ReportMissing(osutils.FindMissingBinaries(PROXY_NEEDED_TOOLS))
missing_image_tools = osutils.FindMissingBinaries(IMAGE_NEEDED_TOOLS)
if (sdk_latest_version == '<unknown>' or
bootstrap_latest_version == '<unknown>'):
cros_build_lib.Die(
'No SDK version was found. '
'Are you in a Chromium source tree instead of Chromium OS?\n\n'
'Please change to a directory inside your Chromium OS source tree\n'
'and retry. If you need to setup a Chromium OS source tree, see\n'
' http://www.chromium.org/chromium-os/developer-guide')
any_snapshot_operation = (options.snapshot_create or options.snapshot_restore
or options.snapshot_delete or options.snapshot_list)
if any_snapshot_operation and not options.use_image:
cros_build_lib.Die('Snapshot operations are not compatible with '
'--nouse-image.')
if (options.snapshot_delete and options.snapshot_delete ==
options.snapshot_restore):
parser.error('Cannot --snapshot_delete the same snapshot you are '
'restoring with --snapshot_restore.')
_ReExecuteIfNeeded([sys.argv[0]] + argv)
lock_path = os.path.dirname(options.chroot)
lock_path = os.path.join(
lock_path, '.%s_lock' % os.path.basename(options.chroot).lstrip('.'))
# Expand out the aliases...
if options.replace:
options.delete = options.create = True
if options.bootstrap:
options.create = True
# If a command is not given, default to enter.
# pylint: disable=protected-access
# This _group_actions access sucks, but upstream decided to not include an
# alternative to optparse's option_list, and this is what they recommend.
options.enter |= not any(getattr(options, x.dest)
for x in commands._group_actions)
# pylint: enable=protected-access
options.enter |= bool(chroot_command)
if (options.delete and not options.create and
(options.enter or any_snapshot_operation)):
parser.error("Trying to enter or snapshot the chroot when --delete "
"was specified makes no sense.")
if options.working_dir is not None and not os.path.isabs(options.working_dir):
options.working_dir = path_util.ToChrootPath(options.working_dir)
# Clean up potential leftovers from previous interrupted builds.
# TODO(bmgordon): Remove this at the end of 2017. That should be long enough
# to get rid of them all.
chroot_build_path = options.chroot + '.build'
if options.use_image and os.path.exists(chroot_build_path):
try:
with cgroups.SimpleContainChildren('cros_sdk'):
with locking.FileLock(lock_path, 'chroot lock') as lock:
logging.notice('Cleaning up leftover build directory %s',
chroot_build_path)
lock.write_lock()
osutils.UmountTree(chroot_build_path)
osutils.RmDir(chroot_build_path)
except cros_build_lib.RunCommandError as e:
logging.warning('Unable to remove %s: %s', chroot_build_path, e)
# Discern if we need to create the chroot.
chroot_ver_file = os.path.join(options.chroot, 'etc', 'cros_chroot_version')
chroot_exists = os.path.exists(chroot_ver_file)
if (options.use_image and not chroot_exists and not options.delete and
not missing_image_tools and
os.path.exists(_ImageFileForChroot(options.chroot))):
# Try to re-mount an existing image in case the user has rebooted.
with cgroups.SimpleContainChildren('cros_sdk'):
with locking.FileLock(lock_path, 'chroot lock') as lock:
logging.debug('Checking if existing chroot image can be mounted.')
lock.write_lock()
cros_sdk_lib.MountChroot(options.chroot, create=False)
chroot_exists = os.path.exists(chroot_ver_file)
if chroot_exists:
logging.notice('Mounted existing image %s on chroot',
_ImageFileForChroot(options.chroot))
if (options.create or options.enter or options.snapshot_create or
options.snapshot_restore):
# Only create if it's being wiped, or if it doesn't exist.
if not options.delete and chroot_exists:
options.create = False
else:
options.download = True
# Finally, flip create if necessary.
if options.enter or options.snapshot_create:
options.create |= not chroot_exists
# Anything that needs to manipulate the main chroot mount or communicate with
# LVM needs to be done here before we enter the new namespaces.
# If deleting, do it regardless of the use_image flag so that a
# previously-created loopback chroot can also be cleaned up.
# TODO(bmgordon): See if the DeleteChroot call below can be removed in
# favor of this block.
chroot_deleted = False
if options.delete:
with cgroups.SimpleContainChildren('cros_sdk'):
with locking.FileLock(lock_path, 'chroot lock') as lock:
lock.write_lock()
if missing_image_tools:
logging.notice('Unmounting chroot.')
osutils.UmountTree(options.chroot)
else:
logging.notice('Deleting chroot.')
cros_sdk_lib.CleanupChrootMount(options.chroot, delete_image=True)
osutils.RmDir(options.chroot, ignore_missing=True)
chroot_deleted = True
# Make sure the main chroot mount is visible. Contents will be filled in
# below if needed.
if options.create and options.use_image:
if missing_image_tools:
raise SystemExit(
'''The tool(s) %s were not found.
Please make sure the lvm2 and thin-provisioning-tools packages
are installed on your host.
Example(ubuntu):
sudo apt-get install lvm2 thin-provisioning-tools
If you want to run without lvm2, pass --nouse-image (chroot
snapshots will be unavailable).''' % ', '.join(missing_image_tools))
logging.debug('Making sure chroot image is mounted.')
with cgroups.SimpleContainChildren('cros_sdk'):
with locking.FileLock(lock_path, 'chroot lock') as lock:
lock.write_lock()
if not cros_sdk_lib.MountChroot(options.chroot, create=True):
cros_build_lib.Die('Unable to mount %s on chroot',
_ImageFileForChroot(options.chroot))
logging.notice('Mounted %s on chroot',
_ImageFileForChroot(options.chroot))
# Snapshot operations will always need the VG/LV, but other actions won't.
if any_snapshot_operation:
with cgroups.SimpleContainChildren('cros_sdk'):
with locking.FileLock(lock_path, 'chroot lock') as lock:
chroot_vg, chroot_lv = cros_sdk_lib.FindChrootMountSource(
options.chroot)
if not chroot_vg or not chroot_lv:
cros_build_lib.Die('Unable to find VG/LV for chroot %s',
options.chroot)
# Delete snapshot before creating a new one. This allows the user to
# throw out old state, create a new snapshot, and enter the chroot in a
# single call to cros_sdk. Since restore involves deleting, also do it
# before creating.
if options.snapshot_restore:
lock.write_lock()
valid_snapshots = ListChrootSnapshots(chroot_vg, chroot_lv)
if options.snapshot_restore not in valid_snapshots:
cros_build_lib.Die('%s is not a valid snapshot to restore to. '
'Valid snapshots: %s', options.snapshot_restore,
', '.join(valid_snapshots))
osutils.UmountTree(options.chroot)
if not RestoreChrootSnapshot(options.snapshot_restore, chroot_vg,
chroot_lv):
cros_build_lib.Die('Unable to restore chroot to snapshot.')
if not cros_sdk_lib.MountChroot(options.chroot, create=False):
cros_build_lib.Die('Unable to mount restored snapshot onto chroot.')
# Use a read lock for snapshot delete and create even though they modify
# the filesystem, because they don't modify the mounted chroot itself.
# The underlying LVM commands take their own locks, so conflicting
# concurrent operations here may crash cros_sdk, but won't corrupt the
# chroot image. This tradeoff seems worth it to allow snapshot
# operations on chroots that have a process inside.
if options.snapshot_delete:
lock.read_lock()
DeleteChrootSnapshot(options.snapshot_delete, chroot_vg, chroot_lv)
if options.snapshot_create:
lock.read_lock()
if not CreateChrootSnapshot(options.snapshot_create, chroot_vg,
chroot_lv):
cros_build_lib.Die('Unable to create snapshot.')
img_path = _ImageFileForChroot(options.chroot)
if (options.use_image and os.path.exists(options.chroot) and
os.path.exists(img_path)):
img_stat = os.stat(img_path)
img_used_bytes = img_stat.st_blocks * 512
mount_stat = os.statvfs(options.chroot)
mount_used_bytes = mount_stat.f_frsize * (mount_stat.f_blocks -
mount_stat.f_bfree)
extra_gbs = (img_used_bytes - mount_used_bytes) / 2**30
if extra_gbs > MAX_UNUSED_IMAGE_GBS:
logging.notice('%s is using %s GiB more than needed. Running '
'fstrim.', img_path, extra_gbs)
cmd = ['fstrim', options.chroot]
try:
cros_build_lib.RunCommand(cmd, print_cmd=False)
except cros_build_lib.RunCommandError as e:
logging.warning('Running fstrim failed. Consider running fstrim on '
'your chroot manually.\nError: %s', e)
# Enter a new set of namespaces. Everything after here cannot directly affect
# the hosts's mounts or alter LVM volumes.
namespaces.SimpleUnshare()
if options.ns_pid:
first_pid = namespaces.CreatePidNs()
else:
first_pid = None
if options.snapshot_list:
for snap in ListChrootSnapshots(chroot_vg, chroot_lv):
print(snap)
sys.exit(0)
if not options.sdk_version:
sdk_version = (bootstrap_latest_version if options.bootstrap
else sdk_latest_version)
else:
sdk_version = options.sdk_version
if options.buildbot_log_version:
logging.PrintBuildbotStepText(sdk_version)
# Based on selections, determine the tarball to fetch.
if options.download:
if options.sdk_url:
urls = [options.sdk_url]
elif options.bootstrap:
urls = GetStage3Urls(sdk_version)
else:
urls = GetArchStageTarballs(sdk_version)
# Get URLs for the toolchains overlay, if one is to be used.
toolchains_overlay_urls = None
if not options.bootstrap:
toolchains = None
if options.toolchains:
toolchains = options.toolchains.split(',')
elif options.board:
toolchains = toolchain.GetToolchainsForBoard(options.board).keys()
if toolchains:
toolchains_overlay_urls = GetToolchainsOverlayUrls(sdk_version,
toolchains)
with cgroups.SimpleContainChildren('cros_sdk', pid=first_pid):
with locking.FileLock(lock_path, 'chroot lock') as lock:
toolchains_overlay_tarball = None
if options.proxy_sim:
_ProxySimSetup(options)
if (options.delete and not chroot_deleted and
(os.path.exists(options.chroot) or
os.path.exists(_ImageFileForChroot(options.chroot)))):
lock.write_lock()
DeleteChroot(options.chroot)
sdk_cache = os.path.join(options.cache_dir, 'sdks')
distfiles_cache = os.path.join(options.cache_dir, 'distfiles')
osutils.SafeMakedirsNonRoot(options.cache_dir)
for target in (sdk_cache, distfiles_cache):
src = os.path.join(constants.SOURCE_ROOT, os.path.basename(target))
if not os.path.exists(src):
osutils.SafeMakedirsNonRoot(target)
continue
lock.write_lock(
"Upgrade to %r needed but chroot is locked; please exit "
"all instances so this upgrade can finish." % src)
if not os.path.exists(src):
# Note that while waiting for the write lock, src may've vanished;
# it's a rare race during the upgrade process that's a byproduct
# of us avoiding taking a write lock to do the src check. If we
# took a write lock for that check, it would effectively limit
# all cros_sdk for a chroot to a single instance.
osutils.SafeMakedirsNonRoot(target)
elif not os.path.exists(target):
# Upgrade occurred, but a reversion, or something whacky
# occurred writing to the old location. Wipe and continue.
os.rename(src, target)
else:
# Upgrade occurred once already, but either a reversion or
# some before/after separate cros_sdk usage is at play.
# Wipe and continue.
osutils.RmDir(src)
if options.download:
lock.write_lock()
sdk_tarball = FetchRemoteTarballs(
sdk_cache, urls, 'stage3' if options.bootstrap else 'SDK')
if toolchains_overlay_urls:
toolchains_overlay_tarball = FetchRemoteTarballs(
sdk_cache, toolchains_overlay_urls, 'SDK toolchains overlay',
allow_none=True)
if options.create:
lock.write_lock()
CreateChroot(options.chroot, sdk_tarball, toolchains_overlay_tarball,
options.cache_dir,
nousepkg=(options.bootstrap or options.nousepkg))
if options.enter:
lock.read_lock()
EnterChroot(options.chroot, options.cache_dir, options.chrome_root,
options.chrome_root_mount, options.workspace,
options.goma_dir, options.goma_client_json,
options.working_dir, chroot_command)
