def RegenCache(overlay):
"""Regenerate the cache of the specified overlay.
overlay: The tree to regenerate the cache for.
"""
repo_name = GetOverlayName(overlay)
if not repo_name:
return
layout = cros_build_lib.LoadKeyValueFile('%s/metadata/layout.conf' %
overlay,
ignore_missing=True)
if layout.get('cache-format') != 'md5-dict':
return
# Regen for the whole repo.
cros_build_lib.RunCommand([
'egencache', '--update', '--repo', repo_name, '--jobs',
str(multiprocessing.cpu_count())
])
# If there was nothing new generated, then let's just bail.
result = cros_build_lib.RunCommand(['git', 'status', '-s', 'metadata/'],
cwd=overlay,
redirect_stdout=True)
if not result.output:
return
# Explicitly add any new files to the index.
cros_build_lib.RunCommand(['git', 'add', 'metadata/'], cwd=overlay)
# Explicitly tell git to also include rm-ed files.
cros_build_lib.RunCommand(
['git', 'commit', '-m', 'regen cache', 'metadata/'], cwd=overlay)
def SourceEnvironment(script, whitelist, ifs=',', env=None):
"""Returns the environment exported by a shell script.
Note that the script is actually executed (sourced), so do not use this on
files that have side effects (such as modify the file system). Stdout will
be sent to /dev/null, so just echoing is OK.
Arguments:
script: The shell script to 'source'.
whitelist: An iterable of environment variables to retrieve values for.
ifs: When showing arrays, what separator to use.
env: A dict of the initial env to pass down. You can also pass it None
(to clear the env) or True (to preserve the current env).
Returns:
A dictionary containing the values of the whitelisted environment
variables that are set.
"""
dump_script = ['source "%s" >/dev/null' % script,
'IFS="%s"' % ifs]
for var in whitelist:
dump_script.append(
'[[ "${%(var)s+set}" == "set" ]] && echo %(var)s="${%(var)s[*]}"'
% {'var': var})
dump_script.append('exit 0')
if env is None:
env = {}
elif env is True:
env = None
output = cros_build_lib.RunCommand(['bash'], env=env, redirect_stdout=True,
redirect_stderr=True, print_cmd=False,
input='\n'.join(dump_script)).output
return cros_build_lib.LoadKeyValueFile(cStringIO.StringIO(output))
def GetUpdateStatus(cls, device, keys=None):
"""Returns the status of the update engine on the |device|.
Retrieves the status from update engine and confirms all keys are
in the status.
Args:
device: A ChromiumOSDevice object.
keys: the keys to look for in the status result (defaults to
['CURRENT_OP']).
Returns:
A list of values in the order of |keys|.
"""
keys = keys or ['CURRENT_OP']
result = device.RunCommand([cls.UPDATE_ENGINE_BIN, '--status'],
capture_output=True)
if not result.output:
raise Exception('Cannot get update status')
try:
status = cros_build_lib.LoadKeyValueFile(
cStringIO.StringIO(result.output))
except ValueError:
raise ValueError('Cannot parse update status')
values = []
for key in keys:
if key not in status:
raise ValueError('Missing %s in the update engine status')
values.append(status.get(key))
return values
def GetArchForTarget(target):
"""Returns the arch used by the given toolchain.
Args:
target: a toolchain.
"""
info = cros_build_lib.RunCommand(['crossdev', '--show-target-cfg', target],
capture_output=True, quiet=True).output
return cros_build_lib.LoadKeyValueFile(cStringIO.StringIO(info)).get('arch')
def _EnvdGetVar(envd, var):
"""Given a Gentoo env.d file, extract a var from it
Args:
envd: The env.d file to load (may be a glob path)
var: The var to extract
Returns:
The value of |var|
"""
envds = glob.glob(envd)
assert len(envds) == 1, '%s: should have exactly 1 env.d file' % envd
envd = envds[0]
return cros_build_lib.LoadKeyValueFile(envd)[var]
def main(argv):
usage = """usage: %prog [options] [VAR1=val1 .. VARn=valn -- args]
This script is used for manipulating local chroot environments; creating,
deleting, downloading, etc. If given --enter (or no args), it defaults
to an interactive bash shell within the chroot.
If given args those are passed to the chroot environment, and executed."""
conf = cros_build_lib.LoadKeyValueFile(os.path.join(
constants.SOURCE_ROOT, constants.SDK_VERSION_FILE),
ignore_missing=True)
sdk_latest_version = conf.get('COREOS_SDK_VERSION', '<unknown>')
parser = commandline.OptionParser(usage=usage, caching=True)
commands = parser.add_option_group("Commands")
commands.add_option('--enter',
action='store_true',
default=False,
help='Enter the SDK chroot. Implies --create.')
commands.add_option(
'--create',
action='store_true',
default=False,
help='Create the chroot only if it does not already exist. '
'Implies --download.')
commands.add_option(
'--bootstrap',
action='store_true',
default=False,
help='Build everything from scratch, including the sdk. '
'Use this only if you need to validate a change '
'that affects SDK creation itself (toolchain and '
'build are typically the only folk who need this). '
'Note this will quite heavily slow down the build. '
'This option implies --create --nousepkg.')
commands.add_option(
'-r',
'--replace',
action='store_true',
default=False,
help='Replace an existing SDK chroot. Basically an alias '
'for --delete --create.')
commands.add_option('--delete',
action='store_true',
default=False,
help='Delete the current SDK chroot if it exists.')
commands.add_option('--download',
action='store_true',
default=False,
help='Download the sdk.')
# Global options:
default_chroot = os.path.join(constants.SOURCE_ROOT,
constants.DEFAULT_CHROOT_DIR)
parser.add_option('--chroot',
dest='chroot',
default=default_chroot,
type='path',
help=('SDK chroot dir name [%s]' %
constants.DEFAULT_CHROOT_DIR))
parser.add_option('--chrome_root',
default=None,
type='path',
help='Mount this chrome root into the SDK chroot')
parser.add_option('--chrome_root_mount',
default=None,
type='path',
help='Mount chrome into this path inside SDK chroot')
parser.add_option(
'--nousepkg',
action='store_true',
default=False,
help='Do not use binary packages when creating a chroot.')
parser.add_option('--nogetbinpkg',
action='store_true',
default=False,
help='Do not fetch remote binary packages.')
parser.add_option('-u',
'--url',
dest='sdk_url',
default=None,
help=('''Use sdk tarball located at this url.
Use file:// for local files.'''))
parser.add_option('--sdk-version',
default=sdk_latest_version,
help='Use this sdk version. Current is %default.')
options, chroot_command = parser.parse_args(argv)
# Some sanity checks first, before we ask for sudo credentials.
cros_build_lib.AssertOutsideChroot()
host = os.uname()[4]
if host != 'x86_64':
parser.error(
"cros_sdk is currently only supported on x86_64; you're running"
" %s. Please find a x86_64 machine." % (host, ))
missing = osutils.FindMissingBinaries(NEEDED_TOOLS)
if missing:
parser.error(
('The tool(s) %s were not found.\n'
'Please install the appropriate package in your host.\n'
'Example(ubuntu):\n'
' sudo apt-get install <packagename>' % (', '.join(missing))))
_ReExecuteIfNeeded([sys.argv[0]] + argv)
# Expand out the aliases...
if options.replace:
options.delete = options.create = True
if options.bootstrap:
options.create = True
# If a command is not given, default to enter.
options.enter |= not any(
getattr(options, x.dest) for x in commands.option_list)
options.enter |= bool(chroot_command)
if options.enter and options.delete and not options.create:
parser.error("Trying to enter the chroot when --delete "
"was specified makes no sense.")
# Finally, discern if we need to create the chroot.
chroot_exists = os.path.exists(options.chroot)
if options.create or options.enter:
# Only create if it's being wiped, or if it doesn't exist.
if not options.delete and chroot_exists:
options.create = False
else:
options.download = True
# Finally, flip create if necessary.
if options.enter:
options.create |= not chroot_exists
# Based on selections, fetch the tarball.
if options.sdk_url:
urls = [options.sdk_url]
else:
urls = GetArchStageTarballs(options.sdk_version)
lock_path = os.path.dirname(options.chroot)
lock_path = os.path.join(lock_path,
'.%s_lock' % os.path.basename(options.chroot))
with cgroups.SimpleContainChildren('cros_sdk'):
with locking.FileLock(lock_path, 'chroot lock') as lock:
if options.delete and os.path.exists(options.chroot):
lock.write_lock()
DeleteChroot(options.chroot)
sdk_cache = os.path.join(options.cache_dir, 'sdks')
distfiles_cache = os.path.join(options.cache_dir, 'distfiles')
osutils.SafeMakedirs(options.cache_dir)
for target in (sdk_cache, distfiles_cache):
src = os.path.join(constants.SOURCE_ROOT,
os.path.basename(target))
if not os.path.exists(src):
osutils.SafeMakedirs(target)
continue
lock.write_lock(
"Upgrade to %r needed but chroot is locked; please exit "
"all instances so this upgrade can finish." % src)
if not os.path.exists(src):
# Note that while waiting for the write lock, src may've vanished;
# it's a rare race during the upgrade process that's a byproduct
# of us avoiding taking a write lock to do the src check. If we
# took a write lock for that check, it would effectively limit
# all cros_sdk for a chroot to a single instance.
osutils.SafeMakedirs(target)
elif not os.path.exists(target):
# Upgrade occurred, but a reversion, or something whacky
# occurred writing to the old location. Wipe and continue.
os.rename(src, target)
else:
# Upgrade occurred once already, but either a reversion or
# some before/after separate cros_sdk usage is at play.
# Wipe and continue.
osutils.RmDir(src)
if options.download:
lock.write_lock()
sdk_tarball = FetchRemoteTarballs(sdk_cache, urls)
if options.create:
lock.write_lock()
CreateChroot(options.chroot,
sdk_tarball,
options.cache_dir,
nousepkg=(options.bootstrap or options.nousepkg),
nogetbinpkg=options.nogetbinpkg)
if options.enter:
lock.read_lock()
EnterChroot(options.chroot, options.cache_dir,
options.chrome_root, options.chrome_root_mount,
chroot_command)
def PerformStage(self):
config = self._run.config
build_root = self._build_root
logging.info('Build re-executions have finished. Chromite source '
'will not be modified for remainder of run.')
logging.info("config['important']=%s", config['important'])
logging.PrintBuildbotStepText("config['important']=%s" %
config['important'])
# Flat list of all child config boards. Since child configs
# are not allowed to have children, it is not necessary to search
# deeper than one generation.
child_configs = GetChildConfigListMetadata(
child_configs=config['child_configs'], config_status_map=None)
sdk_verinfo = cros_build_lib.LoadKeyValueFile(os.path.join(
build_root, constants.SDK_VERSION_FILE),
ignore_missing=True)
verinfo = self._run.GetVersionInfo()
platform_tag = getattr(self._run.attrs, 'release_tag')
if not platform_tag:
platform_tag = verinfo.VersionString()
version = {
'full': self._run.GetVersion(),
'milestone': verinfo.chrome_branch,
'platform': platform_tag,
}
metadata = {
# Version of the metadata format.
'metadata-version': '2',
'boards': config['boards'],
'child-configs': child_configs,
'build_type': config['build_type'],
'important': config['important'],
# Data for the toolchain used.
'sdk-version': sdk_verinfo.get('SDK_LATEST_VERSION', '<unknown>'),
'toolchain-url': sdk_verinfo.get('TC_PATH', '<unknown>'),
}
if len(config['boards']) == 1:
toolchains = toolchain.GetToolchainsForBoard(config['boards'][0],
buildroot=build_root)
metadata['toolchain-tuple'] = (
toolchain.FilterToolchains(toolchains, 'default', True).keys()
+ toolchain.FilterToolchains(toolchains, 'default',
False).keys())
logging.info('Metadata being written: %s', metadata)
self._run.attrs.metadata.UpdateWithDict(metadata)
# Update 'version' separately to avoid overwriting the existing
# entries in it (e.g. PFQ builders may have written the Chrome
# version to uprev).
logging.info("Metadata 'version' being written: %s", version)
self._run.attrs.metadata.UpdateKeyDictWithDict('version', version)
# Ensure that all boards and child config boards have a per-board
# metadata subdict.
for b in config['boards']:
self._run.attrs.metadata.UpdateBoardDictWithDict(b, {})
for cc in child_configs:
for b in cc['boards']:
self._run.attrs.metadata.UpdateBoardDictWithDict(b, {})
# Upload build metadata (and write it to database if necessary)
self.UploadMetadata(filename=constants.PARTIAL_METADATA_JSON)
# Write child-per-build and board-per-build rows to database
build_id, db = self._run.GetCIDBHandle()
if db:
# TODO(akeshet): replace this with a GetValue call once crbug.com/406522
# is resolved
per_board_dict = self._run.attrs.metadata.GetDict(
)['board-metadata']
for board, board_metadata in per_board_dict.items():
db.InsertBoardPerBuild(build_id, board)
if board_metadata:
db.UpdateBoardPerBuildMetadata(build_id, board,
board_metadata)
for child_config in self._run.attrs.metadata.GetValue(
'child-configs'):
db.InsertChildConfigPerBuild(build_id, child_config['name'])
# If this build has a master build, ensure that the master full_version
# is the same as this build's full_version. This is a sanity check to
# avoid bugs in master-slave logic.
master_id = self._run.attrs.metadata.GetDict().get(
'master_build_id')
if master_id is not None:
master_full_version = db.GetBuildStatus(
master_id)['full_version']
my_full_version = self._run.attrs.metadata.GetValue(
'version').get('full')
if master_full_version != my_full_version:
raise failures_lib.MasterSlaveVersionMismatchFailure(
'Master build id %s has full_version %s, while slave version is '
'%s.' %
(master_id, master_full_version, my_full_version))
# Abort previous hw test suites. This happens after reexecution as it
# requires chromite/third_party/swarming.client, which is not available
# untill after reexecution.
self._AbortPreviousHWTestSuites(version['milestone'])
def main(argv):
conf = cros_build_lib.LoadKeyValueFile(
os.path.join(constants.SOURCE_ROOT, constants.SDK_VERSION_FILE),
ignore_missing=True)
sdk_latest_version = conf.get('SDK_LATEST_VERSION', '<unknown>')
bootstrap_latest_version = conf.get('BOOTSTRAP_LATEST_VERSION', '<unknown>')
parser, commands = _CreateParser(sdk_latest_version, bootstrap_latest_version)
options = parser.parse_args(argv)
chroot_command = options.commands
# Some sanity checks first, before we ask for sudo credentials.
cros_build_lib.AssertOutsideChroot()
host = os.uname()[4]
if host != 'x86_64':
parser.error(
"cros_sdk is currently only supported on x86_64; you're running"
" %s. Please find a x86_64 machine." % (host,))
_ReportMissing(osutils.FindMissingBinaries(NEEDED_TOOLS))
if options.proxy_sim:
_ReportMissing(osutils.FindMissingBinaries(PROXY_NEEDED_TOOLS))
_ReExecuteIfNeeded([sys.argv[0]] + argv)
if options.ns_pid:
first_pid = namespaces.CreatePidNs()
else:
first_pid = None
# Expand out the aliases...
if options.replace:
options.delete = options.create = True
if options.bootstrap:
options.create = True
# If a command is not given, default to enter.
# pylint: disable=protected-access
# This _group_actions access sucks, but upstream decided to not include an
# alternative to optparse's option_list, and this is what they recommend.
options.enter |= not any(getattr(options, x.dest)
for x in commands._group_actions)
# pylint: enable=protected-access
options.enter |= bool(chroot_command)
if options.enter and options.delete and not options.create:
parser.error("Trying to enter the chroot when --delete "
"was specified makes no sense.")
# Finally, discern if we need to create the chroot.
chroot_exists = os.path.exists(options.chroot)
if options.create or options.enter:
# Only create if it's being wiped, or if it doesn't exist.
if not options.delete and chroot_exists:
options.create = False
else:
options.download = True
# Finally, flip create if necessary.
if options.enter:
options.create |= not chroot_exists
if not options.sdk_version:
sdk_version = (bootstrap_latest_version if options.bootstrap
else sdk_latest_version)
else:
sdk_version = options.sdk_version
if options.buildbot_log_version:
logging.PrintBuildbotStepText(sdk_version)
# Based on selections, determine the tarball to fetch.
if options.sdk_url:
urls = [options.sdk_url]
elif options.bootstrap:
urls = GetStage3Urls(sdk_version)
else:
urls = GetArchStageTarballs(sdk_version)
# Get URLs for the toolchains overlay, if one is to be used.
toolchains_overlay_urls = None
if not options.bootstrap:
toolchains = None
if options.toolchains:
toolchains = options.toolchains.split(',')
elif options.board:
toolchains = toolchain.GetToolchainsForBoard(options.board).keys()
if toolchains:
toolchains_overlay_urls = GetToolchainsOverlayUrls(sdk_version,
toolchains)
lock_path = os.path.dirname(options.chroot)
lock_path = os.path.join(
lock_path, '.%s_lock' % os.path.basename(options.chroot).lstrip('.'))
with cgroups.SimpleContainChildren('cros_sdk', pid=first_pid):
with locking.FileLock(lock_path, 'chroot lock') as lock:
toolchains_overlay_tarball = None
if options.proxy_sim:
_ProxySimSetup(options)
if options.delete and os.path.exists(options.chroot):
lock.write_lock()
DeleteChroot(options.chroot)
sdk_cache = os.path.join(options.cache_dir, 'sdks')
distfiles_cache = os.path.join(options.cache_dir, 'distfiles')
osutils.SafeMakedirsNonRoot(options.cache_dir)
for target in (sdk_cache, distfiles_cache):
src = os.path.join(constants.SOURCE_ROOT, os.path.basename(target))
if not os.path.exists(src):
osutils.SafeMakedirsNonRoot(target)
continue
lock.write_lock(
"Upgrade to %r needed but chroot is locked; please exit "
"all instances so this upgrade can finish." % src)
if not os.path.exists(src):
# Note that while waiting for the write lock, src may've vanished;
# it's a rare race during the upgrade process that's a byproduct
# of us avoiding taking a write lock to do the src check. If we
# took a write lock for that check, it would effectively limit
# all cros_sdk for a chroot to a single instance.
osutils.SafeMakedirsNonRoot(target)
elif not os.path.exists(target):
# Upgrade occurred, but a reversion, or something whacky
# occurred writing to the old location. Wipe and continue.
os.rename(src, target)
else:
# Upgrade occurred once already, but either a reversion or
# some before/after separate cros_sdk usage is at play.
# Wipe and continue.
osutils.RmDir(src)
if options.download:
lock.write_lock()
sdk_tarball = FetchRemoteTarballs(
sdk_cache, urls, 'stage3' if options.bootstrap else 'SDK')
if toolchains_overlay_urls:
toolchains_overlay_tarball = FetchRemoteTarballs(
sdk_cache, toolchains_overlay_urls, 'SDK toolchains overlay',
allow_none=True)
if options.create:
lock.write_lock()
CreateChroot(options.chroot, sdk_tarball, toolchains_overlay_tarball,
options.cache_dir,
nousepkg=(options.bootstrap or options.nousepkg))
if options.enter:
lock.read_lock()
EnterChroot(options.chroot, options.cache_dir, options.chrome_root,
options.chrome_root_mount, options.workspace,
chroot_command)
def _RunAndCompare(self, test_input):
result = cros_build_lib.LoadKeyValueFile(test_input)
self.assertEqual(self.expected, result)
def main(argv):
conf = cros_build_lib.LoadKeyValueFile(
os.path.join(constants.SOURCE_ROOT, constants.SDK_VERSION_FILE),
ignore_missing=True)
sdk_latest_version = conf.get('SDK_LATEST_VERSION', '<unknown>')
bootstrap_latest_version = conf.get('BOOTSTRAP_LATEST_VERSION', '<unknown>')
parser, commands = _CreateParser(sdk_latest_version, bootstrap_latest_version)
options = parser.parse_args(argv)
chroot_command = options.commands
# Some sanity checks first, before we ask for sudo credentials.
cros_build_lib.AssertOutsideChroot()
host = os.uname()[4]
if host != 'x86_64':
cros_build_lib.Die(
"cros_sdk is currently only supported on x86_64; you're running"
" %s. Please find a x86_64 machine." % (host,))
_ReportMissing(osutils.FindMissingBinaries(NEEDED_TOOLS))
if options.proxy_sim:
_ReportMissing(osutils.FindMissingBinaries(PROXY_NEEDED_TOOLS))
missing_image_tools = osutils.FindMissingBinaries(IMAGE_NEEDED_TOOLS)
if (sdk_latest_version == '<unknown>' or
bootstrap_latest_version == '<unknown>'):
cros_build_lib.Die(
'No SDK version was found. '
'Are you in a Chromium source tree instead of Chromium OS?\n\n'
'Please change to a directory inside your Chromium OS source tree\n'
'and retry. If you need to setup a Chromium OS source tree, see\n'
' http://www.chromium.org/chromium-os/developer-guide')
any_snapshot_operation = (options.snapshot_create or options.snapshot_restore
or options.snapshot_delete or options.snapshot_list)
if any_snapshot_operation and not options.use_image:
cros_build_lib.Die('Snapshot operations are not compatible with '
'--nouse-image.')
if (options.snapshot_delete and options.snapshot_delete ==
options.snapshot_restore):
parser.error('Cannot --snapshot_delete the same snapshot you are '
'restoring with --snapshot_restore.')
_ReExecuteIfNeeded([sys.argv[0]] + argv)
lock_path = os.path.dirname(options.chroot)
lock_path = os.path.join(
lock_path, '.%s_lock' % os.path.basename(options.chroot).lstrip('.'))
# Expand out the aliases...
if options.replace:
options.delete = options.create = True
if options.bootstrap:
options.create = True
# If a command is not given, default to enter.
# pylint: disable=protected-access
# This _group_actions access sucks, but upstream decided to not include an
# alternative to optparse's option_list, and this is what they recommend.
options.enter |= not any(getattr(options, x.dest)
for x in commands._group_actions)
# pylint: enable=protected-access
options.enter |= bool(chroot_command)
if (options.delete and not options.create and
(options.enter or any_snapshot_operation)):
parser.error("Trying to enter or snapshot the chroot when --delete "
"was specified makes no sense.")
if options.working_dir is not None and not os.path.isabs(options.working_dir):
options.working_dir = path_util.ToChrootPath(options.working_dir)
# Clean up potential leftovers from previous interrupted builds.
# TODO(bmgordon): Remove this at the end of 2017. That should be long enough
# to get rid of them all.
chroot_build_path = options.chroot + '.build'
if options.use_image and os.path.exists(chroot_build_path):
try:
with cgroups.SimpleContainChildren('cros_sdk'):
with locking.FileLock(lock_path, 'chroot lock') as lock:
logging.notice('Cleaning up leftover build directory %s',
chroot_build_path)
lock.write_lock()
osutils.UmountTree(chroot_build_path)
osutils.RmDir(chroot_build_path)
except cros_build_lib.RunCommandError as e:
logging.warning('Unable to remove %s: %s', chroot_build_path, e)
# Discern if we need to create the chroot.
chroot_ver_file = os.path.join(options.chroot, 'etc', 'cros_chroot_version')
chroot_exists = os.path.exists(chroot_ver_file)
if (options.use_image and not chroot_exists and not options.delete and
not missing_image_tools and
os.path.exists(_ImageFileForChroot(options.chroot))):
# Try to re-mount an existing image in case the user has rebooted.
with cgroups.SimpleContainChildren('cros_sdk'):
with locking.FileLock(lock_path, 'chroot lock') as lock:
logging.debug('Checking if existing chroot image can be mounted.')
lock.write_lock()
cros_sdk_lib.MountChroot(options.chroot, create=False)
chroot_exists = os.path.exists(chroot_ver_file)
if chroot_exists:
logging.notice('Mounted existing image %s on chroot',
_ImageFileForChroot(options.chroot))
if (options.create or options.enter or options.snapshot_create or
options.snapshot_restore):
# Only create if it's being wiped, or if it doesn't exist.
if not options.delete and chroot_exists:
options.create = False
else:
options.download = True
# Finally, flip create if necessary.
if options.enter or options.snapshot_create:
options.create |= not chroot_exists
# Anything that needs to manipulate the main chroot mount or communicate with
# LVM needs to be done here before we enter the new namespaces.
# If deleting, do it regardless of the use_image flag so that a
# previously-created loopback chroot can also be cleaned up.
# TODO(bmgordon): See if the DeleteChroot call below can be removed in
# favor of this block.
chroot_deleted = False
if options.delete:
with cgroups.SimpleContainChildren('cros_sdk'):
with locking.FileLock(lock_path, 'chroot lock') as lock:
lock.write_lock()
if missing_image_tools:
logging.notice('Unmounting chroot.')
osutils.UmountTree(options.chroot)
else:
logging.notice('Deleting chroot.')
cros_sdk_lib.CleanupChrootMount(options.chroot, delete_image=True)
osutils.RmDir(options.chroot, ignore_missing=True)
chroot_deleted = True
# Make sure the main chroot mount is visible. Contents will be filled in
# below if needed.
if options.create and options.use_image:
if missing_image_tools:
raise SystemExit(
'''The tool(s) %s were not found.
Please make sure the lvm2 and thin-provisioning-tools packages
are installed on your host.
Example(ubuntu):
sudo apt-get install lvm2 thin-provisioning-tools
If you want to run without lvm2, pass --nouse-image (chroot
snapshots will be unavailable).''' % ', '.join(missing_image_tools))
logging.debug('Making sure chroot image is mounted.')
with cgroups.SimpleContainChildren('cros_sdk'):
with locking.FileLock(lock_path, 'chroot lock') as lock:
lock.write_lock()
if not cros_sdk_lib.MountChroot(options.chroot, create=True):
cros_build_lib.Die('Unable to mount %s on chroot',
_ImageFileForChroot(options.chroot))
logging.notice('Mounted %s on chroot',
_ImageFileForChroot(options.chroot))
# Snapshot operations will always need the VG/LV, but other actions won't.
if any_snapshot_operation:
with cgroups.SimpleContainChildren('cros_sdk'):
with locking.FileLock(lock_path, 'chroot lock') as lock:
chroot_vg, chroot_lv = cros_sdk_lib.FindChrootMountSource(
options.chroot)
if not chroot_vg or not chroot_lv:
cros_build_lib.Die('Unable to find VG/LV for chroot %s',
options.chroot)
# Delete snapshot before creating a new one. This allows the user to
# throw out old state, create a new snapshot, and enter the chroot in a
# single call to cros_sdk. Since restore involves deleting, also do it
# before creating.
if options.snapshot_restore:
lock.write_lock()
valid_snapshots = ListChrootSnapshots(chroot_vg, chroot_lv)
if options.snapshot_restore not in valid_snapshots:
cros_build_lib.Die('%s is not a valid snapshot to restore to. '
'Valid snapshots: %s', options.snapshot_restore,
', '.join(valid_snapshots))
osutils.UmountTree(options.chroot)
if not RestoreChrootSnapshot(options.snapshot_restore, chroot_vg,
chroot_lv):
cros_build_lib.Die('Unable to restore chroot to snapshot.')
if not cros_sdk_lib.MountChroot(options.chroot, create=False):
cros_build_lib.Die('Unable to mount restored snapshot onto chroot.')
# Use a read lock for snapshot delete and create even though they modify
# the filesystem, because they don't modify the mounted chroot itself.
# The underlying LVM commands take their own locks, so conflicting
# concurrent operations here may crash cros_sdk, but won't corrupt the
# chroot image. This tradeoff seems worth it to allow snapshot
# operations on chroots that have a process inside.
if options.snapshot_delete:
lock.read_lock()
DeleteChrootSnapshot(options.snapshot_delete, chroot_vg, chroot_lv)
if options.snapshot_create:
lock.read_lock()
if not CreateChrootSnapshot(options.snapshot_create, chroot_vg,
chroot_lv):
cros_build_lib.Die('Unable to create snapshot.')
img_path = _ImageFileForChroot(options.chroot)
if (options.use_image and os.path.exists(options.chroot) and
os.path.exists(img_path)):
img_stat = os.stat(img_path)
img_used_bytes = img_stat.st_blocks * 512
mount_stat = os.statvfs(options.chroot)
mount_used_bytes = mount_stat.f_frsize * (mount_stat.f_blocks -
mount_stat.f_bfree)
extra_gbs = (img_used_bytes - mount_used_bytes) / 2**30
if extra_gbs > MAX_UNUSED_IMAGE_GBS:
logging.notice('%s is using %s GiB more than needed. Running '
'fstrim.', img_path, extra_gbs)
cmd = ['fstrim', options.chroot]
try:
cros_build_lib.RunCommand(cmd, print_cmd=False)
except cros_build_lib.RunCommandError as e:
logging.warning('Running fstrim failed. Consider running fstrim on '
'your chroot manually.\nError: %s', e)
# Enter a new set of namespaces. Everything after here cannot directly affect
# the hosts's mounts or alter LVM volumes.
namespaces.SimpleUnshare()
if options.ns_pid:
first_pid = namespaces.CreatePidNs()
else:
first_pid = None
if options.snapshot_list:
for snap in ListChrootSnapshots(chroot_vg, chroot_lv):
print(snap)
sys.exit(0)
if not options.sdk_version:
sdk_version = (bootstrap_latest_version if options.bootstrap
else sdk_latest_version)
else:
sdk_version = options.sdk_version
if options.buildbot_log_version:
logging.PrintBuildbotStepText(sdk_version)
# Based on selections, determine the tarball to fetch.
if options.download:
if options.sdk_url:
urls = [options.sdk_url]
elif options.bootstrap:
urls = GetStage3Urls(sdk_version)
else:
urls = GetArchStageTarballs(sdk_version)
# Get URLs for the toolchains overlay, if one is to be used.
toolchains_overlay_urls = None
if not options.bootstrap:
toolchains = None
if options.toolchains:
toolchains = options.toolchains.split(',')
elif options.board:
toolchains = toolchain.GetToolchainsForBoard(options.board).keys()
if toolchains:
toolchains_overlay_urls = GetToolchainsOverlayUrls(sdk_version,
toolchains)
with cgroups.SimpleContainChildren('cros_sdk', pid=first_pid):
with locking.FileLock(lock_path, 'chroot lock') as lock:
toolchains_overlay_tarball = None
if options.proxy_sim:
_ProxySimSetup(options)
if (options.delete and not chroot_deleted and
(os.path.exists(options.chroot) or
os.path.exists(_ImageFileForChroot(options.chroot)))):
lock.write_lock()
DeleteChroot(options.chroot)
sdk_cache = os.path.join(options.cache_dir, 'sdks')
distfiles_cache = os.path.join(options.cache_dir, 'distfiles')
osutils.SafeMakedirsNonRoot(options.cache_dir)
for target in (sdk_cache, distfiles_cache):
src = os.path.join(constants.SOURCE_ROOT, os.path.basename(target))
if not os.path.exists(src):
osutils.SafeMakedirsNonRoot(target)
continue
lock.write_lock(
"Upgrade to %r needed but chroot is locked; please exit "
"all instances so this upgrade can finish." % src)
if not os.path.exists(src):
# Note that while waiting for the write lock, src may've vanished;
# it's a rare race during the upgrade process that's a byproduct
# of us avoiding taking a write lock to do the src check. If we
# took a write lock for that check, it would effectively limit
# all cros_sdk for a chroot to a single instance.
osutils.SafeMakedirsNonRoot(target)
elif not os.path.exists(target):
# Upgrade occurred, but a reversion, or something whacky
# occurred writing to the old location. Wipe and continue.
os.rename(src, target)
else:
# Upgrade occurred once already, but either a reversion or
# some before/after separate cros_sdk usage is at play.
# Wipe and continue.
osutils.RmDir(src)
if options.download:
lock.write_lock()
sdk_tarball = FetchRemoteTarballs(
sdk_cache, urls, 'stage3' if options.bootstrap else 'SDK')
if toolchains_overlay_urls:
toolchains_overlay_tarball = FetchRemoteTarballs(
sdk_cache, toolchains_overlay_urls, 'SDK toolchains overlay',
allow_none=True)
if options.create:
lock.write_lock()
CreateChroot(options.chroot, sdk_tarball, toolchains_overlay_tarball,
options.cache_dir,
nousepkg=(options.bootstrap or options.nousepkg))
if options.enter:
lock.read_lock()
EnterChroot(options.chroot, options.cache_dir, options.chrome_root,
options.chrome_root_mount, options.workspace,
options.goma_dir, options.goma_client_json,
options.working_dir, chroot_command)
