def testSharedLocks(self):
"""Test lock conflict for blocking locks."""
# Intial lock is NOT shared.
with locking.FileLock(self.lock_file).write_lock():
self._HelperWithProcess(expected=LOCK_NOT_ACQUIRED, shared=True)
# Intial lock IS shared.
with locking.FileLock(self.lock_file).read_lock():
self._HelperWithProcess(expected=LOCK_ACQUIRED, shared=True)
self._HelperWithProcess(expected=LOCK_NOT_ACQUIRED, shared=False)
def _CacheFileLock(self, cache_file, blocking=False, shared=False):
"""Acquire a lock on a file in the cache.
A shared lock will ensure no other processes are modifying the file, but
getting it does not ensure that the file in question actually exists.
An exclusive lock is required to modify a cache file, this usually means
downloading it.
A shared _PurgeLock should be held before trying to acquire any type
of cache file lock.
Args:
cache_file: The full path of file in cache to lock.
blocking: Block until the lock is available?
shared: Get a shared lock, or an exclusive lock?
Returns:
Locking.FileLock (acquired)
"""
lock_file = os.path.join(self._lock_dir, os.path.basename(cache_file))
lock = locking.FileLock(lock_file,
locktype=locking.FLOCK,
blocking=blocking)
return lock.lock(shared)
def SetCachedField(self, field, value):
"""Sets |field| to |value| in the sysroot cache file.
Access to the cache is thread-safe as long as we access it through this
methods or the bash helper in common.sh.
Args:
field: name of the field.
value: value to set. If |value| is None, the field is unset.
"""
# TODO(bsimonnet): add support for values with quotes and newlines.
# crbug.com/476764.
for symbol in '\n`$"\\':
if value and symbol in value:
raise ValueError(
'Cannot use \\n, `, $, \\ or " in cached value.')
with locking.FileLock(self._cache_file_lock,
locktype=locking.FLOCK,
world_writable=True).write_lock():
lines = []
if os.path.exists(self._cache_file):
lines = osutils.ReadFile(self._cache_file).splitlines()
# Remove the old value for field if it exists.
lines = [l for l in lines if not l.startswith(field + '=')]
if value is not None:
lines.append('%s="%s"' % (field, value))
osutils.WriteFile(self._cache_file, '\n'.join(lines), sudo=True)
def __init__(self, store_file,
file_lock_timeout_seconds=_FILE_LOCK_TIMEOUT_SECONDS):
"""
@param store_file: Absolute path to the backing file to use.
@param info: Optional HostInfo to initialize the store. When not None,
any data in store_file will be overwritten.
@param file_lock_timeout_seconds: Timeout for aborting the attempt to
lock the backing file in seconds. Set this to <= 0 to request
just a single attempt.
"""
super(FileStore, self).__init__()
self._store_file = store_file
self._lock_path = '%s.lock' % store_file
if file_lock_timeout_seconds <= 0:
self._lock_max_retry = 0
self._lock_sleep = 0
else:
# A total of 3 attempts at times (0 + sleep + 2*sleep).
self._lock_max_retry = 2
self._lock_sleep = file_lock_timeout_seconds / 3.0
self._lock = locking.FileLock(
self._lock_path,
locktype=locking.FLOCK,
description='Locking FileStore to read/write HostInfo.',
blocking=False)
def _LockForKey(self, key, suffix='.lock'):
"""Returns an unacquired lock associated with a key."""
key_path = self._GetKeyPath(key)
osutils.SafeMakedirsNonRoot(os.path.dirname(key_path))
lock_path = os.path.join(self._cache_dir, os.path.dirname(key_path),
os.path.basename(key_path) + suffix)
return locking.FileLock(lock_path)
def CheckAndGetCIDBCreds(force_update=False, folder=None):
"""Check if CIDB creds exist, download creds if necessary."""
cache_dir = path_util.GetCacheDir()
dir_name = folder if folder is not None else 'cidb_creds'
cidb_dir = os.path.join(cache_dir, dir_name)
cidb_dir_lock = cidb_dir + '.lock'
with locking.FileLock(cidb_dir_lock).write_lock():
if os.path.exists(cidb_dir):
if force_update:
shutil.rmtree(cidb_dir, ignore_errors=True)
logging.debug('Force updating CIDB creds. Deleted %s.',
cidb_dir)
else:
logging.debug('Using cached credentials %s', cidb_dir)
return cidb_dir
os.mkdir(cidb_dir)
try:
GetCIDBCreds(cidb_dir)
return cidb_dir
except Exception as e:
if isinstance(e, gs.GSCommandError):
logging.warning(
'Please check if the GS credentials is configured '
'correctly. Please note the permissions to fetch '
'these credentials are for Googlers only,')
logging.error('Failed to get CIDB credentials. Deleting %s',
cidb_dir)
shutil.rmtree(cidb_dir, ignore_errors=True)
raise
def testNonBlockingConflicts(self):
"""Test that we get a lock conflict for non-blocking locks."""
with locking.FileLock(self.lock_file).write_lock():
self._HelperWithProcess(expected=LOCK_NOT_ACQUIRED)
self._HelperWithProcess(expected=LOCK_NOT_ACQUIRED, shared=True)
# Can grab it after it's released.
self._HelperWithProcess(expected=LOCK_ACQUIRED)
def testContextMgr(self):
"""Make sure we behave properly with 'with'."""
# Create an instance, and use it in a with.
prelock = locking.FileLock(self.lock_file)
self._HelperWithProcess(expected=LOCK_ACQUIRED)
with prelock.write_lock() as lock:
# Assert the instance didn't change.
self.assertIs(prelock, lock)
self._HelperWithProcess(expected=LOCK_NOT_ACQUIRED)
self._HelperWithProcess(expected=LOCK_ACQUIRED)
# Construct the instance in the with expression.
with locking.FileLock(self.lock_file).write_lock() as lock:
self.assertIsInstance(lock, locking.FileLock)
self._HelperWithProcess(expected=LOCK_NOT_ACQUIRED)
self._HelperWithProcess(expected=LOCK_ACQUIRED)
def testDoubleLockWithLockf(self):
"""Tests that double locks don't block with lockf."""
lock1 = locking.FileLock(self.lock_file,
blocking=False,
locktype=locking.LOCKF)
lock2 = locking.FileLock(self.lock_file,
blocking=False,
locktype=locking.LOCKF)
with lock1.write_lock():
self.assertTrue(lock1.IsLocked())
self.assertFalse(lock2.IsLocked())
# With lockf, we can lock the same file twice in the same process.
with lock2.write_lock():
self.assertTrue(lock1.IsLocked())
self.assertTrue(lock2.IsLocked())
self.assertFalse(lock1.IsLocked())
self.assertFalse(lock2.IsLocked())
def RunCleanupCommands(project, cwd):
with locking.FileLock(lock_path,
verbose=False).read_lock() as lock:
# Calculate where the git repository is stored.
relpath = os.path.relpath(cwd, self.directory)
projects_dir = os.path.join(self.directory, '.repo',
'projects')
project_objects_dir = os.path.join(self.directory, '.repo',
'project-objects')
repo_git_store = '%s.git' % os.path.join(projects_dir, relpath)
repo_obj_store = '%s.git' % os.path.join(
project_objects_dir, project)
try:
if os.path.isdir(cwd):
git.CleanAndDetachHead(cwd)
if os.path.isdir(repo_git_store):
git.GarbageCollection(repo_git_store,
prune_all=prune_all)
except cros_build_lib.RunCommandError as e:
result = e.result
logging.PrintBuildbotStepWarnings()
logging.warning('\n%s', result.error)
# If there's no repository corruption, just delete the index.
corrupted = git.IsGitRepositoryCorrupted(repo_git_store)
lock.write_lock()
logging.warning('Deleting %s because %s failed', cwd,
result.cmd)
osutils.RmDir(cwd, ignore_missing=True, sudo=True)
if corrupted:
# Looks like the object dir is corrupted. Delete it.
deleted_objdirs.set()
for store in (repo_git_store, repo_obj_store):
logging.warning('Deleting %s as well', store)
osutils.RmDir(store, ignore_missing=True)
# TODO: Make the deletions below smarter. Look to see what exists,
# instead of just deleting things we think might be there.
# Delete all branches created by cbuildbot.
if os.path.isdir(repo_git_store):
cmd = ['branch', '-D'] + list(constants.CREATED_BRANCHES)
# Ignore errors, since we delete branches without checking existence.
git.RunGit(repo_git_store, cmd, error_code_ok=True)
if os.path.isdir(cwd):
# Above we deleted refs/heads/<branch> for each created branch, now we
# need to delete the bare ref <branch> if it was created somehow.
for ref in constants.CREATED_BRANCHES:
# Ignore errors, since we delete branches without checking
# existence.
git.RunGit(cwd, ['update-ref', '-d', ref],
error_code_ok=True)
def testAcquireBeforeWith(self):
"""Sometimes you want to grab a lock and then return it into 'with'."""
lock = locking.FileLock(self.lock_file, blocking=False)
lock.write_lock()
self._HelperWithProcess(expected=LOCK_NOT_ACQUIRED)
with lock:
self._HelperWithProcess(expected=LOCK_NOT_ACQUIRED)
self._HelperWithProcess(expected=LOCK_ACQUIRED)
def _HelperInsideProcess(self, blocking, shared, locktype=locking.LOCKF):
"""Helper method that runs a basic test with/without blocking."""
try:
lock = locking.FileLock(self.lock_file,
blocking=blocking,
locktype=locktype)
with lock.lock(shared):
pass
sys.exit(LOCK_ACQUIRED)
except locking.LockNotAcquiredError:
sys.exit(LOCK_NOT_ACQUIRED)
def testDoubleLockTimeoutWithFlock(self):
"""Tests that double locking the same lock times out."""
lock1 = locking.FileLock(self.lock_file,
blocking=True,
locktype=locking.FLOCK,
blocking_timeout=1,
verbose=True)
lock2 = locking.FileLock(self.lock_file,
blocking=True,
locktype=locking.FLOCK,
blocking_timeout=1,
verbose=True)
# Verify that if we grab the first lock, trying to use lock2
# (on the same file) will throw an exception.
with lock1.write_lock():
self.assertTrue(lock1.IsLocked())
self.assertFalse(lock2.IsLocked())
self.assertRaises(timeout_util.TimeoutError, lock2.write_lock)
self.assertTrue(lock1.IsLocked())
def test_commit_blocks_for_locked_file(self):
"""Commit blocks when the backing file is locked.
This is a greybox test. We artificially lock the backing file.
This test intentionally uses a real locking.FileLock to ensure that
locking API is used correctly.
"""
# file_lock_timeout of 0 forces no retries (speeds up the test)
store = file_store.FileStore(self._store_file,
file_lock_timeout_seconds=0)
file_lock = locking.FileLock(store._lock_path, locktype=locking.FLOCK)
with file_lock.lock(), self.assertRaises(host_info.StoreError):
store.commit(host_info.HostInfo())
def testBlockingConflicts(self):
"""Test lock conflict for blocking locks."""
# Intial lock is blocking, exclusive.
with locking.FileLock(self.lock_file, blocking=True).write_lock():
self._HelperWithProcess(expected=LOCK_NOT_ACQUIRED, blocking=False)
p = self._HelperStartProcess(blocking=True, shared=False)
# when the with clause exits, p should unblock and get the lock, setting
# its exit code to sucess now.
p.join()
self.assertEqual(p.exitcode, LOCK_ACQUIRED)
# Intial lock is NON blocking.
with locking.FileLock(self.lock_file, blocking=False).write_lock():
self._HelperWithProcess(expected=LOCK_NOT_ACQUIRED)
p = self._HelperStartProcess(blocking=True, shared=False)
# when the with clause exits, p should unblock and get the lock, setting
# it's exit code to sucess now.
p.join()
self.assertEqual(p.exitcode, LOCK_ACQUIRED)
# Intial lock is shared, blocking lock is exclusive.
with locking.FileLock(self.lock_file, blocking=False).read_lock():
self._HelperWithProcess(expected=LOCK_NOT_ACQUIRED)
self._HelperWithProcess(expected=LOCK_ACQUIRED, shared=True)
p = self._HelperStartProcess(blocking=True, shared=False)
q = self._HelperStartProcess(blocking=True, shared=False)
# when the with clause exits, p should unblock and get the lock, setting
# it's exit code to sucess now.
p.join()
self.assertEqual(p.exitcode, LOCK_ACQUIRED)
q.join()
self.assertEqual(p.exitcode, LOCK_ACQUIRED)
def testDoubleLockWithFlock(self):
"""Tests that double locks do block with flock."""
lock1 = locking.FileLock(self.lock_file,
blocking=False,
locktype=locking.FLOCK)
lock2 = locking.FileLock(self.lock_file,
blocking=False,
locktype=locking.FLOCK)
with lock1.write_lock():
self.assertTrue(lock1.IsLocked())
self.assertFalse(lock2.IsLocked())
self.assertRaises(locking.LockNotAcquiredError, lock2.write_lock)
self.assertTrue(lock1.IsLocked())
self.assertFalse(lock2.IsLocked())
self.assertFalse(lock1.IsLocked())
self.assertFalse(lock2.IsLocked())
lock2.unlock()
self.assertFalse(lock1.IsLocked())
self.assertFalse(lock2.IsLocked())
def GetCachedField(self, field):
"""Returns the value of |field| in the sysroot cache file.
Access to the cache is thread-safe as long as we access it through this
methods or the bash helper in common.sh.
Args:
field: name of the field.
"""
if not os.path.exists(self._cache_file):
return None
with locking.FileLock(
self._cache_file_lock, locktype=locking.FLOCK,
world_writable=True).read_lock():
return osutils.SourceEnvironment(self._cache_file, [field]).get(field)
def _PurgeLock(self, blocking=False, shared=False):
"""Acquire a lock on the cache as a whole.
An exclusive lock proves nobody else will modify anything, and nobody
else will hold any _CacheFileLocks. A shared lock is required before
getting any kind of _CacheFileLock.
Args:
blocking: Block until the lock is available?
shared: Get a shared lock, or an exclusive lock?
Returns:
Locking.FileLock (acquired)
"""
lock_file = os.path.join(self._cache_dir, self._CACHE_LOCK)
lock = locking.FileLock(lock_file, locktype=locking.FLOCK,
blocking=blocking)
return lock.lock(shared)
def _EnsureMounted(mnt, args):
if _FileContains('/proc/mounts', [mnt]):
return True
# Grab a lock so in the off chance we have multiple programs (like two
# cros_sdk launched in parallel) running this init logic, we don't end
# up mounting multiple times.
lock_path = '/tmp/.chromite.cgroups.lock'
with locking.FileLock(lock_path, 'cgroup lock') as lock:
lock.write_lock()
if _FileContains('/proc/mounts', [mnt]):
return True
# Not all distros mount cgroup_root to sysfs.
osutils.SafeMakedirs(mnt, sudo=True)
cros_build_lib.SudoRunCommand(['mount'] + args + [mnt], print_cmd=False)
return True
def _HelperSingleLockTest(self, blocking, shared, locktype):
"""Helper method that runs a basic test with/without blocking/sharing."""
self.assertNotExists(self.lock_file)
lock = locking.FileLock(self.lock_file,
blocking=blocking,
locktype=locktype)
self.assertFalse(lock.IsLocked())
lock.write_lock()
self.assertTrue(lock.IsLocked())
self.assertExists(self.lock_file)
# Acquiring the lock again should be safe.
lock.lock(shared)
self.assertTrue(lock.IsLocked())
lock.close()
self.assertFalse(lock.IsLocked())
osutils.SafeUnlink(self.lock_file)
def _EnsureMounted(mnt, args):
for mtab in osutils.IterateMountPoints():
if mtab.destination == mnt:
return True
# Grab a lock so in the off chance we have multiple programs (like two
# cros_sdk launched in parallel) running this init logic, we don't end
# up mounting multiple times.
lock_path = '/tmp/.chromite.cgroups.lock'
with locking.FileLock(lock_path, 'cgroup lock') as lock:
lock.write_lock()
for mtab in osutils.IterateMountPoints():
if mtab.destination == mnt:
return True
# Not all distros mount cgroup_root to sysfs.
osutils.SafeMakedirs(mnt, sudo=True)
cros_build_lib.sudo_run(['mount'] + args + [mnt],
print_cmd=False)
return True
def append_metrics_log(timestamp, name, op, arg=None):
"""Handle appending a list of terms to the metrics log.
If the environment does not specify a metrics log, then skip silently.
Args:
timestamp: A millisecond epoch timestamp.
name: A period-separated string describing the event.
op: One of the OP_* values, determining which type of event this is.
arg: An accessory value for use based on the related |op|.
"""
metrics_log = os.environ.get(UTILS_METRICS_LOG_ENVVAR)
terms = [timestamp, name.replace('|', '_'), op]
if arg is not None:
terms.append(arg)
# Format the actual line to log.
line = '|'.join(str(x) for x in terms)
if metrics_log:
with locking.FileLock(metrics_log).write_lock():
with open(metrics_log, 'a') as f:
f.write('%s\n' % line)
def main(argv):
"""Load generator for a devserver."""
parser = get_parser()
options = parser.parse_args(argv)
# Parse devserver.
if options.server:
if re.match(r'^https?://', options.server):
server = options.server
else:
server = 'http://%s/' % options.server
ds = dev_server.ImageServer(server)
else:
parser.print_usage()
logging.error('Must specify devserver')
sys.exit(1)
# Parse config file and determine master list of duts and their board type,
# filtering by board type if specified.
duts = {}
if options.config:
with open(options.config, 'r') as f:
config = json.load(f)
boards = (options.boards.split(',')
if options.boards else config.keys())
duts_specified = (set(options.duts.split(','))
if options.duts else None)
for board in boards:
duts.update({
dut: board
for dut in config[board]['duts']
if duts_specified is None or dut in duts_specified
})
logging.info('Config file %s: %d boards, %d duts', options.config,
len(boards), len(duts))
else:
parser.print_usage()
logging.error('Must specify config file')
sys.exit(1)
if options.ping:
logging.info('Performing ping tests')
duts_alive = {}
for dut, board in duts.items():
if ping_dut(dut):
duts_alive[dut] = board
else:
logging.error(
'Ignoring DUT %s (%s) for failing initial '
'ping check', dut, board)
duts = duts_alive
logging.info('After ping tests: %d boards, %d duts', len(boards),
len(duts))
# Set up the test runner and stage all the builds.
outputlog = open(options.outputlog, 'a') if options.outputlog else None
runner = Runner(ds,
duts,
config,
simultaneous=options.simultaneous,
total=options.total,
outputlog=outputlog,
ping=options.ping,
blacklist_consecutive=options.blacklist_consecutive,
blacklist_success=options.blacklist_success,
blacklist_total=options.blacklist_total,
dryrun=options.dryrun)
if options.stage:
runner.stage_all()
# Run all the provisions.
with locking.FileLock(options.config, blocking=True).lock():
completed = runner.loop()
logging.info('%s in %s', 'Completed' if completed else 'Interrupted',
runner.elapsed())
# Write all entries as JSON.
entries = runner.get_completed_entries()
if options.output:
with open(options.output, 'w') as f:
dump_entries_as_json(entries, f)
else:
dump_entries_as_json(entries, sys.stdout)
logging.info(
'Summary: %s',
dict(
collections.Counter(
[e['status'] for e in entries if e['name'] != 'Runner'])))
# List blacklisted DUTs.
if runner.dut_blacklist:
logging.warn('Blacklisted DUTs:')
for host_name in runner.dut_blacklist:
logging.warn(' %s', host_name)
def main(argv):
conf = cros_build_lib.LoadKeyValueFile(
os.path.join(constants.SOURCE_ROOT, constants.SDK_VERSION_FILE),
ignore_missing=True)
sdk_latest_version = conf.get('SDK_LATEST_VERSION', '<unknown>')
bootstrap_latest_version = conf.get('BOOTSTRAP_LATEST_VERSION', '<unknown>')
parser, commands = _CreateParser(sdk_latest_version, bootstrap_latest_version)
options = parser.parse_args(argv)
chroot_command = options.commands
# Some sanity checks first, before we ask for sudo credentials.
cros_build_lib.AssertOutsideChroot()
host = os.uname()[4]
if host != 'x86_64':
parser.error(
"cros_sdk is currently only supported on x86_64; you're running"
" %s. Please find a x86_64 machine." % (host,))
_ReportMissing(osutils.FindMissingBinaries(NEEDED_TOOLS))
if options.proxy_sim:
_ReportMissing(osutils.FindMissingBinaries(PROXY_NEEDED_TOOLS))
_ReExecuteIfNeeded([sys.argv[0]] + argv)
if options.ns_pid:
first_pid = namespaces.CreatePidNs()
else:
first_pid = None
# Expand out the aliases...
if options.replace:
options.delete = options.create = True
if options.bootstrap:
options.create = True
# If a command is not given, default to enter.
# pylint: disable=protected-access
# This _group_actions access sucks, but upstream decided to not include an
# alternative to optparse's option_list, and this is what they recommend.
options.enter |= not any(getattr(options, x.dest)
for x in commands._group_actions)
# pylint: enable=protected-access
options.enter |= bool(chroot_command)
if options.enter and options.delete and not options.create:
parser.error("Trying to enter the chroot when --delete "
"was specified makes no sense.")
# Finally, discern if we need to create the chroot.
chroot_exists = os.path.exists(options.chroot)
if options.create or options.enter:
# Only create if it's being wiped, or if it doesn't exist.
if not options.delete and chroot_exists:
options.create = False
else:
options.download = True
# Finally, flip create if necessary.
if options.enter:
options.create |= not chroot_exists
if not options.sdk_version:
sdk_version = (bootstrap_latest_version if options.bootstrap
else sdk_latest_version)
else:
sdk_version = options.sdk_version
if options.buildbot_log_version:
logging.PrintBuildbotStepText(sdk_version)
# Based on selections, determine the tarball to fetch.
if options.sdk_url:
urls = [options.sdk_url]
elif options.bootstrap:
urls = GetStage3Urls(sdk_version)
else:
urls = GetArchStageTarballs(sdk_version)
# Get URLs for the toolchains overlay, if one is to be used.
toolchains_overlay_urls = None
if not options.bootstrap:
toolchains = None
if options.toolchains:
toolchains = options.toolchains.split(',')
elif options.board:
toolchains = toolchain.GetToolchainsForBoard(options.board).keys()
if toolchains:
toolchains_overlay_urls = GetToolchainsOverlayUrls(sdk_version,
toolchains)
lock_path = os.path.dirname(options.chroot)
lock_path = os.path.join(
lock_path, '.%s_lock' % os.path.basename(options.chroot).lstrip('.'))
with cgroups.SimpleContainChildren('cros_sdk', pid=first_pid):
with locking.FileLock(lock_path, 'chroot lock') as lock:
toolchains_overlay_tarball = None
if options.proxy_sim:
_ProxySimSetup(options)
if options.delete and os.path.exists(options.chroot):
lock.write_lock()
DeleteChroot(options.chroot)
sdk_cache = os.path.join(options.cache_dir, 'sdks')
distfiles_cache = os.path.join(options.cache_dir, 'distfiles')
osutils.SafeMakedirsNonRoot(options.cache_dir)
for target in (sdk_cache, distfiles_cache):
src = os.path.join(constants.SOURCE_ROOT, os.path.basename(target))
if not os.path.exists(src):
osutils.SafeMakedirsNonRoot(target)
continue
lock.write_lock(
"Upgrade to %r needed but chroot is locked; please exit "
"all instances so this upgrade can finish." % src)
if not os.path.exists(src):
# Note that while waiting for the write lock, src may've vanished;
# it's a rare race during the upgrade process that's a byproduct
# of us avoiding taking a write lock to do the src check. If we
# took a write lock for that check, it would effectively limit
# all cros_sdk for a chroot to a single instance.
osutils.SafeMakedirsNonRoot(target)
elif not os.path.exists(target):
# Upgrade occurred, but a reversion, or something whacky
# occurred writing to the old location. Wipe and continue.
os.rename(src, target)
else:
# Upgrade occurred once already, but either a reversion or
# some before/after separate cros_sdk usage is at play.
# Wipe and continue.
osutils.RmDir(src)
if options.download:
lock.write_lock()
sdk_tarball = FetchRemoteTarballs(
sdk_cache, urls, 'stage3' if options.bootstrap else 'SDK')
if toolchains_overlay_urls:
toolchains_overlay_tarball = FetchRemoteTarballs(
sdk_cache, toolchains_overlay_urls, 'SDK toolchains overlay',
allow_none=True)
if options.create:
lock.write_lock()
CreateChroot(options.chroot, sdk_tarball, toolchains_overlay_tarball,
options.cache_dir,
nousepkg=(options.bootstrap or options.nousepkg))
if options.enter:
lock.read_lock()
EnterChroot(options.chroot, options.cache_dir, options.chrome_root,
options.chrome_root_mount, options.workspace,
chroot_command)
def main():
test_helper.SetupCommonLoggingFormat()
parser = optparse.OptionParser()
# Options related to which payloads to generate.
parser.add_option('--basic_suite',
default=False,
action='store_true',
help='Prepare to run the basic au test suite.')
parser.add_option('--full_suite',
default=False,
action='store_true',
help='Prepare to run the full au test suite.')
parser.add_option('--full_payload',
default=False,
action='store_true',
help='Generate the full update payload and store it in '
'the nplus1 archive dir.')
parser.add_option(
'--nplus1',
default=False,
action='store_true',
help='Produce nplus1 updates for testing in lab and store '
'them in the nplus1 archive dir.')
parser.add_option('--nplus1_archive_dir',
default=None,
help='Archive nplus1 updates into this directory.')
# Options related to how to generate test payloads for the test harness.
parser.add_option('--novm',
default=True,
action='store_false',
dest='vm',
help='Test Harness payloads will not be tested in a VM.')
parser.add_option('--private_key',
help='Private key to sign payloads for test harness.')
parser.add_option('--public_key',
help='Public key to verify payloads for test harness.')
# Options related to the images to test.
parser.add_option('--board', help='Board used for the images.')
parser.add_option('--base', help='Image we want to test updates from.')
parser.add_option(
'--base_latest_from_dir',
help='Ignore the base '
'option and use the latest image from the specified '
'directory as the base image. If none exists, default to '
'target image.')
parser.add_option('--target', help='Image we want to test updates to.')
# Miscellaneous options.
parser.add_option('--jobs',
default=test_helper.CalculateDefaultJobs(),
type=int,
help='Number of payloads to generate in parallel.')
options = parser.parse_args()[0]
CheckOptions(parser, options)
if options.nplus1_archive_dir and not os.path.exists(
options.nplus1_archive_dir):
os.makedirs(options.nplus1_archive_dir)
# Don't allow this code to be run more than once at a time.
lock_path = os.path.join(os.path.dirname(__file__), '.lock_file')
with locking.FileLock(lock_path, 'generate payloads lock') as lock:
lock.write_lock()
with sudo.SudoKeepAlive():
generator = UpdatePayloadGenerator(options)
generator.GenerateImagesForTesting()
generator.GeneratePayloadRequirements()
cache = generator.GeneratePayloads()
generator.DumpCacheToDisk(cache)
def main(argv):
usage = """usage: %prog [options] [VAR1=val1 .. VARn=valn -- args]
This script is used for manipulating local chroot environments; creating,
deleting, downloading, etc. If given --enter (or no args), it defaults
to an interactive bash shell within the chroot.
If given args those are passed to the chroot environment, and executed."""
conf = cros_build_lib.LoadKeyValueFile(os.path.join(
constants.SOURCE_ROOT, constants.SDK_VERSION_FILE),
ignore_missing=True)
sdk_latest_version = conf.get('COREOS_SDK_VERSION', '<unknown>')
parser = commandline.OptionParser(usage=usage, caching=True)
commands = parser.add_option_group("Commands")
commands.add_option('--enter',
action='store_true',
default=False,
help='Enter the SDK chroot. Implies --create.')
commands.add_option(
'--create',
action='store_true',
default=False,
help='Create the chroot only if it does not already exist. '
'Implies --download.')
commands.add_option(
'--bootstrap',
action='store_true',
default=False,
help='Build everything from scratch, including the sdk. '
'Use this only if you need to validate a change '
'that affects SDK creation itself (toolchain and '
'build are typically the only folk who need this). '
'Note this will quite heavily slow down the build. '
'This option implies --create --nousepkg.')
commands.add_option(
'-r',
'--replace',
action='store_true',
default=False,
help='Replace an existing SDK chroot. Basically an alias '
'for --delete --create.')
commands.add_option('--delete',
action='store_true',
default=False,
help='Delete the current SDK chroot if it exists.')
commands.add_option('--download',
action='store_true',
default=False,
help='Download the sdk.')
# Global options:
default_chroot = os.path.join(constants.SOURCE_ROOT,
constants.DEFAULT_CHROOT_DIR)
parser.add_option('--chroot',
dest='chroot',
default=default_chroot,
type='path',
help=('SDK chroot dir name [%s]' %
constants.DEFAULT_CHROOT_DIR))
parser.add_option('--chrome_root',
default=None,
type='path',
help='Mount this chrome root into the SDK chroot')
parser.add_option('--chrome_root_mount',
default=None,
type='path',
help='Mount chrome into this path inside SDK chroot')
parser.add_option(
'--nousepkg',
action='store_true',
default=False,
help='Do not use binary packages when creating a chroot.')
parser.add_option('--nogetbinpkg',
action='store_true',
default=False,
help='Do not fetch remote binary packages.')
parser.add_option('-u',
'--url',
dest='sdk_url',
default=None,
help=('''Use sdk tarball located at this url.
Use file:// for local files.'''))
parser.add_option('--sdk-version',
default=sdk_latest_version,
help='Use this sdk version. Current is %default.')
options, chroot_command = parser.parse_args(argv)
# Some sanity checks first, before we ask for sudo credentials.
cros_build_lib.AssertOutsideChroot()
host = os.uname()[4]
if host != 'x86_64':
parser.error(
"cros_sdk is currently only supported on x86_64; you're running"
" %s. Please find a x86_64 machine." % (host, ))
missing = osutils.FindMissingBinaries(NEEDED_TOOLS)
if missing:
parser.error(
('The tool(s) %s were not found.\n'
'Please install the appropriate package in your host.\n'
'Example(ubuntu):\n'
' sudo apt-get install <packagename>' % (', '.join(missing))))
_ReExecuteIfNeeded([sys.argv[0]] + argv)
# Expand out the aliases...
if options.replace:
options.delete = options.create = True
if options.bootstrap:
options.create = True
# If a command is not given, default to enter.
options.enter |= not any(
getattr(options, x.dest) for x in commands.option_list)
options.enter |= bool(chroot_command)
if options.enter and options.delete and not options.create:
parser.error("Trying to enter the chroot when --delete "
"was specified makes no sense.")
# Finally, discern if we need to create the chroot.
chroot_exists = os.path.exists(options.chroot)
if options.create or options.enter:
# Only create if it's being wiped, or if it doesn't exist.
if not options.delete and chroot_exists:
options.create = False
else:
options.download = True
# Finally, flip create if necessary.
if options.enter:
options.create |= not chroot_exists
# Based on selections, fetch the tarball.
if options.sdk_url:
urls = [options.sdk_url]
else:
urls = GetArchStageTarballs(options.sdk_version)
lock_path = os.path.dirname(options.chroot)
lock_path = os.path.join(lock_path,
'.%s_lock' % os.path.basename(options.chroot))
with cgroups.SimpleContainChildren('cros_sdk'):
with locking.FileLock(lock_path, 'chroot lock') as lock:
if options.delete and os.path.exists(options.chroot):
lock.write_lock()
DeleteChroot(options.chroot)
sdk_cache = os.path.join(options.cache_dir, 'sdks')
distfiles_cache = os.path.join(options.cache_dir, 'distfiles')
osutils.SafeMakedirs(options.cache_dir)
for target in (sdk_cache, distfiles_cache):
src = os.path.join(constants.SOURCE_ROOT,
os.path.basename(target))
if not os.path.exists(src):
osutils.SafeMakedirs(target)
continue
lock.write_lock(
"Upgrade to %r needed but chroot is locked; please exit "
"all instances so this upgrade can finish." % src)
if not os.path.exists(src):
# Note that while waiting for the write lock, src may've vanished;
# it's a rare race during the upgrade process that's a byproduct
# of us avoiding taking a write lock to do the src check. If we
# took a write lock for that check, it would effectively limit
# all cros_sdk for a chroot to a single instance.
osutils.SafeMakedirs(target)
elif not os.path.exists(target):
# Upgrade occurred, but a reversion, or something whacky
# occurred writing to the old location. Wipe and continue.
os.rename(src, target)
else:
# Upgrade occurred once already, but either a reversion or
# some before/after separate cros_sdk usage is at play.
# Wipe and continue.
osutils.RmDir(src)
if options.download:
lock.write_lock()
sdk_tarball = FetchRemoteTarballs(sdk_cache, urls)
if options.create:
lock.write_lock()
CreateChroot(options.chroot,
sdk_tarball,
options.cache_dir,
nousepkg=(options.bootstrap or options.nousepkg),
nogetbinpkg=options.nogetbinpkg)
if options.enter:
lock.read_lock()
EnterChroot(options.chroot, options.cache_dir,
options.chrome_root, options.chrome_root_mount,
chroot_command)
def main(argv):
# Turn on strict sudo checks.
cros_build_lib.STRICT_SUDO = True
conf = key_value_store.LoadFile(os.path.join(constants.SOURCE_ROOT,
constants.SDK_VERSION_FILE),
ignore_missing=True)
sdk_latest_version = conf.get('SDK_LATEST_VERSION', '<unknown>')
bootstrap_frozen_version = conf.get('BOOTSTRAP_FROZEN_VERSION',
'<unknown>')
# Use latest SDK for bootstrapping if requested. Use a frozen version of SDK
# for bootstrapping if BOOTSTRAP_FROZEN_VERSION is set.
bootstrap_latest_version = (sdk_latest_version if bootstrap_frozen_version
== '<unknown>' else bootstrap_frozen_version)
parser, commands = _CreateParser(sdk_latest_version,
bootstrap_latest_version)
options = parser.parse_args(argv)
chroot_command = options.commands
# Some sanity checks first, before we ask for sudo credentials.
cros_build_lib.AssertOutsideChroot()
host = os.uname()[4]
if host != 'x86_64':
cros_build_lib.Die(
"cros_sdk is currently only supported on x86_64; you're running"
' %s. Please find a x86_64 machine.' % (host, ))
# Merge the outside PATH setting if we re-execed ourselves.
if 'CHROMEOS_SUDO_PATH' in os.environ:
os.environ['PATH'] = '%s:%s' % (os.environ.pop('CHROMEOS_SUDO_PATH'),
os.environ['PATH'])
_ReportMissing(osutils.FindMissingBinaries(NEEDED_TOOLS))
if options.proxy_sim:
_ReportMissing(osutils.FindMissingBinaries(PROXY_NEEDED_TOOLS))
missing_image_tools = osutils.FindMissingBinaries(IMAGE_NEEDED_TOOLS)
if (sdk_latest_version == '<unknown>'
or bootstrap_latest_version == '<unknown>'):
cros_build_lib.Die(
'No SDK version was found. '
'Are you in a Chromium source tree instead of Chromium OS?\n\n'
'Please change to a directory inside your Chromium OS source tree\n'
'and retry. If you need to setup a Chromium OS source tree, see\n'
' https://dev.chromium.org/chromium-os/developer-guide')
any_snapshot_operation = (options.snapshot_create
or options.snapshot_restore
or options.snapshot_delete
or options.snapshot_list)
if any_snapshot_operation and not options.use_image:
cros_build_lib.Die('Snapshot operations are not compatible with '
'--nouse-image.')
if (options.snapshot_delete
and options.snapshot_delete == options.snapshot_restore):
parser.error('Cannot --snapshot_delete the same snapshot you are '
'restoring with --snapshot_restore.')
_ReExecuteIfNeeded([sys.argv[0]] + argv)
lock_path = os.path.dirname(options.chroot)
lock_path = os.path.join(
lock_path, '.%s_lock' % os.path.basename(options.chroot).lstrip('.'))
# Expand out the aliases...
if options.replace:
options.delete = options.create = True
if options.bootstrap:
options.create = True
# If a command is not given, default to enter.
# pylint: disable=protected-access
# This _group_actions access sucks, but upstream decided to not include an
# alternative to optparse's option_list, and this is what they recommend.
options.enter |= not any(
getattr(options, x.dest) for x in commands._group_actions)
# pylint: enable=protected-access
options.enter |= bool(chroot_command)
if (options.delete and not options.create
and (options.enter or any_snapshot_operation)):
parser.error('Trying to enter or snapshot the chroot when --delete '
'was specified makes no sense.')
if (options.unmount
and (options.create or options.enter or any_snapshot_operation)):
parser.error(
'--unmount cannot be specified with other chroot actions.')
if options.working_dir is not None and not os.path.isabs(
options.working_dir):
options.working_dir = path_util.ToChrootPath(options.working_dir)
# Discern if we need to create the chroot.
chroot_exists = cros_sdk_lib.IsChrootReady(options.chroot)
if (options.use_image and not chroot_exists and not options.delete
and not options.unmount and not missing_image_tools
and os.path.exists(_ImageFileForChroot(options.chroot))):
# Try to re-mount an existing image in case the user has rebooted.
with cgroups.SimpleContainChildren('cros_sdk'):
with locking.FileLock(lock_path, 'chroot lock') as lock:
logging.debug(
'Checking if existing chroot image can be mounted.')
lock.write_lock()
cros_sdk_lib.MountChroot(options.chroot, create=False)
chroot_exists = cros_sdk_lib.IsChrootReady(options.chroot)
if chroot_exists:
logging.notice('Mounted existing image %s on chroot',
_ImageFileForChroot(options.chroot))
# Finally, flip create if necessary.
if options.enter or options.snapshot_create:
options.create |= not chroot_exists
# Make sure we will download if we plan to create.
options.download |= options.create
# Anything that needs to manipulate the main chroot mount or communicate with
# LVM needs to be done here before we enter the new namespaces.
# If deleting, do it regardless of the use_image flag so that a
# previously-created loopback chroot can also be cleaned up.
# TODO(bmgordon): See if the DeleteChroot call below can be removed in
# favor of this block.
chroot_deleted = False
if options.delete:
with cgroups.SimpleContainChildren('cros_sdk'):
# Set a timeout of 300 seconds when getting the lock.
with locking.FileLock(lock_path,
'chroot lock',
blocking_timeout=300) as lock:
try:
lock.write_lock()
except timeout_util.TimeoutError as e:
logging.error('Acquiring write_lock on %s failed: %s',
lock_path, e)
if not options.force:
cros_build_lib.Die(
'Exiting; use --force to continue w/o lock.')
else:
logging.warning(
'cros_sdk was invoked with force option, continuing.'
)
if missing_image_tools:
logging.notice('Unmounting chroot.')
osutils.UmountTree(options.chroot)
else:
logging.notice('Deleting chroot.')
cros_sdk_lib.CleanupChrootMount(options.chroot,
delete=True)
chroot_deleted = True
# If cleanup was requested, we have to do it while we're still in the original
# namespace. Since cleaning up the mount will interfere with any other
# commands, we exit here. The check above should have made sure that no other
# action was requested, anyway.
if options.unmount:
# Set a timeout of 300 seconds when getting the lock.
with locking.FileLock(lock_path, 'chroot lock',
blocking_timeout=300) as lock:
try:
lock.write_lock()
except timeout_util.TimeoutError as e:
logging.error('Acquiring write_lock on %s failed: %s',
lock_path, e)
logging.warning(
'Continuing with CleanupChroot(%s), which will umount the tree.',
options.chroot)
# We can call CleanupChroot (which calls cros_sdk_lib.CleanupChrootMount)
# even if we don't get the lock because it will attempt to unmount the
# tree and will print diagnostic information from 'fuser', 'lsof', and
# 'ps'.
CleanupChroot(options.chroot)
sys.exit(0)
# Make sure the main chroot mount is visible. Contents will be filled in
# below if needed.
if options.create and options.use_image:
if missing_image_tools:
raise SystemExit("""The tool(s) %s were not found.
Please make sure the lvm2 and thin-provisioning-tools packages
are installed on your host.
Example(ubuntu):
sudo apt-get install lvm2 thin-provisioning-tools
If you want to run without lvm2, pass --nouse-image (chroot
snapshots will be unavailable).""" % ', '.join(missing_image_tools))
logging.debug('Making sure chroot image is mounted.')
with cgroups.SimpleContainChildren('cros_sdk'):
with locking.FileLock(lock_path, 'chroot lock') as lock:
lock.write_lock()
if not cros_sdk_lib.MountChroot(options.chroot, create=True):
cros_build_lib.Die('Unable to mount %s on chroot',
_ImageFileForChroot(options.chroot))
logging.notice('Mounted %s on chroot',
_ImageFileForChroot(options.chroot))
# Snapshot operations will always need the VG/LV, but other actions won't.
if any_snapshot_operation:
with cgroups.SimpleContainChildren('cros_sdk'):
with locking.FileLock(lock_path, 'chroot lock') as lock:
chroot_vg, chroot_lv = cros_sdk_lib.FindChrootMountSource(
options.chroot)
if not chroot_vg or not chroot_lv:
cros_build_lib.Die('Unable to find VG/LV for chroot %s',
options.chroot)
# Delete snapshot before creating a new one. This allows the user to
# throw out old state, create a new snapshot, and enter the chroot in a
# single call to cros_sdk. Since restore involves deleting, also do it
# before creating.
if options.snapshot_restore:
lock.write_lock()
valid_snapshots = ListChrootSnapshots(chroot_vg, chroot_lv)
if options.snapshot_restore not in valid_snapshots:
cros_build_lib.Die(
'%s is not a valid snapshot to restore to. '
'Valid snapshots: %s', options.snapshot_restore,
', '.join(valid_snapshots))
osutils.UmountTree(options.chroot)
if not RestoreChrootSnapshot(options.snapshot_restore,
chroot_vg, chroot_lv):
cros_build_lib.Die(
'Unable to restore chroot to snapshot.')
if not cros_sdk_lib.MountChroot(options.chroot,
create=False):
cros_build_lib.Die(
'Unable to mount restored snapshot onto chroot.')
# Use a read lock for snapshot delete and create even though they modify
# the filesystem, because they don't modify the mounted chroot itself.
# The underlying LVM commands take their own locks, so conflicting
# concurrent operations here may crash cros_sdk, but won't corrupt the
# chroot image. This tradeoff seems worth it to allow snapshot
# operations on chroots that have a process inside.
if options.snapshot_delete:
lock.read_lock()
DeleteChrootSnapshot(options.snapshot_delete, chroot_vg,
chroot_lv)
if options.snapshot_create:
lock.read_lock()
if not CreateChrootSnapshot(options.snapshot_create,
chroot_vg, chroot_lv):
cros_build_lib.Die('Unable to create snapshot.')
img_path = _ImageFileForChroot(options.chroot)
if (options.use_image and os.path.exists(options.chroot)
and os.path.exists(img_path)):
img_stat = os.stat(img_path)
img_used_bytes = img_stat.st_blocks * 512
mount_stat = os.statvfs(options.chroot)
mount_used_bytes = mount_stat.f_frsize * (mount_stat.f_blocks -
mount_stat.f_bfree)
extra_gbs = (img_used_bytes - mount_used_bytes) // 2**30
if extra_gbs > MAX_UNUSED_IMAGE_GBS:
logging.notice(
'%s is using %s GiB more than needed. Running '
'fstrim.', img_path, extra_gbs)
cmd = ['fstrim', options.chroot]
try:
cros_build_lib.dbg_run(cmd)
except cros_build_lib.RunCommandError as e:
logging.warning(
'Running fstrim failed. Consider running fstrim on '
'your chroot manually.\n%s', e)
# Enter a new set of namespaces. Everything after here cannot directly affect
# the hosts's mounts or alter LVM volumes.
namespaces.SimpleUnshare()
if options.ns_pid:
first_pid = namespaces.CreatePidNs()
else:
first_pid = None
if options.snapshot_list:
for snap in ListChrootSnapshots(chroot_vg, chroot_lv):
print(snap)
sys.exit(0)
if not options.sdk_version:
sdk_version = (bootstrap_latest_version
if options.bootstrap else sdk_latest_version)
else:
sdk_version = options.sdk_version
if options.buildbot_log_version:
logging.PrintBuildbotStepText(sdk_version)
# Based on selections, determine the tarball to fetch.
if options.download:
if options.sdk_url:
urls = [options.sdk_url]
else:
urls = GetArchStageTarballs(sdk_version)
with cgroups.SimpleContainChildren('cros_sdk', pid=first_pid):
with locking.FileLock(lock_path, 'chroot lock') as lock:
if options.proxy_sim:
_ProxySimSetup(options)
if (options.delete and not chroot_deleted and
(os.path.exists(options.chroot)
or os.path.exists(_ImageFileForChroot(options.chroot)))):
lock.write_lock()
DeleteChroot(options.chroot)
sdk_cache = os.path.join(options.cache_dir, 'sdks')
distfiles_cache = os.path.join(options.cache_dir, 'distfiles')
osutils.SafeMakedirsNonRoot(options.cache_dir)
for target in (sdk_cache, distfiles_cache):
src = os.path.join(constants.SOURCE_ROOT,
os.path.basename(target))
if not os.path.exists(src):
osutils.SafeMakedirsNonRoot(target)
continue
lock.write_lock(
'Upgrade to %r needed but chroot is locked; please exit '
'all instances so this upgrade can finish.' % src)
if not os.path.exists(src):
# Note that while waiting for the write lock, src may've vanished;
# it's a rare race during the upgrade process that's a byproduct
# of us avoiding taking a write lock to do the src check. If we
# took a write lock for that check, it would effectively limit
# all cros_sdk for a chroot to a single instance.
osutils.SafeMakedirsNonRoot(target)
elif not os.path.exists(target):
# Upgrade occurred, but a reversion, or something whacky
# occurred writing to the old location. Wipe and continue.
os.rename(src, target)
else:
# Upgrade occurred once already, but either a reversion or
# some before/after separate cros_sdk usage is at play.
# Wipe and continue.
osutils.RmDir(src)
if options.download:
lock.write_lock()
sdk_tarball = FetchRemoteTarballs(
sdk_cache, urls, 'stage3' if options.bootstrap else 'SDK')
if options.create:
lock.write_lock()
# Recheck if the chroot is set up here before creating to make sure we
# account for whatever the various delete/unmount/remount steps above
# have done.
if cros_sdk_lib.IsChrootReady(options.chroot):
logging.debug('Chroot already exists. Skipping creation.')
else:
CreateChroot(options.chroot,
sdk_tarball,
options.cache_dir,
nousepkg=(options.bootstrap
or options.nousepkg))
if options.enter:
lock.read_lock()
EnterChroot(options.chroot, options.cache_dir,
options.chrome_root, options.chrome_root_mount,
options.goma_dir, options.goma_client_json,
options.working_dir, chroot_command)
