def _CleanWorkspace(self):
logging.info('Cleaning up workspace checkout.')
assert self._run.options.workspace
workspace = self._run.options.workspace
logging.info('Remove Chroot.')
chroot_dir = os.path.join(workspace, constants.DEFAULT_CHROOT_DIR)
if os.path.exists(chroot_dir) or os.path.exists(chroot_dir + '.img'):
cros_sdk_lib.CleanupChrootMount(chroot_dir, delete=True)
logging.info('Remove all workspace files except .repo.')
repository.ClearBuildRoot(workspace, ['.repo'])
def PerformStage(self):
if (not (self._run.options.buildbot or self._run.options.remote_trybot)
and self._run.options.clobber):
if not commands.ValidateClobber(self._build_root):
cros_build_lib.Die("--clobber in local mode must be approved.")
# If we can't get a manifest out of it, then it's not usable and must be
# clobbered.
manifest = None
if not self._run.options.clobber:
try:
manifest = git.ManifestCheckout.Cached(self._build_root,
search=False)
except (KeyboardInterrupt, MemoryError, SystemExit):
raise
except Exception as e:
# Either there is no repo there, or the manifest isn't usable. If the
# directory exists, log the exception for debugging reasons. Either
# way, the checkout needs to be wiped since it's in an unknown
# state.
if os.path.exists(self._build_root):
logging.warning("ManifestCheckout at %s is unusable: %s",
self._build_root, e)
# Clean mount points first to be safe about deleting.
commands.CleanUpMountPoints(self._build_root)
if manifest is None:
self._DeleteChroot()
repository.ClearBuildRoot(self._build_root,
self._run.options.preserve_paths)
else:
tasks = [
self._BuildRootGitCleanup, self._WipeOldOutput,
self._DeleteArchivedTrybotImages,
self._DeleteArchivedPerfResults,
self._DeleteAutotestSitePackages
]
if self._run.options.chrome_root:
tasks.append(self._DeleteChromeBuildOutput)
if self._run.config.chroot_replace and self._run.options.build:
tasks.append(self._DeleteChroot)
else:
tasks.append(self._CleanChroot)
# Only enable CancelObsoleteSlaveBuilds on the master-paladin,
# it checks for builds in ChromiumOs and ChromeOs waterfalls.
if self._run.config.name == constants.CQ_MASTER:
tasks.append(self.CancelObsoleteSlaveBuilds)
parallel.RunParallelSteps(tasks)
def PerformStage(self):
if (not (self._run.options.buildbot or self._run.options.remote_trybot)
and self._run.options.clobber):
if not commands.ValidateClobber(self._build_root):
cros_build_lib.Die('--clobber in local mode must be approved.')
# If we can't get a manifest out of it, then it's not usable and must be
# clobbered.
manifest = None
delete_chroot = False
if not self._run.options.clobber:
try:
manifest = git.ManifestCheckout.Cached(self._build_root,
search=False)
except (KeyboardInterrupt, MemoryError, SystemExit):
raise
except Exception as e:
# Either there is no repo there, or the manifest isn't usable. If the
# directory exists, log the exception for debugging reasons. Either
# way, the checkout needs to be wiped since it's in an unknown
# state.
if os.path.exists(self._build_root):
logging.warning('ManifestCheckout at %s is unusable: %s',
self._build_root, e)
delete_chroot = True
# Clean mount points first to be safe about deleting.
chroot_path = os.path.join(self._build_root,
constants.DEFAULT_CHROOT_DIR)
cros_sdk_lib.CleanupChrootMount(chroot=chroot_path)
osutils.UmountTree(self._build_root)
if not delete_chroot:
delete_chroot = not self.CanReuseChroot(chroot_path)
# If we're going to delete the chroot and we can use a snapshot instead,
# try to revert. If the revert succeeds, we don't need to delete after all.
if delete_chroot and self.CanUseChrootSnapshotToDelete(chroot_path):
delete_chroot = not self._RevertChrootToCleanSnapshot()
# Re-mount chroot image if it exists so that subsequent steps can clean up
# inside.
if not delete_chroot and self._run.config.chroot_use_image:
try:
cros_sdk_lib.MountChroot(chroot=chroot_path, create=False)
except cros_build_lib.RunCommandError as e:
logging.error(
'Unable to mount chroot under %s. Deleting chroot. '
'Error: %s', self._build_root, e)
delete_chroot = True
if manifest is None:
self._DeleteChroot()
repository.ClearBuildRoot(self._build_root,
self._run.options.preserve_paths)
else:
tasks = [
self._BuildRootGitCleanup, self._WipeOldOutput,
self._DeleteArchivedTrybotImages,
self._DeleteArchivedPerfResults,
self._DeleteAutotestSitePackages
]
if self._run.options.chrome_root:
tasks.append(self._DeleteChromeBuildOutput)
if delete_chroot:
tasks.append(self._DeleteChroot)
else:
tasks.append(self._CleanChroot)
if self._run.options.workspace:
tasks.append(self._CleanWorkspace)
# CancelObsoleteSlaveBuilds, if there are slave builds to cancel.
if self._run.config.slave_configs:
tasks.append(self.CancelObsoleteSlaveBuilds)
parallel.RunParallelSteps(tasks)
# If chroot.img still exists after everything is cleaned up, it means we're
# planning to reuse it. This chroot was created by the previous run, so its
# creation isn't affected by any potential changes in the current run.
# Therefore, if this run fails, having the subsequent run revert to this
# snapshot will still produce a clean chroot. If this run succeeds, the
# next run will reuse the chroot without needing to revert it. Thus, taking
# a snapshot now should be correct regardless of whether this run will
# ultimately succeed or not.
if os.path.exists(chroot_path + '.img'):
self._CreateCleanSnapshot()
def CleanBuildRoot(root, repo, cache_dir, build_state):
"""Some kinds of branch transitions break builds.
This method ensures that cbuildbot's buildroot is a clean checkout on the
given branch when it starts. If necessary (a branch transition) it will wipe
assorted state that cannot be safely reused from the previous build.
Args:
root: Root directory owned by cbuildbot_launch.
repo: repository.RepoRepository instance.
cache_dir: Cache directory.
build_state: BuildSummary object containing the current build state that
will be saved into the cleaned root. The distfiles_ts property will
be updated if the distfiles cache is cleaned.
"""
previous_state = GetLastBuildState(root)
SetLastBuildState(root, build_state)
SanitizeCacheDir(cache_dir)
build_state.distfiles_ts = _MaybeCleanDistfiles(
cache_dir, previous_state.distfiles_ts)
if previous_state.buildroot_layout != BUILDROOT_BUILDROOT_LAYOUT:
logging.PrintBuildbotStepText('Unknown layout: Wiping buildroot.')
metrics.Counter(METRIC_CLOBBER).increment(
fields=field({}, reason='layout_change'))
chroot_dir = os.path.join(root, constants.DEFAULT_CHROOT_DIR)
if os.path.exists(chroot_dir) or os.path.exists(chroot_dir + '.img'):
cros_sdk_lib.CleanupChrootMount(chroot_dir, delete=True)
osutils.RmDir(root, ignore_missing=True, sudo=True)
osutils.RmDir(cache_dir, ignore_missing=True, sudo=True)
else:
if previous_state.branch != repo.branch:
logging.PrintBuildbotStepText('Branch change: Cleaning buildroot.')
logging.info('Unmatched branch: %s -> %s', previous_state.branch,
repo.branch)
metrics.Counter(METRIC_BRANCH_CLEANUP).increment(
fields=field({}, old_branch=previous_state.branch))
logging.info('Remove Chroot.')
chroot_dir = os.path.join(repo.directory,
constants.DEFAULT_CHROOT_DIR)
if os.path.exists(chroot_dir) or os.path.exists(chroot_dir +
'.img'):
cros_sdk_lib.CleanupChrootMount(chroot_dir, delete=True)
logging.info('Remove Chrome checkout.')
osutils.RmDir(os.path.join(repo.directory, '.cache', 'distfiles'),
ignore_missing=True,
sudo=True)
try:
# If there is any failure doing the cleanup, wipe everything.
# The previous run might have been killed in the middle leaving stale git
# locks. Clean those up, first.
repo.PreLoad()
# If the previous build didn't exit normally, run an expensive step to
# cleanup abandoned git locks.
if previous_state.status not in (constants.BUILDER_STATUS_FAILED,
constants.BUILDER_STATUS_PASSED):
repo.CleanStaleLocks()
repo.BuildRootGitCleanup(prune_all=True)
except Exception:
logging.info('Checkout cleanup failed, wiping buildroot:',
exc_info=True)
metrics.Counter(METRIC_CLOBBER).increment(
fields=field({}, reason='repo_cleanup_failure'))
repository.ClearBuildRoot(repo.directory)
# Ensure buildroot exists. Save the state we are prepped for.
osutils.SafeMakedirs(repo.directory)
SetLastBuildState(root, build_state)
def CleanBuildRoot(root, repo, metrics_fields, build_state):
"""Some kinds of branch transitions break builds.
This method ensures that cbuildbot's buildroot is a clean checkout on the
given branch when it starts. If necessary (a branch transition) it will wipe
assorted state that cannot be safely reused from the previous build.
Args:
root: Root directory owned by cbuildbot_launch.
repo: repository.RepoRepository instance.
metrics_fields: Dictionary of fields to include in metrics.
build_state: BuildSummary object containing the current build state that
will be saved into the cleaned root. The distfiles_ts property will
be updated if the distfiles cache is cleaned.
"""
previous_state = GetLastBuildState(root)
build_state.distfiles_ts = _MaybeCleanDistfiles(
repo, previous_state.distfiles_ts, metrics_fields)
if previous_state.buildroot_layout != BUILDROOT_BUILDROOT_LAYOUT:
logging.PrintBuildbotStepText('Unknown layout: Wiping buildroot.')
metrics.Counter(METRIC_CLOBBER).increment(
field(metrics_fields, reason='layout_change'))
chroot_dir = os.path.join(root, constants.DEFAULT_CHROOT_DIR)
if os.path.exists(chroot_dir) or os.path.exists(chroot_dir + '.img'):
cros_sdk_lib.CleanupChrootMount(chroot_dir, delete_image=True)
osutils.RmDir(root, ignore_missing=True, sudo=True)
else:
if previous_state.branch != repo.branch:
logging.PrintBuildbotStepText('Branch change: Cleaning buildroot.')
logging.info('Unmatched branch: %s -> %s', previous_state.branch,
repo.branch)
metrics.Counter(METRIC_BRANCH_CLEANUP).increment(
field(metrics_fields, old_branch=previous_state.branch))
logging.info('Remove Chroot.')
chroot_dir = os.path.join(repo.directory,
constants.DEFAULT_CHROOT_DIR)
if os.path.exists(chroot_dir) or os.path.exists(chroot_dir +
'.img'):
cros_sdk_lib.CleanupChrootMount(chroot_dir, delete_image=True)
osutils.RmDir(chroot_dir, ignore_missing=True, sudo=True)
logging.info('Remove Chrome checkout.')
osutils.RmDir(os.path.join(repo.directory, '.cache', 'distfiles'),
ignore_missing=True,
sudo=True)
try:
# If there is any failure doing the cleanup, wipe everything.
repo.BuildRootGitCleanup(prune_all=True)
except Exception:
logging.info('Checkout cleanup failed, wiping buildroot:',
exc_info=True)
metrics.Counter(METRIC_CLOBBER).increment(
field(metrics_fields, reason='repo_cleanup_failure'))
repository.ClearBuildRoot(repo.directory)
# Ensure buildroot exists. Save the state we are prepped for.
osutils.SafeMakedirs(repo.directory)
if not build_state.distfiles_ts:
build_state.distfiles_ts = time.time()
SetLastBuildState(root, build_state)