def testPrintBuildbotFunctionsWithMarker(self):
"""PrintBuildbot* with markers should be recognized by buildbot."""
logging.EnableBuildbotMarkers()
self.AssertLogContainsMsg(
'@@@STEP_LINK@name@url@@@',
lambda: logging.PrintBuildbotLink('name', 'url'),
check_stderr=True)
self.AssertLogContainsMsg(
'@@@STEP_TEXT@text@@@',
lambda: logging.PrintBuildbotStepText('text'),
check_stderr=True)
self.AssertLogContainsMsg('@@@STEP_WARNINGS@@@',
logging.PrintBuildbotStepWarnings,
check_stderr=True)
self.AssertLogContainsMsg('@@@STEP_FAILURE@@@',
logging.PrintBuildbotStepFailure,
check_stderr=True)
self.AssertLogContainsMsg(
'@@@BUILD_STEP@name@@@',
lambda: logging.PrintBuildbotStepName('name'),
check_stderr=True)
self.AssertLogContainsMsg(
'@@@SET_BUILD_PROPERTY@name@"value"@@@',
lambda: logging.PrintKitchenSetBuildProperty('name', 'value'),
check_stderr=True)
def main(argv):
logging.EnableBuildbotMarkers()
parser = GetParser()
options = parser.parse_args(argv)
options.Freeze()
overlay_dir = os.path.abspath(_OVERLAY_DIR % {'srcroot': options.srcroot})
android_package_dir = os.path.join(
overlay_dir,
portage_util.GetFullAndroidPortagePackageName(options.android_package))
version_to_uprev = None
(unstable_ebuild, stable_ebuilds) = FindAndroidCandidates(android_package_dir)
acls = MakeAclDict(android_package_dir)
build_targets = MakeBuildTargetDict(options.android_package,
options.android_build_branch)
# Mirror artifacts, i.e., images and some sdk tools (e.g., adb, aapt).
version_to_uprev = MirrorArtifacts(options.android_bucket_url,
options.android_build_branch,
options.arc_bucket_url, acls,
build_targets,
options.force_version)
stable_candidate = portage_util.BestEBuild(stable_ebuilds)
if stable_candidate:
logging.info('Stable candidate found %s', stable_candidate.version)
else:
logging.info('No stable candidate found.')
tracking_branch = 'remotes/m/%s' % os.path.basename(options.tracking_branch)
existing_branch = git.GetCurrentBranch(android_package_dir)
work_branch = cros_mark_as_stable.GitBranch(constants.STABLE_EBUILD_BRANCH,
tracking_branch,
android_package_dir)
work_branch.CreateBranch()
# In the case of uprevving overlays that have patches applied to them,
# include the patched changes in the stabilizing branch.
if existing_branch:
git.RunGit(overlay_dir, ['rebase', existing_branch])
android_version_atom = MarkAndroidEBuildAsStable(
stable_candidate, unstable_ebuild, options.android_package,
version_to_uprev, android_package_dir,
options.android_build_branch, options.arc_bucket_url,
options.runtime_artifacts_bucket_url, build_targets)
if android_version_atom:
if options.boards:
cros_mark_as_stable.CleanStalePackages(options.srcroot,
options.boards.split(':'),
[android_version_atom])
# Explicit print to communicate to caller.
print('ANDROID_VERSION_ATOM=%s' % android_version_atom)
def setUp(self):
# Always stub RunCommmand out as we use it in every method.
self._bot_id = 'amd64-generic-paladin'
self.buildstore = FakeBuildStore()
site_config = config_lib_unittest.MockSiteConfig()
build_config = site_config[self._bot_id]
self.build_root = '/fake_root'
# This test compares log output from the stages, so turn on buildbot
# logging.
logging.EnableBuildbotMarkers()
self.db = fake_cidb.FakeCIDBConnection()
cidb.CIDBConnectionFactory.SetupMockCidb(self.db)
# Create a class to hold
class Options(object):
"""Dummy class to hold option values."""
options = Options()
options.archive_base = 'gs://dontcare'
options.buildroot = self.build_root
options.debug = False
options.prebuilts = False
options.clobber = False
options.nosdk = False
options.remote_trybot = False
options.latest_toolchain = False
options.buildnumber = 1234
options.android_rev = None
options.chrome_rev = None
options.branch = 'dontcare'
options.chrome_root = False
options.build_config_name = ''
self._manager = parallel.Manager()
# Pylint-1.9 has a false positive on this for some reason.
self._manager.__enter__() # pylint: disable=no-value-for-parameter
self._run = cbuildbot_run.BuilderRun(options, site_config,
build_config, self._manager)
results_lib.Results.Clear()
def main(argv):
# We get false positives with the options object.
# pylint: disable=attribute-defined-outside-init
# Turn on strict sudo checks.
cros_build_lib.STRICT_SUDO = True
# Set umask to 022 so files created by buildbot are readable.
os.umask(0o22)
parser = _CreateParser()
options = ParseCommandLine(parser, argv)
# Fetch our site_config now, because we need it to do anything else.
site_config = config_lib.GetConfig()
_PostParseCheck(parser, options, site_config)
cros_build_lib.AssertOutsideChroot()
if options.enable_buildbot_tags:
logging.EnableBuildbotMarkers()
if (options.buildbot and not options.debug
and not options.build_config_name == constants.BRANCH_UTIL_CONFIG
and not cros_build_lib.HostIsCIBuilder()):
# --buildbot can only be used on a real builder, unless it's debug, or
# 'branch-util'.
cros_build_lib.Die('This host is not a supported build machine.')
# Only one config arg is allowed in this mode, which was confirmed earlier.
build_config = site_config[options.build_config_name]
# TODO: Re-enable this block when reference_repo support handles this
# properly. (see chromium:330775)
# if options.reference_repo is None:
# repo_path = os.path.join(options.sourceroot, '.repo')
# # If we're being run from a repo checkout, reuse the repo's git pool to
# # cut down on sync time.
# if os.path.exists(repo_path):
# options.reference_repo = options.sourceroot
if options.reference_repo:
if not os.path.exists(options.reference_repo):
parser.error('Reference path %s does not exist' %
(options.reference_repo, ))
elif not os.path.exists(os.path.join(options.reference_repo, '.repo')):
parser.error('Reference path %s does not look to be the base of a '
'repo checkout; no .repo exists in the root.' %
(options.reference_repo, ))
if (options.buildbot or options.remote_trybot) and not options.resume:
if not options.cgroups:
parser.error(
'Options --buildbot/--remote-trybot and --nocgroups cannot '
'be used together. Cgroup support is required for '
'buildbot/remote-trybot mode.')
if not cgroups.Cgroup.IsSupported():
parser.error(
'Option --buildbot/--remote-trybot was given, but this '
'system does not support cgroups. Failing.')
missing = osutils.FindMissingBinaries(_BUILDBOT_REQUIRED_BINARIES)
if missing:
parser.error(
'Option --buildbot/--remote-trybot requires the following '
"binaries which couldn't be found in $PATH: %s" %
(', '.join(missing)))
if options.reference_repo:
options.reference_repo = os.path.abspath(options.reference_repo)
# Sanity check of buildroot- specifically that it's not pointing into the
# midst of an existing repo since git-repo doesn't support nesting.
if (not repository.IsARepoRoot(options.buildroot)
and git.FindRepoDir(options.buildroot)):
cros_build_lib.Die(
'Configured buildroot %s is a subdir of an existing repo checkout.'
% options.buildroot)
if not options.log_dir:
options.log_dir = os.path.join(options.buildroot, _DEFAULT_LOG_DIR)
log_file = None
if options.tee:
log_file = os.path.join(options.log_dir, _BUILDBOT_LOG_FILE)
osutils.SafeMakedirs(options.log_dir)
_BackupPreviousLog(log_file)
with cros_build_lib.ContextManagerStack() as stack:
options.preserve_paths = set()
if log_file is not None:
# We don't want the critical section to try to clean up the tee process,
# so we run Tee (forked off) outside of it. This prevents a deadlock
# because the Tee process only exits when its pipe is closed, and the
# critical section accidentally holds on to that file handle.
stack.Add(tee.Tee, log_file)
options.preserve_paths.add(_DEFAULT_LOG_DIR)
critical_section = stack.Add(cleanup.EnforcedCleanupSection)
stack.Add(sudo.SudoKeepAlive)
if not options.resume:
# If we're in resume mode, use our parents tempdir rather than
# nesting another layer.
stack.Add(osutils.TempDir, prefix='cbuildbot-tmp', set_global=True)
logging.debug('Cbuildbot tempdir is %r.', os.environ.get('TMP'))
if options.cgroups:
stack.Add(cgroups.SimpleContainChildren, 'cbuildbot')
# Mark everything between EnforcedCleanupSection and here as having to
# be rolled back via the contextmanager cleanup handlers. This
# ensures that sudo bits cannot outlive cbuildbot, that anything
# cgroups would kill gets killed, etc.
stack.Add(critical_section.ForkWatchdog)
if options.mock_tree_status is not None:
stack.Add(_ObjectMethodPatcher,
tree_status,
'_GetStatus',
return_value=options.mock_tree_status)
if options.mock_slave_status is not None:
with open(options.mock_slave_status, 'r') as f:
mock_statuses = pickle.load(f)
for key, value in mock_statuses.iteritems():
mock_statuses[key] = builder_status_lib.BuilderStatus(
**value)
stack.Add(_ObjectMethodPatcher,
completion_stages.MasterSlaveSyncCompletionStage,
'_FetchSlaveStatuses',
return_value=mock_statuses)
stack.Add(_SetupConnections, options, build_config)
retry_stats.SetupStats()
timeout_display_message = None
# For master-slave builds: Update slave's timeout using master's published
# deadline.
if options.buildbot and options.master_build_id is not None:
slave_timeout = None
if cidb.CIDBConnectionFactory.IsCIDBSetup():
cidb_handle = cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder(
)
if cidb_handle:
slave_timeout = cidb_handle.GetTimeToDeadline(
options.master_build_id)
if slave_timeout is not None:
# We artificially set a minimum slave_timeout because '0' is handled
# specially, and because we don't want to timeout while trying to set
# things up.
slave_timeout = max(slave_timeout, 20)
if options.timeout == 0 or slave_timeout < options.timeout:
logging.info(
'Updating slave build timeout to %d seconds enforced '
'by the master', slave_timeout)
options.timeout = slave_timeout
timeout_display_message = (
'This build has reached the timeout deadline set by the master. '
'Either this stage or a previous one took too long (see stage '
'timing historical summary in ReportStage) or the build failed '
'to start on time.')
else:
logging.warning(
'Could not get master deadline for master-slave build. '
'Can not set slave timeout.')
if options.timeout > 0:
stack.Add(timeout_util.FatalTimeout, options.timeout,
timeout_display_message)
try:
_RunBuildStagesWrapper(options, site_config, build_config)
except failures_lib.ExitEarlyException as ex:
# This build finished successfully. Do not re-raise ExitEarlyException.
logging.info('One stage exited early: %s', ex)
def _main(options, argv):
"""main method of script.
Args:
options: preparsed options object for the build.
argv: All command line arguments to pass as list of strings.
Returns:
Return code of cbuildbot as an integer.
"""
branchname = options.branch or 'master'
root = options.buildroot
buildroot = os.path.join(root, 'repository')
workspace = os.path.join(root, 'workspace')
depot_tools_path = os.path.join(buildroot, constants.DEPOT_TOOLS_SUBPATH)
# Does the entire build pass or fail.
with metrics.Presence(METRIC_ACTIVE), \
metrics.SuccessCounter(METRIC_COMPLETED) as s_fields:
# Preliminary set, mostly command line parsing.
with metrics.SuccessCounter(METRIC_INVOKED):
if options.enable_buildbot_tags:
logging.EnableBuildbotMarkers()
ConfigureGlobalEnvironment()
# Prepare the buildroot with source for the build.
with metrics.SuccessCounter(METRIC_PREP):
manifest_url = config_lib.GetSiteParams().MANIFEST_INT_URL
repo = repository.RepoRepository(
manifest_url,
buildroot,
branch=branchname,
git_cache_dir=options.git_cache_dir)
previous_build_state = GetLastBuildState(root)
# Clean up the buildroot to a safe state.
with metrics.SecondsTimer(METRIC_CLEAN):
build_state = GetCurrentBuildState(options, branchname)
CleanBuildRoot(root, repo, options.cache_dir, build_state)
# Get a checkout close enough to the branch that cbuildbot can handle it.
if options.sync:
with metrics.SecondsTimer(METRIC_INITIAL):
InitialCheckout(repo)
# Run cbuildbot inside the full ChromeOS checkout, on the specified branch.
with metrics.SecondsTimer(METRIC_CBUILDBOT), \
metrics.SecondsInstanceTimer(METRIC_CBUILDBOT_INSTANCE):
if previous_build_state.is_valid():
argv.append('--previous-build-state')
argv.append(
base64.b64encode(previous_build_state.to_json().encode(
'utf-8')).decode('utf-8'))
argv.extend(['--workspace', workspace])
if not options.cache_dir_specified:
argv.extend(['--cache-dir', options.cache_dir])
result = Cbuildbot(buildroot, depot_tools_path, argv)
s_fields['success'] = (result == 0)
build_state.status = (constants.BUILDER_STATUS_PASSED if result
== 0 else constants.BUILDER_STATUS_FAILED)
SetLastBuildState(root, build_state)
with metrics.SecondsTimer(METRIC_CHROOT_CLEANUP):
CleanupChroot(buildroot)
return result
def _main(argv):
"""main method of script.
Args:
argv: All command line arguments to pass as list of strings.
Returns:
Return code of cbuildbot as an integer.
"""
options = PreParseArguments(argv)
branchname = options.branch or 'master'
root = options.buildroot
buildroot = os.path.join(root, 'repository')
depot_tools_path = os.path.join(buildroot, constants.DEPOT_TOOLS_SUBPATH)
metrics_fields = {
'branch_name': branchname,
'build_config': options.build_config_name,
'tryjob': options.remote_trybot,
}
# Does the entire build pass or fail.
with metrics.Presence(METRIC_ACTIVE, metrics_fields), \
metrics.SuccessCounter(METRIC_COMPLETED, metrics_fields) as s_fields:
# Preliminary set, mostly command line parsing.
with metrics.SuccessCounter(METRIC_INVOKED, metrics_fields):
if options.enable_buildbot_tags:
logging.EnableBuildbotMarkers()
ConfigureGlobalEnvironment()
# Prepare the buildroot with source for the build.
with metrics.SuccessCounter(METRIC_PREP, metrics_fields):
site_config = config_lib.GetConfig()
manifest_url = site_config.params['MANIFEST_INT_URL']
repo = repository.RepoRepository(
manifest_url,
buildroot,
branch=branchname,
git_cache_dir=options.git_cache_dir)
previous_build_state = GetLastBuildState(root)
# Clean up the buildroot to a safe state.
with metrics.SecondsTimer(METRIC_CLEAN, fields=metrics_fields):
build_state = GetCurrentBuildState(options, branchname)
CleanBuildRoot(root, repo, metrics_fields, build_state)
# Get a checkout close enough to the branch that cbuildbot can handle it.
if options.sync:
with metrics.SecondsTimer(METRIC_INITIAL,
fields=metrics_fields):
InitialCheckout(repo)
# Get a checkout close enough to the branch that cbuildbot can handle it.
with metrics.SecondsTimer(METRIC_DEPOT_TOOLS,
fields=metrics_fields):
DepotToolsEnsureBootstrap(depot_tools_path)
# Run cbuildbot inside the full ChromeOS checkout, on the specified branch.
with metrics.SecondsTimer(METRIC_CBUILDBOT, fields=metrics_fields):
if previous_build_state.is_valid():
argv.append('--previous-build-state')
argv.append(base64.b64encode(previous_build_state.to_json()))
result = Cbuildbot(buildroot, depot_tools_path, argv)
s_fields['success'] = (result == 0)
build_state.status = (constants.BUILDER_STATUS_PASSED if result
== 0 else constants.BUILDER_STATUS_FAILED)
SetLastBuildState(root, build_state)
CleanupChroot(buildroot)
return result
