def testPrintBuildbotFunctionsWithMarker(self): """PrintBuildbot* with markers should be recognized by buildbot.""" logging.EnableBuildbotMarkers() self.AssertLogContainsMsg( '@@@STEP_LINK@name@url@@@', lambda: logging.PrintBuildbotLink('name', 'url'), check_stderr=True) self.AssertLogContainsMsg( '@@@STEP_TEXT@text@@@', lambda: logging.PrintBuildbotStepText('text'), check_stderr=True) self.AssertLogContainsMsg('@@@STEP_WARNINGS@@@', logging.PrintBuildbotStepWarnings, check_stderr=True) self.AssertLogContainsMsg('@@@STEP_FAILURE@@@', logging.PrintBuildbotStepFailure, check_stderr=True) self.AssertLogContainsMsg( '@@@BUILD_STEP@name@@@', lambda: logging.PrintBuildbotStepName('name'), check_stderr=True) self.AssertLogContainsMsg( '@@@SET_BUILD_PROPERTY@name@"value"@@@', lambda: logging.PrintKitchenSetBuildProperty('name', 'value'), check_stderr=True)
def main(argv): logging.EnableBuildbotMarkers() parser = GetParser() options = parser.parse_args(argv) options.Freeze() overlay_dir = os.path.abspath(_OVERLAY_DIR % {'srcroot': options.srcroot}) android_package_dir = os.path.join( overlay_dir, portage_util.GetFullAndroidPortagePackageName(options.android_package)) version_to_uprev = None (unstable_ebuild, stable_ebuilds) = FindAndroidCandidates(android_package_dir) acls = MakeAclDict(android_package_dir) build_targets = MakeBuildTargetDict(options.android_package, options.android_build_branch) # Mirror artifacts, i.e., images and some sdk tools (e.g., adb, aapt). version_to_uprev = MirrorArtifacts(options.android_bucket_url, options.android_build_branch, options.arc_bucket_url, acls, build_targets, options.force_version) stable_candidate = portage_util.BestEBuild(stable_ebuilds) if stable_candidate: logging.info('Stable candidate found %s', stable_candidate.version) else: logging.info('No stable candidate found.') tracking_branch = 'remotes/m/%s' % os.path.basename(options.tracking_branch) existing_branch = git.GetCurrentBranch(android_package_dir) work_branch = cros_mark_as_stable.GitBranch(constants.STABLE_EBUILD_BRANCH, tracking_branch, android_package_dir) work_branch.CreateBranch() # In the case of uprevving overlays that have patches applied to them, # include the patched changes in the stabilizing branch. if existing_branch: git.RunGit(overlay_dir, ['rebase', existing_branch]) android_version_atom = MarkAndroidEBuildAsStable( stable_candidate, unstable_ebuild, options.android_package, version_to_uprev, android_package_dir, options.android_build_branch, options.arc_bucket_url, options.runtime_artifacts_bucket_url, build_targets) if android_version_atom: if options.boards: cros_mark_as_stable.CleanStalePackages(options.srcroot, options.boards.split(':'), [android_version_atom]) # Explicit print to communicate to caller. print('ANDROID_VERSION_ATOM=%s' % android_version_atom)
def setUp(self): # Always stub RunCommmand out as we use it in every method. self._bot_id = 'amd64-generic-paladin' self.buildstore = FakeBuildStore() site_config = config_lib_unittest.MockSiteConfig() build_config = site_config[self._bot_id] self.build_root = '/fake_root' # This test compares log output from the stages, so turn on buildbot # logging. logging.EnableBuildbotMarkers() self.db = fake_cidb.FakeCIDBConnection() cidb.CIDBConnectionFactory.SetupMockCidb(self.db) # Create a class to hold class Options(object): """Dummy class to hold option values.""" options = Options() options.archive_base = 'gs://dontcare' options.buildroot = self.build_root options.debug = False options.prebuilts = False options.clobber = False options.nosdk = False options.remote_trybot = False options.latest_toolchain = False options.buildnumber = 1234 options.android_rev = None options.chrome_rev = None options.branch = 'dontcare' options.chrome_root = False options.build_config_name = '' self._manager = parallel.Manager() # Pylint-1.9 has a false positive on this for some reason. self._manager.__enter__() # pylint: disable=no-value-for-parameter self._run = cbuildbot_run.BuilderRun(options, site_config, build_config, self._manager) results_lib.Results.Clear()
def main(argv): # We get false positives with the options object. # pylint: disable=attribute-defined-outside-init # Turn on strict sudo checks. cros_build_lib.STRICT_SUDO = True # Set umask to 022 so files created by buildbot are readable. os.umask(0o22) parser = _CreateParser() options = ParseCommandLine(parser, argv) # Fetch our site_config now, because we need it to do anything else. site_config = config_lib.GetConfig() _PostParseCheck(parser, options, site_config) cros_build_lib.AssertOutsideChroot() if options.enable_buildbot_tags: logging.EnableBuildbotMarkers() if (options.buildbot and not options.debug and not options.build_config_name == constants.BRANCH_UTIL_CONFIG and not cros_build_lib.HostIsCIBuilder()): # --buildbot can only be used on a real builder, unless it's debug, or # 'branch-util'. cros_build_lib.Die('This host is not a supported build machine.') # Only one config arg is allowed in this mode, which was confirmed earlier. build_config = site_config[options.build_config_name] # TODO: Re-enable this block when reference_repo support handles this # properly. (see chromium:330775) # if options.reference_repo is None: # repo_path = os.path.join(options.sourceroot, '.repo') # # If we're being run from a repo checkout, reuse the repo's git pool to # # cut down on sync time. # if os.path.exists(repo_path): # options.reference_repo = options.sourceroot if options.reference_repo: if not os.path.exists(options.reference_repo): parser.error('Reference path %s does not exist' % (options.reference_repo, )) elif not os.path.exists(os.path.join(options.reference_repo, '.repo')): parser.error('Reference path %s does not look to be the base of a ' 'repo checkout; no .repo exists in the root.' % (options.reference_repo, )) if (options.buildbot or options.remote_trybot) and not options.resume: if not options.cgroups: parser.error( 'Options --buildbot/--remote-trybot and --nocgroups cannot ' 'be used together. Cgroup support is required for ' 'buildbot/remote-trybot mode.') if not cgroups.Cgroup.IsSupported(): parser.error( 'Option --buildbot/--remote-trybot was given, but this ' 'system does not support cgroups. Failing.') missing = osutils.FindMissingBinaries(_BUILDBOT_REQUIRED_BINARIES) if missing: parser.error( 'Option --buildbot/--remote-trybot requires the following ' "binaries which couldn't be found in $PATH: %s" % (', '.join(missing))) if options.reference_repo: options.reference_repo = os.path.abspath(options.reference_repo) # Sanity check of buildroot- specifically that it's not pointing into the # midst of an existing repo since git-repo doesn't support nesting. if (not repository.IsARepoRoot(options.buildroot) and git.FindRepoDir(options.buildroot)): cros_build_lib.Die( 'Configured buildroot %s is a subdir of an existing repo checkout.' % options.buildroot) if not options.log_dir: options.log_dir = os.path.join(options.buildroot, _DEFAULT_LOG_DIR) log_file = None if options.tee: log_file = os.path.join(options.log_dir, _BUILDBOT_LOG_FILE) osutils.SafeMakedirs(options.log_dir) _BackupPreviousLog(log_file) with cros_build_lib.ContextManagerStack() as stack: options.preserve_paths = set() if log_file is not None: # We don't want the critical section to try to clean up the tee process, # so we run Tee (forked off) outside of it. This prevents a deadlock # because the Tee process only exits when its pipe is closed, and the # critical section accidentally holds on to that file handle. stack.Add(tee.Tee, log_file) options.preserve_paths.add(_DEFAULT_LOG_DIR) critical_section = stack.Add(cleanup.EnforcedCleanupSection) stack.Add(sudo.SudoKeepAlive) if not options.resume: # If we're in resume mode, use our parents tempdir rather than # nesting another layer. stack.Add(osutils.TempDir, prefix='cbuildbot-tmp', set_global=True) logging.debug('Cbuildbot tempdir is %r.', os.environ.get('TMP')) if options.cgroups: stack.Add(cgroups.SimpleContainChildren, 'cbuildbot') # Mark everything between EnforcedCleanupSection and here as having to # be rolled back via the contextmanager cleanup handlers. This # ensures that sudo bits cannot outlive cbuildbot, that anything # cgroups would kill gets killed, etc. stack.Add(critical_section.ForkWatchdog) if options.mock_tree_status is not None: stack.Add(_ObjectMethodPatcher, tree_status, '_GetStatus', return_value=options.mock_tree_status) if options.mock_slave_status is not None: with open(options.mock_slave_status, 'r') as f: mock_statuses = pickle.load(f) for key, value in mock_statuses.iteritems(): mock_statuses[key] = builder_status_lib.BuilderStatus( **value) stack.Add(_ObjectMethodPatcher, completion_stages.MasterSlaveSyncCompletionStage, '_FetchSlaveStatuses', return_value=mock_statuses) stack.Add(_SetupConnections, options, build_config) retry_stats.SetupStats() timeout_display_message = None # For master-slave builds: Update slave's timeout using master's published # deadline. if options.buildbot and options.master_build_id is not None: slave_timeout = None if cidb.CIDBConnectionFactory.IsCIDBSetup(): cidb_handle = cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder( ) if cidb_handle: slave_timeout = cidb_handle.GetTimeToDeadline( options.master_build_id) if slave_timeout is not None: # We artificially set a minimum slave_timeout because '0' is handled # specially, and because we don't want to timeout while trying to set # things up. slave_timeout = max(slave_timeout, 20) if options.timeout == 0 or slave_timeout < options.timeout: logging.info( 'Updating slave build timeout to %d seconds enforced ' 'by the master', slave_timeout) options.timeout = slave_timeout timeout_display_message = ( 'This build has reached the timeout deadline set by the master. ' 'Either this stage or a previous one took too long (see stage ' 'timing historical summary in ReportStage) or the build failed ' 'to start on time.') else: logging.warning( 'Could not get master deadline for master-slave build. ' 'Can not set slave timeout.') if options.timeout > 0: stack.Add(timeout_util.FatalTimeout, options.timeout, timeout_display_message) try: _RunBuildStagesWrapper(options, site_config, build_config) except failures_lib.ExitEarlyException as ex: # This build finished successfully. Do not re-raise ExitEarlyException. logging.info('One stage exited early: %s', ex)
def _main(options, argv): """main method of script. Args: options: preparsed options object for the build. argv: All command line arguments to pass as list of strings. Returns: Return code of cbuildbot as an integer. """ branchname = options.branch or 'master' root = options.buildroot buildroot = os.path.join(root, 'repository') workspace = os.path.join(root, 'workspace') depot_tools_path = os.path.join(buildroot, constants.DEPOT_TOOLS_SUBPATH) # Does the entire build pass or fail. with metrics.Presence(METRIC_ACTIVE), \ metrics.SuccessCounter(METRIC_COMPLETED) as s_fields: # Preliminary set, mostly command line parsing. with metrics.SuccessCounter(METRIC_INVOKED): if options.enable_buildbot_tags: logging.EnableBuildbotMarkers() ConfigureGlobalEnvironment() # Prepare the buildroot with source for the build. with metrics.SuccessCounter(METRIC_PREP): manifest_url = config_lib.GetSiteParams().MANIFEST_INT_URL repo = repository.RepoRepository( manifest_url, buildroot, branch=branchname, git_cache_dir=options.git_cache_dir) previous_build_state = GetLastBuildState(root) # Clean up the buildroot to a safe state. with metrics.SecondsTimer(METRIC_CLEAN): build_state = GetCurrentBuildState(options, branchname) CleanBuildRoot(root, repo, options.cache_dir, build_state) # Get a checkout close enough to the branch that cbuildbot can handle it. if options.sync: with metrics.SecondsTimer(METRIC_INITIAL): InitialCheckout(repo) # Run cbuildbot inside the full ChromeOS checkout, on the specified branch. with metrics.SecondsTimer(METRIC_CBUILDBOT), \ metrics.SecondsInstanceTimer(METRIC_CBUILDBOT_INSTANCE): if previous_build_state.is_valid(): argv.append('--previous-build-state') argv.append( base64.b64encode(previous_build_state.to_json().encode( 'utf-8')).decode('utf-8')) argv.extend(['--workspace', workspace]) if not options.cache_dir_specified: argv.extend(['--cache-dir', options.cache_dir]) result = Cbuildbot(buildroot, depot_tools_path, argv) s_fields['success'] = (result == 0) build_state.status = (constants.BUILDER_STATUS_PASSED if result == 0 else constants.BUILDER_STATUS_FAILED) SetLastBuildState(root, build_state) with metrics.SecondsTimer(METRIC_CHROOT_CLEANUP): CleanupChroot(buildroot) return result
def _main(argv): """main method of script. Args: argv: All command line arguments to pass as list of strings. Returns: Return code of cbuildbot as an integer. """ options = PreParseArguments(argv) branchname = options.branch or 'master' root = options.buildroot buildroot = os.path.join(root, 'repository') depot_tools_path = os.path.join(buildroot, constants.DEPOT_TOOLS_SUBPATH) metrics_fields = { 'branch_name': branchname, 'build_config': options.build_config_name, 'tryjob': options.remote_trybot, } # Does the entire build pass or fail. with metrics.Presence(METRIC_ACTIVE, metrics_fields), \ metrics.SuccessCounter(METRIC_COMPLETED, metrics_fields) as s_fields: # Preliminary set, mostly command line parsing. with metrics.SuccessCounter(METRIC_INVOKED, metrics_fields): if options.enable_buildbot_tags: logging.EnableBuildbotMarkers() ConfigureGlobalEnvironment() # Prepare the buildroot with source for the build. with metrics.SuccessCounter(METRIC_PREP, metrics_fields): site_config = config_lib.GetConfig() manifest_url = site_config.params['MANIFEST_INT_URL'] repo = repository.RepoRepository( manifest_url, buildroot, branch=branchname, git_cache_dir=options.git_cache_dir) previous_build_state = GetLastBuildState(root) # Clean up the buildroot to a safe state. with metrics.SecondsTimer(METRIC_CLEAN, fields=metrics_fields): build_state = GetCurrentBuildState(options, branchname) CleanBuildRoot(root, repo, metrics_fields, build_state) # Get a checkout close enough to the branch that cbuildbot can handle it. if options.sync: with metrics.SecondsTimer(METRIC_INITIAL, fields=metrics_fields): InitialCheckout(repo) # Get a checkout close enough to the branch that cbuildbot can handle it. with metrics.SecondsTimer(METRIC_DEPOT_TOOLS, fields=metrics_fields): DepotToolsEnsureBootstrap(depot_tools_path) # Run cbuildbot inside the full ChromeOS checkout, on the specified branch. with metrics.SecondsTimer(METRIC_CBUILDBOT, fields=metrics_fields): if previous_build_state.is_valid(): argv.append('--previous-build-state') argv.append(base64.b64encode(previous_build_state.to_json())) result = Cbuildbot(buildroot, depot_tools_path, argv) s_fields['success'] = (result == 0) build_state.status = (constants.BUILDER_STATUS_PASSED if result == 0 else constants.BUILDER_STATUS_FAILED) SetLastBuildState(root, build_state) CleanupChroot(buildroot) return result