def ShowConfig(name): """Show the toolchain tuples used by |name| Args: name: The board name to query. """ toolchains = toolchain.GetToolchainsForBoard(name) # Make sure we display the default toolchain first. print(','.join( toolchain.FilterToolchains(toolchains, 'default', True).keys() + toolchain.FilterToolchains(toolchains, 'default', False).keys()))
def ShowBoardConfig(board): """Show the toolchain tuples used by |board| Args: board: The board to query. """ toolchains = toolchain.GetToolchainsForBoard(board) # Make sure we display the default toolchain first. print ','.join( toolchain.FilterToolchains(toolchains, 'default', True).keys() + toolchain.FilterToolchains(toolchains, 'default', False).keys())
def ShowConfig(name): """Show the toolchain tuples used by |name| Args: name: The board name to query. """ toolchains = toolchain.GetToolchainsForBoard(name) # Make sure we display the default toolchain first. # Note: Do not use logging here as this is meant to be used by other tools. print(','.join( toolchain.FilterToolchains(toolchains, 'default', True).keys() + toolchain.FilterToolchains(toolchains, 'default', False).keys()))
def ExpandTargets(targets_wanted): """Expand any possible toolchain aliases into full targets This will expand 'all' and 'sdk' into the respective toolchain tuples. Args: targets_wanted: The targets specified by the user. Returns: Dictionary of concrete targets and their toolchain tuples. """ targets_wanted = set(targets_wanted) if targets_wanted == set(['boards']): # Only pull targets from the included boards. return {} all_targets = toolchain.GetAllTargets() if targets_wanted == set(['all']): return all_targets if targets_wanted == set(['sdk']): # Filter out all the non-sdk toolchains as we don't want to mess # with those in all of our builds. return toolchain.FilterToolchains(all_targets, 'sdk', True) # Verify user input. nonexistent = targets_wanted.difference(all_targets) if nonexistent: raise ValueError('Invalid targets: %s', ','.join(nonexistent)) return {t: all_targets[t] for t in targets_wanted}
def _GenerateConfig(self, toolchains, board_overlays, portdir_overlays, header, **kwargs): """Create common config settings for boards and bricks. Args: toolchains: ToolchainList object to use. board_overlays: List of board overlays. portdir_overlays: List of portage overlays. header: Header comment string; must start with #. kwargs: Additional configuration values to set. Returns: Configuration string. Raises: ConfigurationError: Could not generate a valid configuration. """ config = {} default_toolchains = toolchain.FilterToolchains(toolchains, 'default', True) if not default_toolchains: raise ConfigurationError('No default toolchain could be found.') config['CHOST'] = default_toolchains.keys()[0] config['ARCH'] = toolchain.GetArchForTarget(config['CHOST']) config['BOARD_OVERLAY'] = '\n'.join(board_overlays) config['PORTDIR_OVERLAY'] = '\n'.join(portdir_overlays) config['MAKEOPTS'] = '-j%s' % str(multiprocessing.cpu_count()) config['ROOT'] = self.path + '/' config['PKG_CONFIG'] = self._WrapperPath('pkg-config') config.update(kwargs) return '\n'.join((header, _DictToKeyValue(config)))
def ExpandTargets(targets_wanted): """Expand any possible toolchain aliases into full targets This will expand 'all' and 'sdk' into the respective toolchain tuples. Args: targets_wanted: The targets specified by the user. Returns: Full list of tuples with pseudo targets removed. """ alltargets = toolchain.GetAllTargets() targets_wanted = set(targets_wanted) if targets_wanted == set(['all']): targets = alltargets elif targets_wanted == set(['sdk']): # Filter out all the non-sdk toolchains as we don't want to mess # with those in all of our builds. targets = toolchain.FilterToolchains(alltargets, 'sdk', True) else: # Verify user input. nonexistent = targets_wanted.difference(alltargets) if nonexistent: raise ValueError('Invalid targets: %s', ','.join(nonexistent)) targets = dict((t, alltargets[t]) for t in targets_wanted) return targets
def GetCrossGdb(self): """Find the appropriate cross-version of gdb for the board.""" toolchains = toolchain.GetToolchainsForBoard(self.board) tc = toolchain.FilterToolchains(toolchains, 'default', True).keys() cross_gdb = tc[0] + '-gdb' if not osutils.Which(cross_gdb): raise GdbMissingDebuggerError('Cannot find %s; do you need to run ' 'setup_board?' % cross_gdb) return cross_gdb
def PerformStage(self): config = self._run.config build_root = self._build_root logging.info('Build re-executions have finished. Chromite source ' 'will not be modified for remainder of run.') logging.info("config['important']=%s", config['important']) logging.PrintBuildbotStepText("config['important']=%s" % config['important']) # Flat list of all child config boards. Since child configs # are not allowed to have children, it is not necessary to search # deeper than one generation. child_configs = GetChildConfigListMetadata( child_configs=config['child_configs'], config_status_map=None) sdk_verinfo = cros_build_lib.LoadKeyValueFile(os.path.join( build_root, constants.SDK_VERSION_FILE), ignore_missing=True) verinfo = self._run.GetVersionInfo() platform_tag = getattr(self._run.attrs, 'release_tag') if not platform_tag: platform_tag = verinfo.VersionString() version = { 'full': self._run.GetVersion(), 'milestone': verinfo.chrome_branch, 'platform': platform_tag, } metadata = { # Version of the metadata format. 'metadata-version': '2', 'boards': config['boards'], 'child-configs': child_configs, 'build_type': config['build_type'], 'important': config['important'], # Data for the toolchain used. 'sdk-version': sdk_verinfo.get('SDK_LATEST_VERSION', '<unknown>'), 'toolchain-url': sdk_verinfo.get('TC_PATH', '<unknown>'), } if len(config['boards']) == 1: toolchains = toolchain.GetToolchainsForBoard(config['boards'][0], buildroot=build_root) metadata['toolchain-tuple'] = ( toolchain.FilterToolchains(toolchains, 'default', True).keys() + toolchain.FilterToolchains(toolchains, 'default', False).keys()) logging.info('Metadata being written: %s', metadata) self._run.attrs.metadata.UpdateWithDict(metadata) # Update 'version' separately to avoid overwriting the existing # entries in it (e.g. PFQ builders may have written the Chrome # version to uprev). logging.info("Metadata 'version' being written: %s", version) self._run.attrs.metadata.UpdateKeyDictWithDict('version', version) # Ensure that all boards and child config boards have a per-board # metadata subdict. for b in config['boards']: self._run.attrs.metadata.UpdateBoardDictWithDict(b, {}) for cc in child_configs: for b in cc['boards']: self._run.attrs.metadata.UpdateBoardDictWithDict(b, {}) # Upload build metadata (and write it to database if necessary) self.UploadMetadata(filename=constants.PARTIAL_METADATA_JSON) # Write child-per-build and board-per-build rows to database build_id, db = self._run.GetCIDBHandle() if db: # TODO(akeshet): replace this with a GetValue call once crbug.com/406522 # is resolved per_board_dict = self._run.attrs.metadata.GetDict( )['board-metadata'] for board, board_metadata in per_board_dict.items(): db.InsertBoardPerBuild(build_id, board) if board_metadata: db.UpdateBoardPerBuildMetadata(build_id, board, board_metadata) for child_config in self._run.attrs.metadata.GetValue( 'child-configs'): db.InsertChildConfigPerBuild(build_id, child_config['name']) # If this build has a master build, ensure that the master full_version # is the same as this build's full_version. This is a sanity check to # avoid bugs in master-slave logic. master_id = self._run.attrs.metadata.GetDict().get( 'master_build_id') if master_id is not None: master_full_version = db.GetBuildStatus( master_id)['full_version'] my_full_version = self._run.attrs.metadata.GetValue( 'version').get('full') if master_full_version != my_full_version: raise failures_lib.MasterSlaveVersionMismatchFailure( 'Master build id %s has full_version %s, while slave version is ' '%s.' % (master_id, master_full_version, my_full_version)) # Abort previous hw test suites. This happens after reexecution as it # requires chromite/third_party/swarming.client, which is not available # untill after reexecution. self._AbortPreviousHWTestSuites(version['milestone'])
def PerformStage(self): """Perform the actual work for this stage. This includes final metadata archival, and update CIDB with our final status as well as producting a logged build result summary. """ build_identifier, _ = self._run.GetCIDBHandle() build_id = build_identifier.cidb_id buildbucket_id = build_identifier.buildbucket_id if results_lib.Results.BuildSucceededSoFar(self.buildstore, buildbucket_id, self.name): final_status = constants.BUILDER_STATUS_PASSED else: final_status = constants.BUILDER_STATUS_FAILED if not hasattr(self._run.attrs, 'release_tag'): # If, for some reason, sync stage was not completed and # release_tag was not set. Set it to None here because # ArchiveResults() depends the existence of this attr. self._run.attrs.release_tag = None # Set up our report metadata. self._run.attrs.metadata.UpdateWithDict( self.GetReportMetadata( final_status=final_status, completion_instance=self._completion_instance)) src_root = self._build_root # Workspace builders use a different buildroot for overlays. if self._run.config.workspace_branch and self._run.options.workspace: src_root = self._run.options.workspace # Add tags for the arches and statuses of the build. # arches requires crossdev which isn't available at the early part of the # build. arches = [] for board in self._run.config['boards']: toolchains = toolchain.GetToolchainsForBoard(board, buildroot=src_root) default = list( toolchain.FilterToolchains(toolchains, 'default', True)) if default: try: arches.append(toolchain.GetArchForTarget(default[0])) except cros_build_lib.RunCommandError as e: logging.warning( 'Unable to retrieve arch for board %s default toolchain %s: %s', board, default, e) tags = { 'arches': arches, 'status': final_status, } results = self._run.attrs.metadata.GetValue('results') for stage in results: tags['stage_status:%s' % stage['name']] = stage['status'] tags['stage_summary:%s' % stage['name']] = stage['summary'] self._run.attrs.metadata.UpdateKeyDictWithDict(constants.METADATA_TAGS, tags) # Some operations can only be performed if a valid version is available. try: self._run.GetVersionInfo() self.ArchiveResults(final_status) metadata_url = os.path.join(self.upload_url, constants.METADATA_JSON) except cbuildbot_run.VersionNotSetError: logging.error('A valid version was never set for this run. ' 'Can not archive results.') metadata_url = '' results_lib.Results.Report(sys.stdout, current_version=(self._run.attrs.release_tag or '')) # Upload goma log if used for BuildPackage and TestSimpleChrome. _UploadAndLinkGomaLogIfNecessary( 'BuildPackages', self._run.config.name, self._run.options.goma_dir, self._run.options.goma_client_json, self._run.attrs.metadata.GetValueWithDefault('goma_tmp_dir')) _UploadAndLinkGomaLogIfNecessary( 'TestSimpleChromeWorkflow', self._run.config.name, self._run.options.goma_dir, self._run.options.goma_client_json, self._run.attrs.metadata.GetValueWithDefault( 'goma_tmp_dir_for_simple_chrome')) if self.buildstore.AreClientsReady(): status_for_db = final_status # TODO(pprabhu): After BuildData and CBuildbotMetdata are merged, remove # this extra temporary object creation. # XXX:HACK We're creating a BuildData with an empty URL. Don't try to # MarkGathered this object. build_data = metadata_lib.BuildData( '', self._run.attrs.metadata.GetDict()) # TODO(akeshet): Find a clearer way to get the "primary upload url" for # the metadata.json file. One alternative is _GetUploadUrls(...)[0]. # Today it seems that element 0 of its return list is the primary upload # url, but there is no guarantee or unit test coverage of that. self.buildstore.FinishBuild(build_id, status=status_for_db, summary=build_data.failure_message, metadata_url=metadata_url) duration = self._GetBuildDuration() mon_fields = { 'status': status_for_db, 'build_config': self._run.config.name, 'important': self._run.config.important } metrics.Counter( constants.MON_BUILD_COMP_COUNT).increment(fields=mon_fields) metrics.CumulativeSecondsDistribution( constants.MON_BUILD_DURATION).add(duration, fields=mon_fields) if self._run.options.sanity_check_build: metrics.Counter( constants.MON_BUILD_SANITY_COMP_COUNT).increment( fields=mon_fields) metrics.Gauge( constants.MON_BUILD_SANITY_ID, description= 'The build number of the latest sanity build. Used ' 'for recovering the link to the latest failing build ' 'in the alert when a sanity build fails.', field_spec=[ ts_mon.StringField('status'), ts_mon.StringField('build_config'), ts_mon.StringField('builder_name'), ts_mon.BooleanField('important') ]).set(self._run.buildnumber, fields=dict( mon_fields, builder_name=self._run.GetBuilderName())) if config_lib.IsMasterCQ(self._run.config): self_destructed = self._run.attrs.metadata.GetValueWithDefault( constants.SELF_DESTRUCTED_BUILD, False) mon_fields = { 'status': status_for_db, 'self_destructed': self_destructed } metrics.CumulativeSecondsDistribution( constants.MON_CQ_BUILD_DURATION).add(duration, fields=mon_fields) annotator_link = uri_lib.ConstructAnnotatorUri(build_id) logging.PrintBuildbotLink('Build annotator', annotator_link) # From this point forward, treat all exceptions as warnings. self._post_completion = True # Dump report about things we retry. retry_stats.ReportStats(sys.stdout)