def WriteBasicMetadata(builder_run): """Writes basic metadata that should be known at start of execution. This method writes to |build_run|'s metadata instance the basic metadata values that should be known at the beginning of the first cbuildbot execution, prior to any reexecutions. In particular, this method does not write any metadata values that depend on the builder config, as the config may be modified by patches that are applied before the final reexectuion. (exception: the config's name itself) This method is safe to run more than once (for instance, once per cbuildbot execution) because it will write the same data each time. Args: builder_run: The BuilderRun instance for this build. """ start_time = results_lib.Results.start_time start_time_stamp = cros_build_lib.UserDateTimeFormat(timeval=start_time) metadata = { # Data for this build. 'bot-hostname': cros_build_lib.GetHostName(fully_qualified=True), 'build-number': builder_run.buildnumber, 'builder-name': builder_run.GetBuilderName(), 'buildbot-url': os.environ.get('BUILDBOT_BUILDBOTURL', ''), 'buildbot-master-name': os.environ.get('BUILDBOT_MASTERNAME', ''), 'bot-config': builder_run.config['name'], 'time': { 'start': start_time_stamp, }, 'master_build_id': builder_run.options.master_build_id, } builder_run.attrs.metadata.UpdateWithDict(metadata)
def testUserDateTimeDateTimeInEST(self): """Test that we correctly switch from PDT to EST.""" expected = 'Wed, 16 Jan 1980 00:00:00 -0500 (EST)' with cros_test_lib.SetTimeZone('US/Eastern'): timeval = datetime.datetime(1980, 1, 16) self.assertEqual( cros_build_lib.UserDateTimeFormat(timeval=timeval), expected)
def testUserDateTimeDateTime(self): """Test with a datetime object.""" expected = 'Mon, 16 Jun 1980 00:00:00 -0700 (PDT)' with cros_test_lib.SetTimeZone('US/Pacific'): timeval = datetime.datetime(1980, 6, 16) self.assertEqual( cros_build_lib.UserDateTimeFormat(timeval=timeval), expected)
def testUserDateTime(self): """Test with a raw time value.""" expected = 'Mon, 16 Jun 1980 05:03:20 -0700 (PDT)' with cros_test_lib.SetTimeZone('US/Pacific'): timeval = 330005000 self.assertEqual( cros_build_lib.UserDateTimeFormat(timeval=timeval), expected)
def testUserDateTimeDateTime(self): # cros_test_lib.TestCase takes care of saving/restoring the environ. os.environ['TZ'] = '0' time.tzset() timeval = datetime.datetime(1980, 6, 16) expected = 'Mon, 16 Jun 1980 00:00:00 +0000 ()' self.assertEqual(cros_build_lib.UserDateTimeFormat(timeval=timeval), expected)
def _Begin(self): """Called before a stage is performed. May be overriden.""" # Tell the buildbot we are starting a new step for the waterfall logging.PrintBuildbotStepName(self.name) self._PrintLoudly('Start Stage %s - %s\n\n%s' % ( self.name, cros_build_lib.UserDateTimeFormat(), self.__doc__))
def _BeginStepForBuildbot(self, tag=None): """Called before a stage is performed. Args: tag: Extra tag to add to the stage name on the waterfall. """ waterfall_name = self.name if tag is not None: waterfall_name += tag logging.PrintBuildbotStepName(waterfall_name) self._PrintLoudly( 'Start Stage %s - %s\n\n%s' % (self.name, cros_build_lib.UserDateTimeFormat(), self.__doc__))
def testUserDateTime(self): old_tz = os.environ.get('TZ') os.environ['TZ'] = '0' time.tzset() timeval = 330005000 expected = 'Mon, 16 Jun 1980 12:03:20 +0000 ()' try: self.assertEqual(cros_build_lib.UserDateTimeFormat(timeval=timeval), expected) finally: if old_tz: os.environ['TZ'] = old_tz else: os.environ.pop('TZ')
def CreateDummyMetadataJson(self): """Create/publish the firmware build artifact for the current board.""" workspace_version_info = self.GetWorkspaceVersionInfo() # Use the metadata for the main build, with selected fields modified. board_metadata = self._run.attrs.metadata.GetDict() board_metadata['boards'] = [self._current_board] board_metadata['branch'] = self._run.config.workspace_branch board_metadata['version_full'] = self.dummy_version board_metadata['version_milestone'] = \ workspace_version_info.chrome_branch board_metadata['version_platform'] = \ workspace_version_info.VersionString() board_metadata['version'] = { 'platform': workspace_version_info.VersionString(), 'full': self.dummy_version, 'milestone': workspace_version_info.chrome_branch, } current_time = datetime.datetime.now() current_time_stamp = cros_build_lib.UserDateTimeFormat( timeval=current_time) # We report the build as passing, since we can't get here if isn't. board_metadata['status'] = { 'status': 'pass', 'summary': '', 'current-time': current_time_stamp, } with osutils.TempDir(prefix='metadata') as tempdir: metadata_path = os.path.join(tempdir, constants.METADATA_JSON) logging.info('Writing metadata to %s.', metadata_path) osutils.WriteFile(metadata_path, json.dumps(board_metadata, indent=2, sort_keys=True), atomic=True) self.UploadDummyArtifact(metadata_path)
def _Finish(self): """Called after a stage has been performed. May be overriden.""" self._PrintLoudly('Finished Stage %s - %s' % (self.name, cros_build_lib.UserDateTimeFormat()))
def GetReportMetadataDict(builder_run, get_statuses_from_slaves, config=None, stage=None, final_status=None, completion_instance=None, child_configs_list=None): """Return a metadata dictionary summarizing a build. This method replaces code that used to exist in the ArchivingStageMixin class from cbuildbot_stage. It contains all the Report-stage-time metadata construction logic. The logic here is intended to be gradually refactored out so that the metadata is constructed gradually by the stages that are responsible for pieces of data, as they run. Args: builder_run: BuilderRun instance for this run. get_statuses_from_slaves: If True, status information of slave builders will be recorded. config: The build config for this run. Defaults to self._run.config. stage: The stage name that this metadata file is being uploaded for. final_status: Whether the build passed or failed. If None, the build will be treated as still running. completion_instance: The stage instance that was used to wait for slave completion. Used to add slave build information to master builder's metadata. If None, no such status information will be included. It not None, this should be a derivative of MasterSlaveSyncCompletionStage. child_configs_list: The list of child config metadata. If specified it should be added to the metadata. Returns: A metadata dictionary suitable to be json-serialized. """ config = config or builder_run.config start_time = results_lib.Results.start_time current_time = datetime.datetime.now() start_time_stamp = cros_build_lib.UserDateTimeFormat(timeval=start_time) current_time_stamp = cros_build_lib.UserDateTimeFormat(timeval=current_time) duration = '%s' % (current_time - start_time,) metadata = { 'status': { 'current-time': current_time_stamp, 'status': final_status if final_status else 'running', 'summary': stage or '', }, 'time': { 'start': start_time_stamp, 'finish': current_time_stamp if final_status else '', 'duration': duration, } } metadata['results'] = [] for entry in results_lib.Results.Get(): timestr = datetime.timedelta(seconds=math.ceil(entry.time)) if entry.result in results_lib.Results.NON_FAILURE_TYPES: status = constants.BUILDER_STATUS_PASSED else: status = constants.BUILDER_STATUS_FAILED metadata['results'].append({ 'name': entry.name, 'status': status, # The result might be a custom exception. 'summary': str(entry.result), 'duration': '%s' % timestr, 'board': entry.board, 'description': entry.description, 'log': builder_run.ConstructDashboardURL(stage=entry.name), }) if child_configs_list: metadata['child-configs'] = child_configs_list # If we were a CQ master, then include a summary of the status of slave cq # builders in metadata if get_statuses_from_slaves: statuses = completion_instance.GetSlaveStatuses() if not statuses: logging.warning('completion_instance did not have any statuses ' 'to report. Will not add slave status to metadata.') metadata['slave_targets'] = {} for builder, status in statuses.iteritems(): metadata['slave_targets'][builder] = status.AsFlatDict() return metadata
def testParseUserDateTimeFormat(self): stringtime = cros_build_lib.UserDateTimeFormat(100000.0) self.assertEqual(cros_build_lib.ParseUserDateTimeFormat(stringtime), 100000.0)
def testUserDateTimeCurrentTime(self): """Test that we can get the current time.""" cros_build_lib.UserDateTimeFormat()
def Run(self): """Have the builder execute the stage.""" skip_stage = self._ShouldSkipStage() previous_record = results_lib.Results.PreviouslyCompletedRecord( self.name) if skip_stage: self._BeginStepForBuildbot(' : [SKIPPED]') elif previous_record is not None: self._BeginStepForBuildbot(' : [PREVIOUSLY PROCESSED]') else: self._BeginStepForBuildbot() try: # Set default values result = None cidb_result = None description = None board = '' elapsed_time = None start_time = time.time() if skip_stage: self._StartBuildStageInCIDB() self._PrintLoudly('Not running Stage %s' % self.name) self.HandleSkip() result = results_lib.Results.SKIPPED return if previous_record: self._StartBuildStageInCIDB() self._PrintLoudly('Stage %s processed previously' % self.name) self.HandleSkip() # Success is stored in the results log for a stage that completed # successfully in a previous run. But, we report the truth to CIDB. result = results_lib.Results.SUCCESS cidb_result = constants.BUILDER_STATUS_SKIPPED # Copy over metadata from the previous record. instead of returning # metadata about the current run. board = previous_record.board elapsed_time = float(previous_record.time) return self._WaitBuildStageInCIDB() ready = self.WaitUntilReady() if not ready: self._PrintLoudly( 'Stage %s precondition failed while waiting to start.' % self.name) # If WaitUntilReady is false, mark stage as skipped in Results and CIDB result = results_lib.Results.SKIPPED return # Ready to start, mark buildStage as inflight in CIDB self._Print('Preconditions for the stage successfully met. ' 'Beginning to execute stage...') self._StartBuildStageInCIDB() start_time = time.time() sys.stdout.flush() sys.stderr.flush() # TODO(davidjames): Verify that PerformStage always returns None. See # crbug.com/264781 self.PerformStage() result = results_lib.Results.SUCCESS except SystemExit as e: if e.code != 0: result, description, _ = self._TopHandleStageException() raise except Exception as e: if isinstance(e, failures_lib.ExitEarlyException): # One stage finished and exited early, not a failure. result = results_lib.Results.SUCCESS raise # Tell the build bot this step failed for the waterfall. result, description, retrying = self._TopHandleStageException() if result not in (results_lib.Results.FORGIVEN, results_lib.Results.SUCCESS): if isinstance(e, failures_lib.StepFailure): raise else: raise failures_lib.StepFailure() elif retrying: raise failures_lib.RetriableStepFailure() except BaseException: result, description, _ = self._TopHandleStageException() raise finally: # Some cases explicitly set a cidb status. For others, infer. if cidb_result is None: cidb_result = self._TranslateResultToCIDBStatus(result) if elapsed_time is None: elapsed_time = time.time() - start_time self._RecordResult(self.name, result, description, prefix=self._prefix, board=board, time=elapsed_time, build_stage_id=self._build_stage_id) self._FinishBuildStageInCIDBAndMonarch(cidb_result, elapsed_time) if isinstance(result, BaseException) and self._build_stage_id is not None: _, db = self._run.GetCIDBHandle() if db: failures_lib.ReportStageFailure( db, self._build_stage_id, result, build_config=self.build_config) try: self.Finish() except Exception as e: # Failures here are OUTSIDE of the stage and not handled well. Log and # continue with the assumption that the ReportStage will re-upload this # data or report a failure correctly. logging.warning('IGNORED: Finish failure: %s', e) self._PrintLoudly('Finished Stage %s - %s' % (self.name, cros_build_lib.UserDateTimeFormat())) sys.stdout.flush() sys.stderr.flush()