def WriteBasicMetadata(builder_run):
"""Writes basic metadata that should be known at start of execution.
This method writes to |build_run|'s metadata instance the basic metadata
values that should be known at the beginning of the first cbuildbot
execution, prior to any reexecutions.
In particular, this method does not write any metadata values that depend
on the builder config, as the config may be modified by patches that are
applied before the final reexectuion. (exception: the config's name itself)
This method is safe to run more than once (for instance, once per cbuildbot
execution) because it will write the same data each time.
Args:
builder_run: The BuilderRun instance for this build.
"""
start_time = results_lib.Results.start_time
start_time_stamp = cros_build_lib.UserDateTimeFormat(timeval=start_time)
metadata = {
# Data for this build.
'bot-hostname': cros_build_lib.GetHostName(fully_qualified=True),
'build-number': builder_run.buildnumber,
'builder-name': builder_run.GetBuilderName(),
'buildbot-url': os.environ.get('BUILDBOT_BUILDBOTURL', ''),
'buildbot-master-name': os.environ.get('BUILDBOT_MASTERNAME', ''),
'bot-config': builder_run.config['name'],
'time': {
'start': start_time_stamp,
},
'master_build_id': builder_run.options.master_build_id,
}
builder_run.attrs.metadata.UpdateWithDict(metadata)
def testUserDateTimeDateTimeInEST(self):
"""Test that we correctly switch from PDT to EST."""
expected = 'Wed, 16 Jan 1980 00:00:00 -0500 (EST)'
with cros_test_lib.SetTimeZone('US/Eastern'):
timeval = datetime.datetime(1980, 1, 16)
self.assertEqual(
cros_build_lib.UserDateTimeFormat(timeval=timeval), expected)
def testUserDateTimeDateTime(self):
"""Test with a datetime object."""
expected = 'Mon, 16 Jun 1980 00:00:00 -0700 (PDT)'
with cros_test_lib.SetTimeZone('US/Pacific'):
timeval = datetime.datetime(1980, 6, 16)
self.assertEqual(
cros_build_lib.UserDateTimeFormat(timeval=timeval), expected)
def testUserDateTime(self):
"""Test with a raw time value."""
expected = 'Mon, 16 Jun 1980 05:03:20 -0700 (PDT)'
with cros_test_lib.SetTimeZone('US/Pacific'):
timeval = 330005000
self.assertEqual(
cros_build_lib.UserDateTimeFormat(timeval=timeval), expected)
def testUserDateTimeDateTime(self):
# cros_test_lib.TestCase takes care of saving/restoring the environ.
os.environ['TZ'] = '0'
time.tzset()
timeval = datetime.datetime(1980, 6, 16)
expected = 'Mon, 16 Jun 1980 00:00:00 +0000 ()'
self.assertEqual(cros_build_lib.UserDateTimeFormat(timeval=timeval),
expected)
def _Begin(self):
"""Called before a stage is performed. May be overriden."""
# Tell the buildbot we are starting a new step for the waterfall
logging.PrintBuildbotStepName(self.name)
self._PrintLoudly('Start Stage %s - %s\n\n%s' % (
self.name, cros_build_lib.UserDateTimeFormat(), self.__doc__))
def _BeginStepForBuildbot(self, tag=None):
"""Called before a stage is performed.
Args:
tag: Extra tag to add to the stage name on the waterfall.
"""
waterfall_name = self.name
if tag is not None:
waterfall_name += tag
logging.PrintBuildbotStepName(waterfall_name)
self._PrintLoudly(
'Start Stage %s - %s\n\n%s' %
(self.name, cros_build_lib.UserDateTimeFormat(), self.__doc__))
def testUserDateTime(self):
old_tz = os.environ.get('TZ')
os.environ['TZ'] = '0'
time.tzset()
timeval = 330005000
expected = 'Mon, 16 Jun 1980 12:03:20 +0000 ()'
try:
self.assertEqual(cros_build_lib.UserDateTimeFormat(timeval=timeval),
expected)
finally:
if old_tz:
os.environ['TZ'] = old_tz
else:
os.environ.pop('TZ')
def CreateDummyMetadataJson(self):
"""Create/publish the firmware build artifact for the current board."""
workspace_version_info = self.GetWorkspaceVersionInfo()
# Use the metadata for the main build, with selected fields modified.
board_metadata = self._run.attrs.metadata.GetDict()
board_metadata['boards'] = [self._current_board]
board_metadata['branch'] = self._run.config.workspace_branch
board_metadata['version_full'] = self.dummy_version
board_metadata['version_milestone'] = \
workspace_version_info.chrome_branch
board_metadata['version_platform'] = \
workspace_version_info.VersionString()
board_metadata['version'] = {
'platform': workspace_version_info.VersionString(),
'full': self.dummy_version,
'milestone': workspace_version_info.chrome_branch,
}
current_time = datetime.datetime.now()
current_time_stamp = cros_build_lib.UserDateTimeFormat(
timeval=current_time)
# We report the build as passing, since we can't get here if isn't.
board_metadata['status'] = {
'status': 'pass',
'summary': '',
'current-time': current_time_stamp,
}
with osutils.TempDir(prefix='metadata') as tempdir:
metadata_path = os.path.join(tempdir, constants.METADATA_JSON)
logging.info('Writing metadata to %s.', metadata_path)
osutils.WriteFile(metadata_path,
json.dumps(board_metadata,
indent=2,
sort_keys=True),
atomic=True)
self.UploadDummyArtifact(metadata_path)
def _Finish(self):
"""Called after a stage has been performed. May be overriden."""
self._PrintLoudly('Finished Stage %s - %s' %
(self.name, cros_build_lib.UserDateTimeFormat()))
def GetReportMetadataDict(builder_run, get_statuses_from_slaves,
config=None, stage=None, final_status=None,
completion_instance=None, child_configs_list=None):
"""Return a metadata dictionary summarizing a build.
This method replaces code that used to exist in the ArchivingStageMixin
class from cbuildbot_stage. It contains all the Report-stage-time
metadata construction logic. The logic here is intended to be gradually
refactored out so that the metadata is constructed gradually by the
stages that are responsible for pieces of data, as they run.
Args:
builder_run: BuilderRun instance for this run.
get_statuses_from_slaves: If True, status information of slave
builders will be recorded.
config: The build config for this run. Defaults to self._run.config.
stage: The stage name that this metadata file is being uploaded for.
final_status: Whether the build passed or failed. If None, the build
will be treated as still running.
completion_instance: The stage instance that was used to wait for slave
completion. Used to add slave build information to
master builder's metadata. If None, no such status
information will be included. It not None, this
should be a derivative of
MasterSlaveSyncCompletionStage.
child_configs_list: The list of child config metadata. If specified it
should be added to the metadata.
Returns:
A metadata dictionary suitable to be json-serialized.
"""
config = config or builder_run.config
start_time = results_lib.Results.start_time
current_time = datetime.datetime.now()
start_time_stamp = cros_build_lib.UserDateTimeFormat(timeval=start_time)
current_time_stamp = cros_build_lib.UserDateTimeFormat(timeval=current_time)
duration = '%s' % (current_time - start_time,)
metadata = {
'status': {
'current-time': current_time_stamp,
'status': final_status if final_status else 'running',
'summary': stage or '',
},
'time': {
'start': start_time_stamp,
'finish': current_time_stamp if final_status else '',
'duration': duration,
}
}
metadata['results'] = []
for entry in results_lib.Results.Get():
timestr = datetime.timedelta(seconds=math.ceil(entry.time))
if entry.result in results_lib.Results.NON_FAILURE_TYPES:
status = constants.BUILDER_STATUS_PASSED
else:
status = constants.BUILDER_STATUS_FAILED
metadata['results'].append({
'name': entry.name,
'status': status,
# The result might be a custom exception.
'summary': str(entry.result),
'duration': '%s' % timestr,
'board': entry.board,
'description': entry.description,
'log': builder_run.ConstructDashboardURL(stage=entry.name),
})
if child_configs_list:
metadata['child-configs'] = child_configs_list
# If we were a CQ master, then include a summary of the status of slave cq
# builders in metadata
if get_statuses_from_slaves:
statuses = completion_instance.GetSlaveStatuses()
if not statuses:
logging.warning('completion_instance did not have any statuses '
'to report. Will not add slave status to metadata.')
metadata['slave_targets'] = {}
for builder, status in statuses.iteritems():
metadata['slave_targets'][builder] = status.AsFlatDict()
return metadata
def testParseUserDateTimeFormat(self):
stringtime = cros_build_lib.UserDateTimeFormat(100000.0)
self.assertEqual(cros_build_lib.ParseUserDateTimeFormat(stringtime),
100000.0)
def testUserDateTimeCurrentTime(self):
"""Test that we can get the current time."""
cros_build_lib.UserDateTimeFormat()
def Run(self):
"""Have the builder execute the stage."""
skip_stage = self._ShouldSkipStage()
previous_record = results_lib.Results.PreviouslyCompletedRecord(
self.name)
if skip_stage:
self._BeginStepForBuildbot(' : [SKIPPED]')
elif previous_record is not None:
self._BeginStepForBuildbot(' : [PREVIOUSLY PROCESSED]')
else:
self._BeginStepForBuildbot()
try:
# Set default values
result = None
cidb_result = None
description = None
board = ''
elapsed_time = None
start_time = time.time()
if skip_stage:
self._StartBuildStageInCIDB()
self._PrintLoudly('Not running Stage %s' % self.name)
self.HandleSkip()
result = results_lib.Results.SKIPPED
return
if previous_record:
self._StartBuildStageInCIDB()
self._PrintLoudly('Stage %s processed previously' % self.name)
self.HandleSkip()
# Success is stored in the results log for a stage that completed
# successfully in a previous run. But, we report the truth to CIDB.
result = results_lib.Results.SUCCESS
cidb_result = constants.BUILDER_STATUS_SKIPPED
# Copy over metadata from the previous record. instead of returning
# metadata about the current run.
board = previous_record.board
elapsed_time = float(previous_record.time)
return
self._WaitBuildStageInCIDB()
ready = self.WaitUntilReady()
if not ready:
self._PrintLoudly(
'Stage %s precondition failed while waiting to start.' %
self.name)
# If WaitUntilReady is false, mark stage as skipped in Results and CIDB
result = results_lib.Results.SKIPPED
return
# Ready to start, mark buildStage as inflight in CIDB
self._Print('Preconditions for the stage successfully met. '
'Beginning to execute stage...')
self._StartBuildStageInCIDB()
start_time = time.time()
sys.stdout.flush()
sys.stderr.flush()
# TODO(davidjames): Verify that PerformStage always returns None. See
# crbug.com/264781
self.PerformStage()
result = results_lib.Results.SUCCESS
except SystemExit as e:
if e.code != 0:
result, description, _ = self._TopHandleStageException()
raise
except Exception as e:
if isinstance(e, failures_lib.ExitEarlyException):
# One stage finished and exited early, not a failure.
result = results_lib.Results.SUCCESS
raise
# Tell the build bot this step failed for the waterfall.
result, description, retrying = self._TopHandleStageException()
if result not in (results_lib.Results.FORGIVEN,
results_lib.Results.SUCCESS):
if isinstance(e, failures_lib.StepFailure):
raise
else:
raise failures_lib.StepFailure()
elif retrying:
raise failures_lib.RetriableStepFailure()
except BaseException:
result, description, _ = self._TopHandleStageException()
raise
finally:
# Some cases explicitly set a cidb status. For others, infer.
if cidb_result is None:
cidb_result = self._TranslateResultToCIDBStatus(result)
if elapsed_time is None:
elapsed_time = time.time() - start_time
self._RecordResult(self.name,
result,
description,
prefix=self._prefix,
board=board,
time=elapsed_time,
build_stage_id=self._build_stage_id)
self._FinishBuildStageInCIDBAndMonarch(cidb_result, elapsed_time)
if isinstance(result,
BaseException) and self._build_stage_id is not None:
_, db = self._run.GetCIDBHandle()
if db:
failures_lib.ReportStageFailure(
db,
self._build_stage_id,
result,
build_config=self.build_config)
try:
self.Finish()
except Exception as e:
# Failures here are OUTSIDE of the stage and not handled well. Log and
# continue with the assumption that the ReportStage will re-upload this
# data or report a failure correctly.
logging.warning('IGNORED: Finish failure: %s', e)
self._PrintLoudly('Finished Stage %s - %s' %
(self.name, cros_build_lib.UserDateTimeFormat()))
sys.stdout.flush()
sys.stderr.flush()
