def UpdateSlaveStatus(self):
"""Update slave statuses by querying CIDB and Buildbucket(if supported)."""
logging.info('Updating slave status...')
if self.config and self.metadata:
scheduled_buildbucket_info_dict = buildbucket_lib.GetBuildInfoDict(
self.metadata)
# It's possible that CQ-master has a list of important slaves configured
# but doesn't schedule any slaves as no CLs were picked up in SyncStage.
# These are set to include only important builds.
self.all_builders = list(scheduled_buildbucket_info_dict)
self.all_buildbucket_info_dict = (
builder_status_lib.SlaveBuilderStatus.
GetAllSlaveBuildbucketInfo(self.buildbucket_client,
scheduled_buildbucket_info_dict,
dry_run=self.dry_run))
self.new_buildbucket_info_dict = self._GetNewSlaveBuildbucketInfo(
self.all_buildbucket_info_dict, self.completed_builds)
self._SetStatusBuildsDict()
self.all_cidb_status_dict = (
builder_status_lib.SlaveBuilderStatus.GetAllSlaveCIDBStatusInfo(
self.buildstore, self.master_build_identifier,
self.all_buildbucket_info_dict))
self.new_cidb_status_dict = self._GetNewSlaveCIDBStatusInfo(
self.all_cidb_status_dict, self.completed_builds)
self.missing_builds = self._GetMissingBuilds()
self.scheduled_builds = self._GetScheduledBuilds()
self.builds_to_retry = self._GetBuildsToRetry()
self.completed_builds = self._GetCompletedBuilds()
def testGetBuildInfoDict(self):
"""Test GetBuildInfoDict with metadata and config."""
metadata = metadata_lib.CBuildbotMetadata()
slaves = [('config_1', 'bb_id_1', 0), ('config_1', 'bb_id_2', 1),
('config_2', 'bb_id_3', 2)]
metadata.ExtendKeyListWithList(
constants.METADATA_SCHEDULED_IMPORTANT_SLAVES, slaves)
buildbucket_info_dict = buildbucket_lib.GetBuildInfoDict(metadata)
self.assertEqual(len(buildbucket_info_dict), 2)
self.assertEqual(
buildbucket_info_dict['config_1'],
buildbucket_lib.BuildbucketInfo('bb_id_2', 1, 1, None, None, None))
self.assertEqual(
buildbucket_info_dict['config_2'],
buildbucket_lib.BuildbucketInfo('bb_id_3', 0, 2, None, None, None))
buildbucket_info_dict_with_experimental = (
buildbucket_lib.GetBuildInfoDict(metadata,
exclude_experimental=False))
self.assertEqual(len(buildbucket_info_dict), 2)
self.assertEqual(
buildbucket_info_dict_with_experimental['config_1'],
buildbucket_lib.BuildbucketInfo('bb_id_2', 1, 1, None, None, None))
self.assertEqual(
buildbucket_info_dict_with_experimental['config_2'],
buildbucket_lib.BuildbucketInfo('bb_id_3', 0, 2, None, None, None))
metadata.UpdateWithDict(
{constants.METADATA_EXPERIMENTAL_BUILDERS: ['config_2']})
buildbucket_info_dict = buildbucket_lib.GetBuildInfoDict(metadata)
self.assertEqual(len(buildbucket_info_dict), 1)
self.assertEqual(
buildbucket_info_dict['config_1'],
buildbucket_lib.BuildbucketInfo('bb_id_2', 1, 1, None, None, None))
buildbucket_info_dict_with_experimental = (
buildbucket_lib.GetBuildInfoDict(metadata,
exclude_experimental=False))
self.assertEqual(len(buildbucket_info_dict_with_experimental), 2)
self.assertEqual(
buildbucket_info_dict_with_experimental['config_1'],
buildbucket_lib.BuildbucketInfo('bb_id_2', 1, 1, None, None, None))
self.assertEqual(
buildbucket_info_dict_with_experimental['config_2'],
buildbucket_lib.BuildbucketInfo('bb_id_3', 0, 2, None, None, None))
def UpdateSlaveStatus(self):
"""Update slave statuses by querying CIDB and Buildbucket(if supported)."""
logging.info('Updating slave status...')
# Fetch experimental builders from tree status and update experimental
# builders in metedata before querying and updating any slave status.
if self.metadata is not None:
try:
experimental_builders = tree_status.GetExperimentalBuilders()
self.metadata.UpdateWithDict({
constants.METADATA_EXPERIMENTAL_BUILDERS:
experimental_builders
})
except timeout_util.TimeoutError:
logging.error(
'Timeout getting experimental builders from the tree'
'status. Not updating metadata.')
# If a slave build was important in previous loop and got added to the
# completed_builds because it completed, but in the current loop it's
# marked as experimental, take it out from the completed_builds list.
self.completed_builds = set([
build for build in self.completed_builds
if build not in experimental_builders
])
if (self.config is not None and self.metadata is not None
and config_lib.UseBuildbucketScheduler(self.config)):
scheduled_buildbucket_info_dict = buildbucket_lib.GetBuildInfoDict(
self.metadata)
# It's possible that CQ-master has a list of important slaves configured
# but doesn't schedule any slaves as no CLs were picked up in SyncStage.
# These are set to include only important builds.
self.all_builders = scheduled_buildbucket_info_dict.keys()
self.all_buildbucket_info_dict = (
builder_status_lib.SlaveBuilderStatus.
GetAllSlaveBuildbucketInfo(self.buildbucket_client,
scheduled_buildbucket_info_dict,
dry_run=self.dry_run))
self.new_buildbucket_info_dict = self._GetNewSlaveBuildbucketInfo(
self.all_buildbucket_info_dict, self.completed_builds)
self._SetStatusBuildsDict()
self.all_cidb_status_dict = (
builder_status_lib.SlaveBuilderStatus.GetAllSlaveCIDBStatusInfo(
self.db, self.master_build_id, self.all_buildbucket_info_dict))
self.new_cidb_status_dict = self._GetNewSlaveCIDBStatusInfo(
self.all_cidb_status_dict, self.completed_builds)
self.missing_builds = self._GetMissingBuilds()
self.scheduled_builds = self._GetScheduledBuilds()
self.builds_to_retry = self._GetBuildsToRetry()
self.completed_builds = self._GetCompletedBuilds()
def _AnnotateNoStatBuilders(self, no_stat):
"""Annotate the build statuses fetched from the Buildbucket.
Some builds may fail to upload statuses to GS. If the builds were
scheduled by Buildbucket, get the build statuses and annotate the results.
Args:
no_stat: Config names of the slave builds with None status.
"""
buildbucket_info_dict = buildbucket_lib.GetBuildInfoDict(
self._run.attrs.metadata)
for config_name in no_stat:
if config_name in buildbucket_info_dict:
buildbucket_id = buildbucket_info_dict[
config_name].buildbucket_id
assert buildbucket_id is not None, 'buildbucket_id is None'
try:
content = self.buildbucket_client.GetBuildRequest(
buildbucket_id, self._run.options.debug)
status = buildbucket_lib.GetBuildStatus(content)
result = buildbucket_lib.GetBuildResult(content)
text = '%s: [status] %s [result] %s' % (config_name,
status, result)
if result == constants.BUILDBUCKET_BUILDER_RESULT_FAILURE:
failure_reason = buildbucket_lib.GetBuildFailureReason(
content)
if failure_reason:
text += ' [failure_reason] %s' % failure_reason
elif result == constants.BUILDBUCKET_BUILDER_RESULT_CANCELED:
cancel_reason = buildbucket_lib.GetBuildCancelationReason(
content)
if cancel_reason:
text += ' [cancelation_reason] %s' % cancel_reason
dashboard_url = buildbucket_lib.GetBuildURL(content)
if dashboard_url:
logging.PrintBuildbotLink(text, dashboard_url)
else:
logging.PrintBuildbotStepText(text)
except buildbucket_lib.BuildbucketResponseException as e:
logging.error('Cannot get status for %s: %s', config_name,
e)
logging.PrintBuildbotStepText(
'No status found for build %s buildbucket_id %s' %
(config_name, buildbucket_id))
else:
logging.PrintBuildbotStepText(
"%s wasn't scheduled by master." % config_name)
def _InitSlaveInfo(self):
"""Init slave info including buildbucket info, cidb info and failures."""
scheduled_buildbucket_info_dict = buildbucket_lib.GetBuildInfoDict(
self.metadata, exclude_experimental=self.exclude_experimental)
self.buildbucket_info_dict = self.GetAllSlaveBuildbucketInfo(
self.buildbucket_client, scheduled_buildbucket_info_dict,
dry_run=self.dry_run)
self.builders_array = list(self.buildbucket_info_dict)
self.cidb_info_dict = self.GetAllSlaveCIDBStatusInfo(
self.buildstore, self.master_build_identifier,
self.buildbucket_info_dict)
self.slave_failures_dict = self._GetSlaveFailures(
self.buildbucket_info_dict)
self.aborted_slaves = self._GetSlavesAbortedBySelfDestruction(
self.cidb_info_dict)