def _ShouldWait(self):
"""Private helper with all the main logic of ShouldWait.
Returns:
A tuple of (bool indicating if we should wait,
bool indicating if slaves remain,
bool indicating if the final slave(s) to complete should
be considered the long-pole reason for terminating)
"""
self._completed_build_history.append(list(self.completed_builds))
uncompleted_experimental_build_buildbucket_ids = (
self._GetUncompletedExperimentalBuildbucketIDs())
# Check if all builders completed.
if self._Completed():
builder_status_lib.CancelBuilds(
list(uncompleted_experimental_build_buildbucket_ids),
self.buildbucket_client,
self.dry_run,
self.config)
return False, False, True
current_time = datetime.datetime.now()
uncompleted_important_builds = self._GetUncompletedBuilds(
self.completed_builds)
uncompleted_important_build_buildbucket_ids = set(
v.buildbucket_id
for k, v in self.all_buildbucket_info_dict.items() if k in
uncompleted_important_builds)
uncompleted_build_buildbucket_ids = list(
uncompleted_important_build_buildbucket_ids |
uncompleted_experimental_build_buildbucket_ids)
if self._ShouldFailForBuilderStartTimeout(current_time):
logging.error('Ending build since at least one builder has not started '
'within 5 mins.')
builder_status_lib.CancelBuilds(uncompleted_build_buildbucket_ids,
self.buildbucket_client,
self.dry_run,
self.config)
return False, False, False
# We got here which means no problems, we should still wait.
logging.info('Still waiting for the following builds to complete: %r',
sorted(set(self._GetExpectedBuilders()) -
self.completed_builds))
if self.builds_to_retry:
retried_builds = self._RetryBuilds(self.builds_to_retry)
self.builds_to_retry -= retried_builds
return True, True, False
def testCancelBuilds(self):
"""Test CancelBuilds."""
buildbucket_id_1 = '100'
buildbucket_id_2 = '200'
buildbucket_ids = [buildbucket_id_1, buildbucket_id_2]
cancel_mock = self.buildbucket_client.CancelBatchBuildsRequest
cancel_mock.return_value = dict()
builder_status_lib.CancelBuilds(buildbucket_ids,
self.buildbucket_client)
self.assertEqual(cancel_mock.call_count, 1)
def CancelObsoleteSlaveBuilds(self):
"""Cancel the obsolete slave builds scheduled by the previous master."""
logging.info('Cancelling obsolete slave builds.')
buildbucket_client = self.GetBuildbucketClient()
if not buildbucket_client:
logging.info('No buildbucket_client, not cancelling slaves.')
return
# Find the 3 most recent master buildbucket ids.
master_builds = buildbucket_client.SearchAllBuilds(
self._run.options.debug,
buckets=constants.ACTIVE_BUCKETS,
limit=3,
tags=[
'cbb_config:%s' % self._run.config.name,
'cbb_branch:%s' % self._run.manifest_branch
],
status=constants.BUILDBUCKET_BUILDER_STATUS_COMPLETED)
slave_ids = []
# Find the scheduled or started slaves for those master builds.
master_ids = buildbucket_lib.ExtractBuildIds(master_builds)
logging.info('Found Previous Master builds: %s', ', '.join(master_ids))
for master_id in master_ids:
for status in [
constants.BUILDBUCKET_BUILDER_STATUS_SCHEDULED,
constants.BUILDBUCKET_BUILDER_STATUS_STARTED
]:
builds = buildbucket_client.SearchAllBuilds(
self._run.options.debug,
tags=[
'buildset:%s' % request_build.SlaveBuildSet(master_id)
],
status=status)
ids = buildbucket_lib.ExtractBuildIds(builds)
if ids:
logging.info(
'Found builds %s in status %s from master %s.', ids,
status, master_id)
slave_ids.extend(ids)
if slave_ids:
builder_status_lib.CancelBuilds(slave_ids, buildbucket_client,
self._run.options.debug,
self._run.config)
def testCancelWithBuildbucketClient(self):
"""Test Buildbucket client cancels successfully during CancelBuilds."""
buildbucket_id_1 = '100'
buildbucket_id_2 = '200'
buildbucket_ids = [buildbucket_id_1, buildbucket_id_2]
self.PatchObject(buildbucket_lib,
'GetServiceAccount',
return_value=True)
send_request_mock = self.PatchObject(buildbucket_lib.BuildbucketClient,
'SendBuildbucketRequest',
return_value=dict())
buildbucket_client = buildbucket_lib.BuildbucketClient(
mock.Mock(), mock.Mock())
builder_status_lib.CancelBuilds(buildbucket_ids, buildbucket_client)
self.assertEqual(send_request_mock.call_count, 1)
def _ShouldWait(self):
"""Private helper with all the main logic of ShouldWait.
Returns:
A tuple of (bool indicating if we should wait,
bool indicating if slaves remain,
bool indicating if the final slave(s) to complete should
be considered the long-pole reason for terminating)
"""
self._completed_build_history.append(list(self.completed_builds))
uncompleted_experimental_build_buildbucket_ids = (
self._GetUncompletedExperimentalBuildbucketIDs())
# Check if all builders completed.
if self._Completed():
builder_status_lib.CancelBuilds(
list(uncompleted_experimental_build_buildbucket_ids),
self.buildbucket_client, self.dry_run, self.config)
return False, False, True
current_time = datetime.datetime.now()
uncompleted_important_builds = self._GetUncompletedBuilds(
self.completed_builds)
uncompleted_important_build_buildbucket_ids = set(
v.buildbucket_id
for k, v in self.all_buildbucket_info_dict.iteritems()
if k in uncompleted_important_builds)
uncompleted_build_buildbucket_ids = list(
uncompleted_important_build_buildbucket_ids
| uncompleted_experimental_build_buildbucket_ids)
if self._ShouldFailForBuilderStartTimeout(current_time):
logging.error(
'Ending build since at least one builder has not started '
'within 5 mins.')
builder_status_lib.CancelBuilds(uncompleted_build_buildbucket_ids,
self.buildbucket_client,
self.dry_run, self.config)
return False, False, False
if self.pool is not None:
triage_relevant_changes = relevant_changes.TriageRelevantChanges(
self.master_build_id,
self.db,
self._GetExpectedBuilders(),
self.config,
self.metadata,
self.version,
self.pool.build_root,
self.pool.applied,
self.all_buildbucket_info_dict,
self.all_cidb_status_dict,
self.completed_builds,
self.dependency_map,
self.buildbucket_client,
dry_run=self.dry_run)
should_self_destruct, should_self_destruct_with_success = (
triage_relevant_changes.ShouldSelfDestruct())
if should_self_destruct:
logging.warning(
'This build will self-destruct given the results of '
'relevant change triages.')
if should_self_destruct_with_success:
logging.info('This build will self-destruct with success.')
self.metadata.UpdateWithDict({
constants.SELF_DESTRUCTED_BUILD:
True,
constants.SELF_DESTRUCTED_WITH_SUCCESS_BUILD:
should_self_destruct_with_success
})
fields = {
'build_config':
self.config.name,
'self_destructed_with_success':
should_self_destruct_with_success
}
metrics.Counter(
constants.MON_CQ_SELF_DESTRUCTION_COUNT).increment(
fields=fields)
# For every uncompleted build, the master build will insert an
# ignored_reason message into the buildMessageTable.
for build in uncompleted_important_builds:
if build in self.all_cidb_status_dict:
self.db.InsertBuildMessage(
self.master_build_id,
message_type=constants.MESSAGE_TYPE_IGNORED_REASON,
message_subtype=constants.
MESSAGE_SUBTYPE_SELF_DESTRUCTION,
message_value=str(
self.all_cidb_status_dict[build].build_id))
builder_status_lib.CancelBuilds(
uncompleted_build_buildbucket_ids, self.buildbucket_client,
self.dry_run, self.config)
return False, False, True
# We got here which means no problems, we should still wait.
logging.info(
'Still waiting for the following builds to complete: %r',
sorted(set(self._GetExpectedBuilders()) - self.completed_builds))
if self.builds_to_retry:
retried_builds = self._RetryBuilds(self.builds_to_retry)
self.builds_to_retry -= retried_builds
return True, True, False
def testCancelNoBuilds(self):
"""Test CancelBuilds with no builds to cancel."""
cancel_mock = self.buildbucket_client.CancelBatchBuildsRequest
builder_status_lib.CancelBuilds([], self.buildbucket_client)
self.assertEqual(cancel_mock.call_count, 0)
