def _ShouldWait(self): """Private helper with all the main logic of ShouldWait. Returns: A tuple of (bool indicating if we should wait, bool indicating if slaves remain, bool indicating if the final slave(s) to complete should be considered the long-pole reason for terminating) """ self._completed_build_history.append(list(self.completed_builds)) uncompleted_experimental_build_buildbucket_ids = ( self._GetUncompletedExperimentalBuildbucketIDs()) # Check if all builders completed. if self._Completed(): builder_status_lib.CancelBuilds( list(uncompleted_experimental_build_buildbucket_ids), self.buildbucket_client, self.dry_run, self.config) return False, False, True current_time = datetime.datetime.now() uncompleted_important_builds = self._GetUncompletedBuilds( self.completed_builds) uncompleted_important_build_buildbucket_ids = set( v.buildbucket_id for k, v in self.all_buildbucket_info_dict.items() if k in uncompleted_important_builds) uncompleted_build_buildbucket_ids = list( uncompleted_important_build_buildbucket_ids | uncompleted_experimental_build_buildbucket_ids) if self._ShouldFailForBuilderStartTimeout(current_time): logging.error('Ending build since at least one builder has not started ' 'within 5 mins.') builder_status_lib.CancelBuilds(uncompleted_build_buildbucket_ids, self.buildbucket_client, self.dry_run, self.config) return False, False, False # We got here which means no problems, we should still wait. logging.info('Still waiting for the following builds to complete: %r', sorted(set(self._GetExpectedBuilders()) - self.completed_builds)) if self.builds_to_retry: retried_builds = self._RetryBuilds(self.builds_to_retry) self.builds_to_retry -= retried_builds return True, True, False
def testCancelBuilds(self): """Test CancelBuilds.""" buildbucket_id_1 = '100' buildbucket_id_2 = '200' buildbucket_ids = [buildbucket_id_1, buildbucket_id_2] cancel_mock = self.buildbucket_client.CancelBatchBuildsRequest cancel_mock.return_value = dict() builder_status_lib.CancelBuilds(buildbucket_ids, self.buildbucket_client) self.assertEqual(cancel_mock.call_count, 1)
def CancelObsoleteSlaveBuilds(self): """Cancel the obsolete slave builds scheduled by the previous master.""" logging.info('Cancelling obsolete slave builds.') buildbucket_client = self.GetBuildbucketClient() if not buildbucket_client: logging.info('No buildbucket_client, not cancelling slaves.') return # Find the 3 most recent master buildbucket ids. master_builds = buildbucket_client.SearchAllBuilds( self._run.options.debug, buckets=constants.ACTIVE_BUCKETS, limit=3, tags=[ 'cbb_config:%s' % self._run.config.name, 'cbb_branch:%s' % self._run.manifest_branch ], status=constants.BUILDBUCKET_BUILDER_STATUS_COMPLETED) slave_ids = [] # Find the scheduled or started slaves for those master builds. master_ids = buildbucket_lib.ExtractBuildIds(master_builds) logging.info('Found Previous Master builds: %s', ', '.join(master_ids)) for master_id in master_ids: for status in [ constants.BUILDBUCKET_BUILDER_STATUS_SCHEDULED, constants.BUILDBUCKET_BUILDER_STATUS_STARTED ]: builds = buildbucket_client.SearchAllBuilds( self._run.options.debug, tags=[ 'buildset:%s' % request_build.SlaveBuildSet(master_id) ], status=status) ids = buildbucket_lib.ExtractBuildIds(builds) if ids: logging.info( 'Found builds %s in status %s from master %s.', ids, status, master_id) slave_ids.extend(ids) if slave_ids: builder_status_lib.CancelBuilds(slave_ids, buildbucket_client, self._run.options.debug, self._run.config)
def testCancelWithBuildbucketClient(self): """Test Buildbucket client cancels successfully during CancelBuilds.""" buildbucket_id_1 = '100' buildbucket_id_2 = '200' buildbucket_ids = [buildbucket_id_1, buildbucket_id_2] self.PatchObject(buildbucket_lib, 'GetServiceAccount', return_value=True) send_request_mock = self.PatchObject(buildbucket_lib.BuildbucketClient, 'SendBuildbucketRequest', return_value=dict()) buildbucket_client = buildbucket_lib.BuildbucketClient( mock.Mock(), mock.Mock()) builder_status_lib.CancelBuilds(buildbucket_ids, buildbucket_client) self.assertEqual(send_request_mock.call_count, 1)
def _ShouldWait(self): """Private helper with all the main logic of ShouldWait. Returns: A tuple of (bool indicating if we should wait, bool indicating if slaves remain, bool indicating if the final slave(s) to complete should be considered the long-pole reason for terminating) """ self._completed_build_history.append(list(self.completed_builds)) uncompleted_experimental_build_buildbucket_ids = ( self._GetUncompletedExperimentalBuildbucketIDs()) # Check if all builders completed. if self._Completed(): builder_status_lib.CancelBuilds( list(uncompleted_experimental_build_buildbucket_ids), self.buildbucket_client, self.dry_run, self.config) return False, False, True current_time = datetime.datetime.now() uncompleted_important_builds = self._GetUncompletedBuilds( self.completed_builds) uncompleted_important_build_buildbucket_ids = set( v.buildbucket_id for k, v in self.all_buildbucket_info_dict.iteritems() if k in uncompleted_important_builds) uncompleted_build_buildbucket_ids = list( uncompleted_important_build_buildbucket_ids | uncompleted_experimental_build_buildbucket_ids) if self._ShouldFailForBuilderStartTimeout(current_time): logging.error( 'Ending build since at least one builder has not started ' 'within 5 mins.') builder_status_lib.CancelBuilds(uncompleted_build_buildbucket_ids, self.buildbucket_client, self.dry_run, self.config) return False, False, False if self.pool is not None: triage_relevant_changes = relevant_changes.TriageRelevantChanges( self.master_build_id, self.db, self._GetExpectedBuilders(), self.config, self.metadata, self.version, self.pool.build_root, self.pool.applied, self.all_buildbucket_info_dict, self.all_cidb_status_dict, self.completed_builds, self.dependency_map, self.buildbucket_client, dry_run=self.dry_run) should_self_destruct, should_self_destruct_with_success = ( triage_relevant_changes.ShouldSelfDestruct()) if should_self_destruct: logging.warning( 'This build will self-destruct given the results of ' 'relevant change triages.') if should_self_destruct_with_success: logging.info('This build will self-destruct with success.') self.metadata.UpdateWithDict({ constants.SELF_DESTRUCTED_BUILD: True, constants.SELF_DESTRUCTED_WITH_SUCCESS_BUILD: should_self_destruct_with_success }) fields = { 'build_config': self.config.name, 'self_destructed_with_success': should_self_destruct_with_success } metrics.Counter( constants.MON_CQ_SELF_DESTRUCTION_COUNT).increment( fields=fields) # For every uncompleted build, the master build will insert an # ignored_reason message into the buildMessageTable. for build in uncompleted_important_builds: if build in self.all_cidb_status_dict: self.db.InsertBuildMessage( self.master_build_id, message_type=constants.MESSAGE_TYPE_IGNORED_REASON, message_subtype=constants. MESSAGE_SUBTYPE_SELF_DESTRUCTION, message_value=str( self.all_cidb_status_dict[build].build_id)) builder_status_lib.CancelBuilds( uncompleted_build_buildbucket_ids, self.buildbucket_client, self.dry_run, self.config) return False, False, True # We got here which means no problems, we should still wait. logging.info( 'Still waiting for the following builds to complete: %r', sorted(set(self._GetExpectedBuilders()) - self.completed_builds)) if self.builds_to_retry: retried_builds = self._RetryBuilds(self.builds_to_retry) self.builds_to_retry -= retried_builds return True, True, False
def testCancelNoBuilds(self): """Test CancelBuilds with no builds to cancel.""" cancel_mock = self.buildbucket_client.CancelBatchBuildsRequest builder_status_lib.CancelBuilds([], self.buildbucket_client) self.assertEqual(cancel_mock.call_count, 0)