def __call__(self): if (self._coordinator.generation < 0 or self._coordinator.need_rejoin()): # no need to send the heartbeat we're not using auto-assignment # or if we are awaiting a rebalance log.info("Skipping heartbeat: no auto-assignment" " or waiting on rebalance") return if self._coordinator.coordinator_unknown(): log.warning("Coordinator unknown during heartbeat -- will retry") self._handle_heartbeat_failure(Errors.GroupCoordinatorNotAvailableError()) return if self._heartbeat.session_expired(): # we haven't received a successful heartbeat in one session interval # so mark the coordinator dead log.error("Heartbeat session expired - marking coordinator dead") self._coordinator.coordinator_dead() return if not self._heartbeat.should_heartbeat(): # we don't need to heartbeat now, so reschedule for when we do ttl = self._heartbeat.ttl() log.debug("Heartbeat task unneeded now, retrying in %s", ttl) self._client.schedule(self, time.time() + ttl) else: self._heartbeat.sent_heartbeat() self._request_in_flight = True future = self._coordinator._send_heartbeat_request() future.add_callback(self._handle_heartbeat_success) future.add_errback(self._handle_heartbeat_failure)
def _send_join_group_request(self): """Join the group and return the assignment for the next generation. This function handles both JoinGroup and SyncGroup, delegating to _perform_assignment() if elected leader by the coordinator. Returns: Future: resolves to the encoded-bytes assignment returned from the group leader """ if self.coordinator_unknown(): e = Errors.GroupCoordinatorNotAvailableError(self.coordinator_id) return Future().failure(e) # send a join group request to the coordinator log.info("(Re-)joining group %s", self.group_id) request = JoinGroupRequest[0]( self.group_id, self.config['session_timeout_ms'], self.member_id, self.protocol_type(), [(protocol, metadata if isinstance(metadata, bytes) else metadata.encode()) for protocol, metadata in self.group_protocols()]) # create the request for the coordinator log.debug("Sending JoinGroup (%s) to coordinator %s", request, self.coordinator_id) future = Future() _f = self._client.send(self.coordinator_id, request) _f.add_callback(self._handle_join_group_response, future) _f.add_errback(self._failed_request, self.coordinator_id, request, future) return future
def _send_heartbeat_request(self): """Send a heartbeat request""" if self.coordinator_unknown(): e = Errors.GroupCoordinatorNotAvailableError(self.coordinator_id) return Future().failure(e) elif not self._client.ready(self.coordinator_id, metadata_priority=False): e = Errors.NodeNotReadyError(self.coordinator_id) return Future().failure(e) version = 0 if self.config["api_version"] < (0, 11, 0) else 1 request = HeartbeatRequest[version](self.group_id, self._generation.generation_id, self._generation.member_id) log.debug( "Heartbeat: %s[%s] %s", request.group, request.generation_id, request.member_id, ) # pylint: disable-msg=no-member future = Future() _f = self._client.send(self.coordinator_id, request) _f.add_callback(self._handle_heartbeat_response, future, time.time()) _f.add_errback(self._failed_request, self.coordinator_id, request, future) return future
def _send_join_group_request(self): """Join the group and return the assignment for the next generation. This function handles both JoinGroup and SyncGroup, delegating to :meth:`._perform_assignment` if elected leader by the coordinator. Returns: Future: resolves to the encoded-bytes assignment returned from the group leader """ if self.coordinator_unknown(): e = Errors.GroupCoordinatorNotAvailableError(self.coordinator_id) return Future().failure(e) elif not self._client.ready(self.coordinator_id, metadata_priority=False): e = Errors.NodeNotReadyError(self.coordinator_id) return Future().failure(e) # send a join group request to the coordinator log.info("(Re-)joining group %s", self.group_id) member_metadata = [ (protocol, metadata if isinstance(metadata, bytes) else metadata.encode()) for protocol, metadata in self.group_protocols() ] if self.config['api_version'] < (0, 9): raise Errors.KafkaError( 'JoinGroupRequest api requires 0.9+ brokers') elif (0, 9) <= self.config['api_version'] < (0, 10, 1): request = JoinGroupRequest[0](self.group_id, self.config['session_timeout_ms'], self._generation.member_id, self.protocol_type(), member_metadata) elif (0, 10, 1) <= self.config['api_version'] < (0, 11, 0): request = JoinGroupRequest[1](self.group_id, self.config['session_timeout_ms'], self.config['max_poll_interval_ms'], self._generation.member_id, self.protocol_type(), member_metadata) else: request = JoinGroupRequest[2](self.group_id, self.config['session_timeout_ms'], self.config['max_poll_interval_ms'], self._generation.member_id, self.protocol_type(), member_metadata) # create the request for the coordinator log.debug("Sending JoinGroup (%s) to coordinator %s", request, self.coordinator_id) future = Future() _f = self._client.send(self.coordinator_id, request) _f.add_callback(self._handle_join_group_response, future, time.time()) _f.add_errback(self._failed_request, self.coordinator_id, request, future) return future
def _send_sync_group_request(self, request): if self.coordinator_unknown(): e = Errors.GroupCoordinatorNotAvailableError(self.coordinator_id) return Future().failure(e) future = Future() _f = self._client.send(self.coordinator_id, request) _f.add_callback(self._handle_sync_group_response, future) _f.add_errback(self._failed_request, self.coordinator_id, request, future) return future
def _send_sync_group_request(self, request): if self.coordinator_unknown(): e = Errors.GroupCoordinatorNotAvailableError(self.coordinator_id) return Future().failure(e) # We assume that coordinator is ready if we're sending SyncGroup # as it typically follows a successful JoinGroup # Also note that if client.ready() enforces a metadata priority policy, # we can get into an infinite loop if the leader assignment process # itself requests a metadata update future = Future() _f = self._client.send(self.coordinator_id, request) _f.add_callback(self._handle_sync_group_response, future, time.time()) _f.add_errback(self._failed_request, self.coordinator_id, request, future) return future