def _process_response(self, read_buffer): assert not self._processing, 'Recursion not supported' self._processing = True ifr = self.in_flight_requests.popleft() if self._sensors: self._sensors.request_time.record( (time.time() - ifr.timestamp) * 1000) # verify send/recv correlation ids match recv_correlation_id = Int32.decode(read_buffer) # 0.8.2 quirk if (self.config['api_version'] == (0, 8, 2) and ifr.response_type is GroupCoordinatorResponse[0] and ifr.correlation_id != 0 and recv_correlation_id == 0): log.warning('Kafka 0.8.2 quirk -- GroupCoordinatorResponse' ' Correlation ID does not match request. This' ' should go away once at least one topic has been' ' initialized on the broker.') elif ifr.correlation_id != recv_correlation_id: error = Errors.CorrelationIdError( '%s: Correlation IDs do not match: sent %d, recv %d' % (self, ifr.correlation_id, recv_correlation_id)) ifr.future.failure(error) self.close(error) self._processing = False return None # decode response try: response = ifr.response_type.decode(read_buffer) except ValueError: read_buffer.seek(0) buf = read_buffer.read() log.error( '%s Response %d [ResponseType: %s Request: %s]:' ' Unable to decode %d-byte buffer: %r', self, ifr.correlation_id, ifr.response_type, ifr.request, len(buf), buf) error = Errors.UnknownError('Unable to decode response') ifr.future.failure(error) self.close(error) self._processing = False return None log.debug('%s Response %d: %s', self, ifr.correlation_id, response) ifr.future.success(response) self._processing = False return response
def await_flush_completion(self, timeout=None): """ Mark all partitions as ready to send and block until the send is complete """ try: for batch in self._incomplete.all(): log.debug('Waiting on produce to %s', batch.produce_future.topic_partition) if not batch.produce_future.wait(timeout=timeout): raise Errors.KafkaTimeoutError('Timeout waiting for future') if not batch.produce_future.is_done: raise Errors.UnknownError('Future not done') if batch.produce_future.failed(): log.warning(batch.produce_future.exception) finally: self._flushes_in_progress.decrement()