def add_message(self, tp, key, value, timeout): """Add message to batch by topic-partition If batch is already full this method waits (`ttl` seconds maximum) until batch is drained by send task """ if self._closed: # this can happen when producer is closing but try to send some # messages in async task raise ProducerClosed() batch = self._batches.get(tp) if not batch: batch = MessageBatch(tp, self._batch_size, self._compression_type, self._batch_ttl, self._loop) self._batches[tp] = batch if not self._wait_data_future.done(): # Wakeup sender task if it waits for data self._wait_data_future.set_result(None) future = batch.append(key, value) if future is None: # Batch is full, can't append data atm, # waiting until batch per topic-partition is drained start = self._loop.time() yield from asyncio.wait([batch.wait_drain()], timeout=timeout, loop=self._loop) timeout -= self._loop.time() - start if timeout <= 0: raise KafkaTimeoutError() return (yield from self.add_message(tp, key, value, timeout)) return future
def send(self, node_id, request): """Send a request to a specific node. Arguments: node_id (int): destination node request (Struct): request object (not-encoded) Raises: kafka.common.KafkaTimeoutError kafka.common.NodeNotReadyError kafka.commom.ConnectionError kafka.common.CorrelationIdError Returns: Future: resolves to Response struct """ if not (yield from self.ready(node_id)): raise NodeNotReadyError( "Attempt to send a request to node" " which is not ready (node id {}).".format(node_id)) # Every request gets a response, except one special case: expect_response = True if isinstance(request, tuple(ProduceRequest)) and \ request.required_acks == 0: expect_response = False future = self._conns[node_id].send( request, expect_response=expect_response) try: result = yield from future except asyncio.TimeoutError: raise KafkaTimeoutError() else: return result
def ensure_topic_exists(self, topic, timeout=30): start_time = time.time() while not self.has_metadata_for_topic(topic): if time.time() > start_time + timeout: raise KafkaTimeoutError( 'Unable to create topic {0}'.format(topic)) self.load_metadata_for_topics(topic, ignore_leadernotavailable=True) time.sleep(.5)
def ensure_topic_exists(self, topic, timeout = 30): start_time = time.time() while not self.has_metadata_for_topic(topic): if time.time() > start_time + timeout: raise KafkaTimeoutError("Unable to create topic {0}".format(topic)) try: self.load_metadata_for_topics(topic) except LeaderNotAvailableError: pass except UnknownTopicOrPartitionError: # Server is not configured to auto-create # retrying in this case will not help raise time.sleep(.5)
def mocked_send(nodeid, req): raise KafkaTimeoutError()