def add_message(self, tp, key, value, timeout):
"""Add message to batch by topic-partition
If batch is already full this method waits (`ttl` seconds maximum)
until batch is drained by send task
"""
if self._closed:
# this can happen when producer is closing but try to send some
# messages in async task
raise ProducerClosed()
batch = self._batches.get(tp)
if not batch:
batch = MessageBatch(tp, self._batch_size, self._compression_type,
self._batch_ttl, self._loop)
self._batches[tp] = batch
if not self._wait_data_future.done():
# Wakeup sender task if it waits for data
self._wait_data_future.set_result(None)
future = batch.append(key, value)
if future is None:
# Batch is full, can't append data atm,
# waiting until batch per topic-partition is drained
start = self._loop.time()
yield from asyncio.wait([batch.wait_drain()],
timeout=timeout,
loop=self._loop)
timeout -= self._loop.time() - start
if timeout <= 0:
raise KafkaTimeoutError()
return (yield from self.add_message(tp, key, value, timeout))
return future
def send(self, node_id, request):
"""Send a request to a specific node.
Arguments:
node_id (int): destination node
request (Struct): request object (not-encoded)
Raises:
kafka.common.KafkaTimeoutError
kafka.common.NodeNotReadyError
kafka.commom.ConnectionError
kafka.common.CorrelationIdError
Returns:
Future: resolves to Response struct
"""
if not (yield from self.ready(node_id)):
raise NodeNotReadyError(
"Attempt to send a request to node"
" which is not ready (node id {}).".format(node_id))
# Every request gets a response, except one special case:
expect_response = True
if isinstance(request, tuple(ProduceRequest)) and \
request.required_acks == 0:
expect_response = False
future = self._conns[node_id].send(
request, expect_response=expect_response)
try:
result = yield from future
except asyncio.TimeoutError:
raise KafkaTimeoutError()
else:
return result
def ensure_topic_exists(self, topic, timeout=30):
start_time = time.time()
while not self.has_metadata_for_topic(topic):
if time.time() > start_time + timeout:
raise KafkaTimeoutError(
'Unable to create topic {0}'.format(topic))
self.load_metadata_for_topics(topic,
ignore_leadernotavailable=True)
time.sleep(.5)
def ensure_topic_exists(self, topic, timeout = 30):
start_time = time.time()
while not self.has_metadata_for_topic(topic):
if time.time() > start_time + timeout:
raise KafkaTimeoutError("Unable to create topic {0}".format(topic))
try:
self.load_metadata_for_topics(topic)
except LeaderNotAvailableError:
pass
except UnknownTopicOrPartitionError:
# Server is not configured to auto-create
# retrying in this case will not help
raise
time.sleep(.5)
def mocked_send(nodeid, req):
raise KafkaTimeoutError()