def multiproduce(self, data, version=DEFAULT_VERSION, **kwargs):
"""
Sends messages to the broker on multiple topics and/or partitions.
>>> client.produce((
... ('topic-1', 0, ('message',)),
... ('topic-2', 0, ('message', 'message',)),
... ))
:param data: sequence of 3-tuples of the format
``(topic, partition, messages)``
:type data: list, generator, or other iterable
:param version: version of message encoding
:type version: int
:param \*\*kwargs: extra (version-specific) keyword arguments
to pass to message encoder
"""
payloads = []
for topic, partition, messages in data:
payload = StructuredBytesIO()
write_request_header(payload, topic, partition)
payload.write(Message.encode(messages, version=version, **kwargs))
payloads.append(payload)
request = StructuredBytesIO()
request.pack(2, REQUEST_TYPE_MULTIPRODUCE)
request.pack(2, len(payloads))
for payload in payloads:
request.write(payload)
return self.handler.request(request, has_response=False)
def multiproduce(self, data, version=DEFAULT_VERSION, **kwargs):
"""
Sends messages to the broker on multiple topics and/or partitions.
>>> client.produce((
... ('topic-1', 0, ('message',)),
... ('topic-2', 0, ('message', 'message',)),
... ))
:param data: sequence of 3-tuples of the format
``(topic, partition, messages)``
:type data: list, generator, or other iterable
:param version: version of message encoding
:type version: int
:param \*\*kwargs: extra (version-specific) keyword arguments
to pass to message encoder
"""
payloads = []
for topic, partition, messages in data:
payload = StructuredBytesIO()
write_request_header(payload, topic, partition)
payload.write(Message.encode(messages, version=version, **kwargs))
payloads.append(payload)
request = StructuredBytesIO()
request.pack(2, REQUEST_TYPE_MULTIPRODUCE)
request.pack(2, len(payloads))
for payload in payloads:
request.write(payload)
return self.handler.request(request, has_response=False)
def produce(self,
topic,
partition,
messages,
version=DEFAULT_VERSION,
**kwargs):
"""
Sends messages to the broker on a single topic/partition combination.
>>> client.produce('topic', 0, ('message',))
:param topic: topic name
:param partition: partition ID
:param messages: the messages to be sent
:type messages: list, generator, or other iterable of strings
:param version: version of message encoding
:type version: int
:param \*\*kwargs: extra (version-specific) keyword arguments
to pass to message encoder
"""
request = StructuredBytesIO()
request.pack(2, REQUEST_TYPE_PRODUCE)
write_request_header(request, topic, partition)
request.write(Message.encode(messages, version=version, **kwargs))
return self.handler.request(request, has_response=False)
def produce(self, topic, partition, messages,
version=DEFAULT_VERSION, **kwargs):
"""
Sends messages to the broker on a single topic/partition combination.
>>> client.produce('topic', 0, ('message',))
:param topic: topic name
:param partition: partition ID
:param messages: the messages to be sent
:type messages: list, generator, or other iterable of strings
:param version: version of message encoding
:type version: int
:param \*\*kwargs: extra (version-specific) keyword arguments
to pass to message encoder
"""
request = StructuredBytesIO()
request.pack(2, REQUEST_TYPE_PRODUCE)
write_request_header(request, topic, partition)
request.write(Message.encode(messages, version=version, **kwargs))
return self.handler.request(request, has_response=False)
def offsets(self, topic, partition, time, max):
"""
Returns message offsets before a certain time for the given
topic/partition.
>>> client.offsets('test', 0, OFFSET_EARLIEST, 1)
[0]
:param topic: topic name
:param partition: partition ID
:param time: the time in milliseconds since the UNIX epoch, or either
``OFFSET_EARLIEST`` or ``OFFSET_LATEST``.
:type time: integer
:param max: the maximum number of offsets to return
:rtype: list of offsets
"""
request = StructuredBytesIO()
request.pack(2, REQUEST_TYPE_OFFSETS)
write_request_header(request, topic, partition)
request.pack(8, time)
request.pack(4, max)
response = self.handler.request(request)
(count,) = OffsetsResponseHeader.unpack_from(response.get())
offsets = []
for i in xrange(0, count):
offsets.append(Offset.unpack_from(response.get(),
offset=OffsetsResponseHeader.size + (i * Offset.size)).value)
return offsets
def fetch(self, topic, partition, offset, size):
"""
Fetches messages from the broker on a single topic/partition.
>>> for offset, message in client.fetch('test', 0, 0, 1000):
... print offset, message
0L 'hello world'
20L 'hello world'
:param topic: topic name
:param partition: partition ID
:param offset: offset to begin read
:type offset: integer
:param size: the maximum number of bytes to return
:rtype: generator of 2-tuples in ``(offset, message)`` format
"""
# TODO: Document failure modes.
request = StructuredBytesIO()
request.pack(2, REQUEST_TYPE_FETCH)
write_request_header(request, topic, partition)
request.pack(8, offset)
request.pack(4, size)
response = self.handler.request(request)
try:
# N.B. Using generator here makes dealing with decode errors hard
return list(decode_messages(response.get(), from_offset=offset))
except SocketDisconnectedError:
return []
except MessageTooLargeError:
# Try again, but larger!
return self.fetch(topic, partition, offset, size*1.5)
def fetch(self, topic, partition, offset, size):
"""
Fetches messages from the broker on a single topic/partition.
>>> for offset, message in client.fetch('test', 0, 0, 1000):
... print offset, message
0L 'hello world'
20L 'hello world'
:param topic: topic name
:param partition: partition ID
:param offset: offset to begin read
:type offset: integer
:param size: the maximum number of bytes to return
:rtype: generator of 2-tuples in ``(offset, message)`` format
"""
# TODO: Document failure modes.
request = StructuredBytesIO()
request.pack(2, REQUEST_TYPE_FETCH)
write_request_header(request, topic, partition)
request.pack(8, offset)
request.pack(4, size)
response = self.handler.request(request)
try:
return decode_messages(response.get(), from_offset=offset)
except SocketDisconnectedError:
return []
except:
e = sys.exc_info()[0]
logger.error('Exception at offset %s: %s' % (offset, e))
raise
def offsets(self, topic, partition, time, max):
"""
Returns message offsets before a certain time for the given
topic/partition.
>>> client.offsets('test', 0, OFFSET_EARLIEST, 1)
[0]
:param topic: topic name
:param partition: partition ID
:param time: the time in milliseconds since the UNIX epoch, or either
``OFFSET_EARLIEST`` or ``OFFSET_LATEST``.
:type time: integer
:param max: the maximum number of offsets to return
:rtype: list of offsets
"""
request = StructuredBytesIO()
request.pack(2, REQUEST_TYPE_OFFSETS)
write_request_header(request, topic, partition)
request.pack(8, time)
request.pack(4, max)
response = self.handler.request(request)
(count, ) = OffsetsResponseHeader.unpack_from(response.get())
offsets = []
for i in xrange(0, count):
offsets.append(
Offset.unpack_from(response.get(),
offset=OffsetsResponseHeader.size +
(i * Offset.size)).value)
return offsets
def fetch(self, topic, partition, offset, size):
"""
Fetches messages from the broker on a single topic/partition.
>>> for offset, message in client.fetch('test', 0, 0, 1000):
... print offset, message
0L 'hello world'
20L 'hello world'
:param topic: topic name
:param partition: partition ID
:param offset: offset to begin read
:type offset: integer
:param size: the maximum number of bytes to return
:rtype: generator of 2-tuples in ``(offset, message)`` format
"""
# TODO: Document failure modes.
request = StructuredBytesIO()
request.pack(2, REQUEST_TYPE_FETCH)
write_request_header(request, topic, partition)
request.pack(8, offset)
request.pack(4, size)
response = self.handler.request(request)
try:
# N.B. Using generator here makes dealing with decode errors hard
return list(decode_messages(response.get(), from_offset=offset))
except SocketDisconnectedError:
return []
except MessageTooLargeError:
# Try again, but larger!
return self.fetch(topic, partition, offset, size * 1.5)
def multifetch(self, data):
"""
Fetches messages from the broker on multiple topics/partitions.
>>> topics = (
... ('topic-1', 0, 0, 1000),
... ('topic-2', 0, 0, 1000),
... )
>>> for i, response in enumerate(client.fetch(topics)):
... print 'response:', i
... for offset, message in messages:
... print offset, message
response 0
0L 'hello world'
20L 'hello world'
response 1
0L 'hello world'
20L 'hello world'
:param data: sequence of 4-tuples of the format
``(topic, partition, offset, size)``
For more information, see :meth:`Client.fetch`.
:rtype: generator of fetch responses (message generators).
For more information, see :meth:`Client.fetch`.
"""
payloads = []
from_offsets = []
for topic, partition, offset, size in data:
payload = StructuredBytesIO()
write_request_header(payload, topic, partition)
from_offsets.append(offset)
payload.pack(8, offset)
payload.pack(4, size)
payloads.append(payload)
request = StructuredBytesIO()
request.pack(2, REQUEST_TYPE_MULTIFETCH)
request.pack(2, len(payloads))
for payload in payloads:
request.write(payload)
response = self.handler.request(request)
return decode_message_sets(response.get(), from_offsets)
def multifetch(self, data):
"""
Fetches messages from the broker on multiple topics/partitions.
>>> topics = (
... ('topic-1', 0, 0, 1000),
... ('topic-2', 0, 0, 1000),
... )
>>> for i, response in enumerate(client.fetch(topics)):
... print 'response:', i
... for offset, message in messages:
... print offset, message
response 0
0L 'hello world'
20L 'hello world'
response 1
0L 'hello world'
20L 'hello world'
:param data: sequence of 4-tuples of the format
``(topic, partition, offset, size)``
For more information, see :meth:`Client.fetch`.
:rtype: generator of fetch responses (message generators).
For more information, see :meth:`Client.fetch`.
"""
payloads = []
from_offsets = []
for topic, partition, offset, size in data:
payload = StructuredBytesIO()
write_request_header(payload, topic, partition)
from_offsets.append(offset)
payload.pack(8, offset)
payload.pack(4, size)
payloads.append(payload)
request = StructuredBytesIO()
request.pack(2, REQUEST_TYPE_MULTIFETCH)
request.pack(2, len(payloads))
for payload in payloads:
request.write(payload)
response = self.handler.request(request)
return decode_message_sets(response.get(), from_offsets)
def test_pack(self):
buf = StructuredBytesIO()
buf.pack(2, 1)
self.assertEqual(str(buf), '\x00\x01')
