def multiproduce(self, data, version=DEFAULT_VERSION, **kwargs):
"""
Sends messages to the broker on multiple topics and/or partitions.
>>> client.produce((
... ('topic-1', 0, ('message',)),
... ('topic-2', 0, ('message', 'message',)),
... ))
:param data: sequence of 3-tuples of the format
``(topic, partition, messages)``
:type data: list, generator, or other iterable
:param version: version of message encoding
:type version: int
:param \*\*kwargs: extra (version-specific) keyword arguments
to pass to message encoder
"""
payloads = []
for topic, partition, messages in data:
payload = StructuredBytesIO()
write_request_header(payload, topic, partition)
payload.write(Message.encode(messages, version=version, **kwargs))
payloads.append(payload)
request = StructuredBytesIO()
request.pack(2, REQUEST_TYPE_MULTIPRODUCE)
request.pack(2, len(payloads))
for payload in payloads:
request.write(payload)
return self.handler.request(request, has_response=False)
def test_write(self):
buf = StructuredBytesIO()
buf.write('test')
self.assertEqual(str(buf), 'test')
buf = StructuredBytesIO()
buf.write(StructuredBytesIO('test'))
self.assertEqual(str(buf), 'test')
def fetch(self, topic, partition, offset, size):
"""
Fetches messages from the broker on a single topic/partition.
>>> for offset, message in client.fetch('test', 0, 0, 1000):
... print offset, message
0L 'hello world'
20L 'hello world'
:param topic: topic name
:param partition: partition ID
:param offset: offset to begin read
:type offset: integer
:param size: the maximum number of bytes to return
:rtype: generator of 2-tuples in ``(offset, message)`` format
"""
# TODO: Document failure modes.
request = StructuredBytesIO()
request.pack(2, REQUEST_TYPE_FETCH)
write_request_header(request, topic, partition)
request.pack(8, offset)
request.pack(4, size)
response = self.handler.request(request)
try:
# N.B. Using generator here makes dealing with decode errors hard
return list(decode_messages(response.get(), from_offset=offset))
except SocketDisconnectedError:
return []
except MessageTooLargeError:
# Try again, but larger!
return self.fetch(topic, partition, offset, size * 1.5)
def offsets(self, topic, partition, time, max):
"""
Returns message offsets before a certain time for the given
topic/partition.
>>> client.offsets('test', 0, OFFSET_EARLIEST, 1)
[0]
:param topic: topic name
:param partition: partition ID
:param time: the time in milliseconds since the UNIX epoch, or either
``OFFSET_EARLIEST`` or ``OFFSET_LATEST``.
:type time: integer
:param max: the maximum number of offsets to return
:rtype: list of offsets
"""
request = StructuredBytesIO()
request.pack(2, REQUEST_TYPE_OFFSETS)
write_request_header(request, topic, partition)
request.pack(8, time)
request.pack(4, max)
response = self.handler.request(request)
(count, ) = OffsetsResponseHeader.unpack_from(response.get())
offsets = []
for i in xrange(0, count):
offsets.append(
Offset.unpack_from(response.get(),
offset=OffsetsResponseHeader.size +
(i * Offset.size)).value)
return offsets
def encode(cls, messages, version, **kwargs):
"""
Encodes multiple messages.
:param messages: messages to publish
:type messages: sequence of strs
:param version: message version to publish ("magic number")
:type version: int
:param \*\*kwargs: extra arguments to pass to :meth:`.pack_into`
:returns: encoded messages
:rtype: :class:`samsa.utils.structuredio.StructuredBytesIO`
"""
message_header_length = (cls.Header.size +
cls.VersionHeaders[version].size)
length = (MessageSetFrameHeader.size + sum(map(len, messages)) +
(len(messages) * message_header_length))
bytea = bytearray(length)
MessageSetFrameHeader.pack_into(bytea, 0, length=length - 4)
offset = MessageSetFrameHeader.size
for message in messages:
written = cls.pack_into(bytea,
offset,
payload=message,
version=version,
**kwargs)
offset += written
return StructuredBytesIO(bytea)
def fetch(self, topic, partition, offset, size):
"""
Fetches messages from the broker on a single topic/partition.
>>> for offset, message in client.fetch('test', 0, 0, 1000):
... print offset, message
0L 'hello world'
20L 'hello world'
:param topic: topic name
:param partition: partition ID
:param offset: offset to begin read
:type offset: integer
:param size: the maximum number of bytes to return
:rtype: generator of 2-tuples in ``(offset, message)`` format
"""
# TODO: Document failure modes.
request = StructuredBytesIO()
request.pack(2, REQUEST_TYPE_FETCH)
write_request_header(request, topic, partition)
request.pack(8, offset)
request.pack(4, size)
response = self.handler.request(request)
try:
# N.B. Using generator here makes dealing with decode errors hard
return list(decode_messages(response.get(), from_offset=offset))
except SocketDisconnectedError:
return []
except MessageTooLargeError:
# Try again, but larger!
return self.fetch(topic, partition, offset, size*1.5)
def offsets(self, topic, partition, time, max):
"""
Returns message offsets before a certain time for the given
topic/partition.
>>> client.offsets('test', 0, OFFSET_EARLIEST, 1)
[0]
:param topic: topic name
:param partition: partition ID
:param time: the time in milliseconds since the UNIX epoch, or either
``OFFSET_EARLIEST`` or ``OFFSET_LATEST``.
:type time: integer
:param max: the maximum number of offsets to return
:rtype: list of offsets
"""
request = StructuredBytesIO()
request.pack(2, REQUEST_TYPE_OFFSETS)
write_request_header(request, topic, partition)
request.pack(8, time)
request.pack(4, max)
response = self.handler.request(request)
(count,) = OffsetsResponseHeader.unpack_from(response.get())
offsets = []
for i in xrange(0, count):
offsets.append(Offset.unpack_from(response.get(),
offset=OffsetsResponseHeader.size + (i * Offset.size)).value)
return offsets
def fetch(self, topic, partition, offset, size):
"""
Fetches messages from the broker on a single topic/partition.
>>> for offset, message in client.fetch('test', 0, 0, 1000):
... print offset, message
0L 'hello world'
20L 'hello world'
:param topic: topic name
:param partition: partition ID
:param offset: offset to begin read
:type offset: integer
:param size: the maximum number of bytes to return
:rtype: generator of 2-tuples in ``(offset, message)`` format
"""
# TODO: Document failure modes.
request = StructuredBytesIO()
request.pack(2, REQUEST_TYPE_FETCH)
write_request_header(request, topic, partition)
request.pack(8, offset)
request.pack(4, size)
response = self.handler.request(request)
try:
return decode_messages(response.get(), from_offset=offset)
except SocketDisconnectedError:
return []
except:
e = sys.exc_info()[0]
logger.error('Exception at offset %s: %s' % (offset, e))
raise
def produce(self, topic, partition, messages,
version=DEFAULT_VERSION, **kwargs):
"""
Sends messages to the broker on a single topic/partition combination.
>>> client.produce('topic', 0, ('message',))
:param topic: topic name
:param partition: partition ID
:param messages: the messages to be sent
:type messages: list, generator, or other iterable of strings
:param version: version of message encoding
:type version: int
:param \*\*kwargs: extra (version-specific) keyword arguments
to pass to message encoder
"""
request = StructuredBytesIO()
request.pack(2, REQUEST_TYPE_PRODUCE)
write_request_header(request, topic, partition)
request.write(Message.encode(messages, version=version, **kwargs))
return self.handler.request(request, has_response=False)
def produce(self,
topic,
partition,
messages,
version=DEFAULT_VERSION,
**kwargs):
"""
Sends messages to the broker on a single topic/partition combination.
>>> client.produce('topic', 0, ('message',))
:param topic: topic name
:param partition: partition ID
:param messages: the messages to be sent
:type messages: list, generator, or other iterable of strings
:param version: version of message encoding
:type version: int
:param \*\*kwargs: extra (version-specific) keyword arguments
to pass to message encoder
"""
request = StructuredBytesIO()
request.pack(2, REQUEST_TYPE_PRODUCE)
write_request_header(request, topic, partition)
request.write(Message.encode(messages, version=version, **kwargs))
return self.handler.request(request, has_response=False)
def test_pack(self):
buf = StructuredBytesIO()
buf.pack(2, 1)
self.assertEqual(str(buf), '\x00\x01')
def multifetch(self, data):
"""
Fetches messages from the broker on multiple topics/partitions.
>>> topics = (
... ('topic-1', 0, 0, 1000),
... ('topic-2', 0, 0, 1000),
... )
>>> for i, response in enumerate(client.fetch(topics)):
... print 'response:', i
... for offset, message in messages:
... print offset, message
response 0
0L 'hello world'
20L 'hello world'
response 1
0L 'hello world'
20L 'hello world'
:param data: sequence of 4-tuples of the format
``(topic, partition, offset, size)``
For more information, see :meth:`Client.fetch`.
:rtype: generator of fetch responses (message generators).
For more information, see :meth:`Client.fetch`.
"""
payloads = []
from_offsets = []
for topic, partition, offset, size in data:
payload = StructuredBytesIO()
write_request_header(payload, topic, partition)
from_offsets.append(offset)
payload.pack(8, offset)
payload.pack(4, size)
payloads.append(payload)
request = StructuredBytesIO()
request.pack(2, REQUEST_TYPE_MULTIFETCH)
request.pack(2, len(payloads))
for payload in payloads:
request.write(payload)
response = self.handler.request(request)
return decode_message_sets(response.get(), from_offsets)
def test_unpack(self):
buf = StructuredBytesIO('\x00\x01')
value = buf.unpack(2)
self.assertEqual(value, 1)
def test_frame(self):
buf = StructuredBytesIO()
buf.frame(1, 'test')
self.assertEqual(str(buf), '\x04test')
def test_string_cast(self):
value = '\x00\x00'
buf = StructuredBytesIO(value)
self.assertEqual(str(buf), value)
buf.seek(2) # SEEK_END
self.assertEqual(str(buf), value)
def multifetch(self, data):
"""
Fetches messages from the broker on multiple topics/partitions.
>>> topics = (
... ('topic-1', 0, 0, 1000),
... ('topic-2', 0, 0, 1000),
... )
>>> for i, response in enumerate(client.fetch(topics)):
... print 'response:', i
... for offset, message in messages:
... print offset, message
response 0
0L 'hello world'
20L 'hello world'
response 1
0L 'hello world'
20L 'hello world'
:param data: sequence of 4-tuples of the format
``(topic, partition, offset, size)``
For more information, see :meth:`Client.fetch`.
:rtype: generator of fetch responses (message generators).
For more information, see :meth:`Client.fetch`.
"""
payloads = []
from_offsets = []
for topic, partition, offset, size in data:
payload = StructuredBytesIO()
write_request_header(payload, topic, partition)
from_offsets.append(offset)
payload.pack(8, offset)
payload.pack(4, size)
payloads.append(payload)
request = StructuredBytesIO()
request.pack(2, REQUEST_TYPE_MULTIFETCH)
request.pack(2, len(payloads))
for payload in payloads:
request.write(payload)
response = self.handler.request(request)
return decode_message_sets(response.get(), from_offsets)
def test_unframe(self):
buf = StructuredBytesIO('\x04test')
self.assertEqual(buf.unframe(1), 'test')
buf = StructuredBytesIO('\x04testextra')
self.assertEqual(buf.unframe(1), 'test')
buf = StructuredBytesIO('\x04tes')
with self.assertRaises(ValueError):
buf.unframe(1)
buf = StructuredBytesIO('\x04tes')
self.assertEqual(buf.unframe(1, validate=False), 'tes')
def test_unwrap(self):
buf = StructuredBytesIO('\x04test').unwrap(1)
self.assertIsInstance(buf, StructuredBytesIO)
self.assertEqual(str(buf), 'test')
def multiproduce(self, data, version=DEFAULT_VERSION, **kwargs):
"""
Sends messages to the broker on multiple topics and/or partitions.
>>> client.produce((
... ('topic-1', 0, ('message',)),
... ('topic-2', 0, ('message', 'message',)),
... ))
:param data: sequence of 3-tuples of the format
``(topic, partition, messages)``
:type data: list, generator, or other iterable
:param version: version of message encoding
:type version: int
:param \*\*kwargs: extra (version-specific) keyword arguments
to pass to message encoder
"""
payloads = []
for topic, partition, messages in data:
payload = StructuredBytesIO()
write_request_header(payload, topic, partition)
payload.write(Message.encode(messages, version=version, **kwargs))
payloads.append(payload)
request = StructuredBytesIO()
request.pack(2, REQUEST_TYPE_MULTIPRODUCE)
request.pack(2, len(payloads))
for payload in payloads:
request.write(payload)
return self.handler.request(request, has_response=False)
def test_length(self):
buf = StructuredBytesIO('\x00\x00')
self.assertEqual(len(buf), 2)