def test_messageset_repr(self):
msg1 = messages.Message(crc=0,
magic=0,
attributes=0,
key="foo",
value=json.dumps({"bar": "bazz"}))
msg2 = messages.Message(crc=0,
magic=0,
attributes=0,
key="bar",
value=json.dumps({"bwee": "bwoo"}))
msg_set = messages.MessageSet([(10, msg1), (11, msg2)])
self.assertEqual(repr(msg_set),
'[foo => {"bar": "bazz"}, bar => {"bwee": "bwoo"}]')
def produce(self, topic, message):
"""
Primary method that queues messages up to be flushed to the brokers.
Performs sanity checks to make sure we're not closing and that the
topic given is known.
If the topic given is *not* known, the ``heal()`` method on the cluster
is called and the check is performed again.
Depending on the ``batch_size`` attribute this call may not actually
send any requests and merely keeps the pending messages in the
``unsent`` structure.
"""
if self.closing:
log.warn("Producing to %s topic while closing.", topic)
return
if topic not in self.cluster.topics:
log.debug("Producing to unknown topic %s, loading metadata", topic)
yield self.cluster.heal()
if topic not in self.cluster.topics:
log.error("Unknown topic %s and not auto-created", topic)
return
self.unsent[topic].append(
messages.Message(magic=0,
attributes=0,
key=self.key_maker(message),
value=self.serializer(message)))
if not self.batch_size or self.unsent_count >= self.batch_size:
yield self.flush()
def test_message_repr(self):
msg = messages.Message(crc=0,
magic=0,
attributes=0,
key="foo",
value=json.dumps({"bar": "bazz"}))
self.assertEqual(repr(msg), 'foo => {"bar": "bazz"}')
def test_leader_not_present_at_first(self):
self.add_topic("test.topic", leaders=(7,))
p = producer.Producer(["kafka01"], batch_size=1)
yield p.produce("test.topic", "foo")
self.add_topic("test.topic", leaders=(1,))
yield p.cluster.heal()
yield p.produce("test.topic", "bar")
self.assert_sent(
broker_id=1,
request=produce.ProduceRequest(
required_acks=-1,
timeout=500,
topics=[
produce.TopicRequest(
name="test.topic",
partitions=[
produce.PartitionRequest(
partition_id=0,
message_set=messages.MessageSet.compressed(
compression=None,
msgs=[
messages.Message(
magic=0, attributes=0, key=None,
value=p.serializer("foo")
),
messages.Message(
magic=0, attributes=0, key=None,
value=p.serializer("bar")
),
]
)
)
]
)
]
)
)
def test_consuming_unknown_topic_reloads_metadata(self):
self.add_topic("test.topic", leaders=(3,))
self.set_responses(
broker_id=3, api="fetch",
responses=[
fetch.FetchResponse(
topics=[
fetch.TopicResponse(
name="test.topic",
partitions=[
fetch.PartitionResponse(
partition_id=0,
error_code=errors.no_error,
highwater_mark_offset=2,
message_set=messages.MessageSet(
messages=[
(
0,
messages.Message(
magic=0, attributes=0,
key=None,
value='{"cat": "dog"}',
)
),
]
)
),
]
),
]
),
]
)
c = FakeConsumer(["kafka01"])
# at this point the cluster hasn't synced, so "test.topic" is unknown
msgs = yield c.consume("test.topic")
self.assertEqual(msgs, [{"cat": "dog"}])
self.assertEqual(len(self.requests_by_broker[3]), 1)
def test_routing_to_partitions(self):
self.add_topic("test.topic", leaders=(1, 1, 8, 3))
self.set_responses(
broker_id=1, api="produce",
responses=[
produce.ProduceResponse(
topics=[
produce.TopicResponse(
name="test.topic",
partitions=[
produce.PartitionResponse(
partition_id=0,
error_code=errors.no_error,
offset=8000,
),
]
),
]
),
produce.ProduceResponse(
topics=[
produce.TopicResponse(
name="test.topic",
partitions=[
produce.PartitionResponse(
partition_id=1,
error_code=errors.no_error,
offset=8000,
),
]
),
]
),
produce.ProduceResponse(
topics=[
produce.TopicResponse(
name="test.topic",
partitions=[
produce.PartitionResponse(
partition_id=0,
error_code=errors.no_error,
offset=8001,
),
]
),
]
),
]
)
self.set_responses(
broker_id=3, api="produce",
responses=[
produce.ProduceResponse(
topics=[
produce.TopicResponse(
name="test.topic",
partitions=[
produce.PartitionResponse(
partition_id=3,
error_code=errors.no_error,
offset=8000,
),
]
),
]
),
]
)
msgs = [
{"key": 0, "msg": "foo"},
{"key": 1, "msg": "bar"},
{"key": 3, "msg": "bwee"},
{"key": 0, "msg": "bwoo"},
]
p = producer.Producer(
["kafka01"],
key_maker=attribute_key, partitioner=key_partitioner
)
for msg in msgs:
yield p.produce("test.topic", msg)
self.assert_sent(
broker_id=1,
request=produce.ProduceRequest(
required_acks=-1,
timeout=500,
topics=[
produce.TopicRequest(
name="test.topic",
partitions=[
produce.PartitionRequest(
partition_id=0,
message_set=messages.MessageSet.compressed(
compression=None,
msgs=[
messages.Message(
magic=0, attributes=0,
key=0,
value=p.serializer(
{"msg": "foo", "key": 0}
)
)
]
)
)
]
)
]
)
)
self.assert_sent(
broker_id=1,
request=produce.ProduceRequest(
required_acks=-1,
timeout=500,
topics=[
produce.TopicRequest(
name="test.topic",
partitions=[
produce.PartitionRequest(
partition_id=1,
message_set=messages.MessageSet.compressed(
compression=None,
msgs=[
messages.Message(
magic=0, attributes=0,
key=1,
value=p.serializer(
{"msg": "bar", "key": 1}
)
)
]
)
)
]
)
]
)
)
self.assert_sent(
broker_id=1,
request=produce.ProduceRequest(
required_acks=-1,
timeout=500,
topics=[
produce.TopicRequest(
name="test.topic",
partitions=[
produce.PartitionRequest(
partition_id=0,
message_set=messages.MessageSet.compressed(
compression=None,
msgs=[
messages.Message(
magic=0, attributes=0,
key=0,
value=p.serializer(
{"msg": "bwoo", "key": 0}
)
)
]
)
)
]
)
]
)
)
self.assert_sent(
broker_id=3,
request=produce.ProduceRequest(
required_acks=-1,
timeout=500,
topics=[
produce.TopicRequest(
name="test.topic",
partitions=[
produce.PartitionRequest(
partition_id=3,
message_set=messages.MessageSet.compressed(
compression=None,
msgs=[
messages.Message(
magic=0, attributes=0,
key=3,
value=p.serializer(
{"msg": "bwee", "key": 3}
)
)
]
)
)
]
)
]
)
)
def test_gzip_compression(self):
self.add_topic("test.topic", leaders=(1,))
self.set_responses(
broker_id=1, api="produce",
responses=[
produce.ProduceResponse(
topics=[
produce.TopicResponse(
name="test.topic",
partitions=[
produce.PartitionResponse(
partition_id=0,
error_code=errors.no_error,
offset=8003,
),
]
),
]
),
]
)
msgs = [u"foö", "bar"]
p = producer.Producer(
["kafka01"], batch_size=2, compression=constants.GZIP
)
for msg in msgs:
yield p.produce("test.topic", msg)
self.assert_sent(
broker_id=1,
request=produce.ProduceRequest(
required_acks=-1,
timeout=500,
topics=[
produce.TopicRequest(
name="test.topic",
partitions=[
produce.PartitionRequest(
partition_id=0,
message_set=messages.MessageSet.compressed(
compression=constants.GZIP,
msgs=[
messages.Message(
magic=0,
attributes=0,
key=None,
value=p.serializer(u"foö"),
),
messages.Message(
magic=0,
attributes=0,
key=None,
value=p.serializer("bar"),
)
]
)
)
]
)
]
)
)
def test_default_routing(self, mock_random):
self.add_topic("test.topic", leaders=(1, 8))
choices_to_make = [0, 1]
def get_next_choice(*args):
return choices_to_make.pop(0)
mock_random.choice.side_effect = get_next_choice
self.set_responses(
broker_id=1, api="produce",
responses=[
produce.ProduceResponse(
topics=[
produce.TopicResponse(
name="test.topic",
partitions=[
produce.PartitionResponse(
partition_id=0,
error_code=errors.no_error,
offset=8000,
),
]
),
]
),
]
)
self.set_responses(
broker_id=8, api="produce",
responses=[
produce.ProduceResponse(
topics=[
produce.TopicResponse(
name="test.topic",
partitions=[
produce.PartitionResponse(
partition_id=1,
error_code=errors.no_error,
offset=8000,
),
]
),
]
),
]
)
msgs = ["foo", "bar"]
p = producer.Producer(["kafka01"])
for msg in msgs:
yield p.produce("test.topic", msg)
self.assert_sent(
broker_id=1,
request=produce.ProduceRequest(
required_acks=-1,
timeout=500,
topics=[
produce.TopicRequest(
name="test.topic",
partitions=[
produce.PartitionRequest(
partition_id=0,
message_set=messages.MessageSet.compressed(
compression=None,
msgs=[
messages.Message(
magic=0, attributes=0, key=None,
value=p.serializer("foo")
)
]
)
)
]
)
]
)
)
self.assert_sent(
broker_id=8,
request=produce.ProduceRequest(
required_acks=-1,
timeout=500,
topics=[
produce.TopicRequest(
name="test.topic",
partitions=[
produce.PartitionRequest(
partition_id=1,
message_set=messages.MessageSet.compressed(
compression=None,
msgs=[
messages.Message(
magic=0, attributes=0, key=None,
value=p.serializer("bar")
)
]
)
)
]
)
]
)
)
def test_retriable_error_for_offset(self):
self.add_topic("test.topic", leaders=(1,))
self.set_responses(
broker_id=1, api="offset",
responses=[
offset.OffsetResponse(
topics=[
offset.TopicResponse(
name="test.topic",
partitions=[
offset.PartitionResponse(
partition_id=0,
error_code=errors.request_timed_out,
offsets=[],
)
]
)
]
),
offset.OffsetResponse(
topics=[
offset.TopicResponse(
name="test.topic",
partitions=[
offset.PartitionResponse(
partition_id=0,
error_code=errors.no_error,
offsets=[99],
)
]
)
]
),
]
)
self.set_responses(
broker_id=1, api="fetch",
responses=[
fetch.FetchResponse(
topics=[
fetch.TopicResponse(
name="test.topic",
partitions=[
fetch.PartitionResponse(
partition_id=0,
error_code=errors.no_error,
highwater_mark_offset=2,
message_set=messages.MessageSet([
(
0,
messages.Message(
magic=0, attributes=0,
key=None,
value='{"cat": "meow"}',
)
),
]),
),
]
),
]
),
]
)
c = single.SingleConsumer(["kafka01"])
yield c.connect()
msgs = yield c.consume("test.topic")
self.assertEqual(msgs, [{"cat": "meow"}])
def test_default_consumes_from_end_offset(self):
self.add_topic("test.topic", leaders=(1,))
self.set_responses(
broker_id=1, api="offset",
responses=[
offset.OffsetResponse(
topics=[
offset.TopicResponse(
name="test.topic",
partitions=[
offset.PartitionResponse(
partition_id=0,
error_code=errors.no_error,
offsets=[99],
)
]
)
]
)
]
)
self.set_responses(
broker_id=1, api="fetch",
responses=[
fetch.FetchResponse(
topics=[
fetch.TopicResponse(
name="test.topic",
partitions=[
fetch.PartitionResponse(
partition_id=0,
error_code=errors.no_error,
highwater_mark_offset=2,
message_set=messages.MessageSet(
messages=[
(
0,
messages.Message(
magic=0, attributes=0,
key=None,
value='{"cat": "meow"}',
)
),
(
1,
messages.Message(
magic=0, attributes=0,
key=None,
value='{"dog": "bark"}',
)
),
]
)
),
]
),
]
),
]
)
c = single.SingleConsumer(["kafka01"])
yield c.connect()
msgs = yield c.consume("test.topic")
yield c.close()
self.assertEqual(msgs, [{"cat": "meow"}, {"dog": "bark"}])
self.assert_sent(
broker_id=1,
request=offset.OffsetRequest(
replica_id=-1,
topics=[
offset.TopicRequest(
name="test.topic",
partitions=[
offset.PartitionRequest(
partition_id=0,
time=-1, # alias for 'end of topic'
max_offsets=1,
)
]
)
]
)
)
self.assert_sent(
broker_id=1,
request=fetch.FetchRequest(
replica_id=-1,
max_wait_time=1000,
min_bytes=1,
topics=[
fetch.TopicRequest(
name="test.topic",
partitions=[
fetch.PartitionRequest(
partition_id=0,
offset=99,
max_bytes=(1024 * 1024),
),
]
)
]
)
)
def test_consume_without_autocommit(self):
self.add_topic("test.topic", leaders=(1, 8))
self.allocator.allocation = {"test.topic": [0, 1]}
self.set_responses(broker_id=3,
api="offset_fetch",
responses=[
offset_fetch.OffsetFetchResponse(topics=[
offset_fetch.TopicResponse(
name="test.topic",
partitions=[
offset_fetch.PartitionResponse(
error_code=errors.no_error,
partition_id=0,
offset=80,
metadata="committed, ok!"),
offset_fetch.PartitionResponse(
error_code=errors.no_error,
partition_id=1,
offset=110,
metadata="committed, ok!"),
])
]),
])
self.set_responses(broker_id=3,
api="offset_commit",
responses=[
offset_commit.OffsetCommitResponse(topics=[
offset_commit.TopicResponse(
name="test.topic",
partitions=[
offset_commit.PartitionResponse(
error_code=errors.no_error,
partition_id=1,
)
]),
]),
])
self.set_responses(
broker_id=1,
api="fetch",
responses=[
fetch.FetchResponse(topics=[
fetch.TopicResponse(
name="test.topic",
partitions=[
fetch.PartitionResponse(
partition_id=0,
error_code=errors.no_error,
highwater_mark_offset=2,
message_set=messages.MessageSet([
(80,
messages.Message(
magic=0,
attributes=0,
key=None,
value='{"cat": "meow"}',
)),
])),
]),
])
])
self.set_responses(
broker_id=8,
api="fetch",
responses=[
fetch.FetchResponse(topics=[
fetch.TopicResponse(
name="test.topic",
partitions=[
fetch.PartitionResponse(
partition_id=1,
error_code=errors.no_error,
highwater_mark_offset=2,
message_set=messages.MessageSet([
(110,
messages.Message(
magic=0,
attributes=0,
key=None,
value='{"cat": "meow"}',
)),
])),
]),
])
])
c = grouped.GroupedConsumer(["kafka01", "kafka02"],
"work-group",
zk_hosts=["zk01", "zk02", "zk03"],
autocommit=False)
yield c.connect()
yield c.consume("test.topic")
self.assert_sent(broker_id=1,
request=fetch.FetchRequest(
replica_id=-1,
max_wait_time=1000,
min_bytes=1,
topics=[
fetch.TopicRequest(name="test.topic",
partitions=[
fetch.PartitionRequest(
partition_id=0,
offset=80,
max_bytes=(1024 *
1024),
),
])
]))
self.assert_sent(broker_id=8,
request=fetch.FetchRequest(
replica_id=-1,
max_wait_time=1000,
min_bytes=1,
topics=[
fetch.TopicRequest(name="test.topic",
partitions=[
fetch.PartitionRequest(
partition_id=1,
offset=110,
max_bytes=(1024 *
1024),
),
])
]))
yield c.commit_offsets()
self.assert_sent(
broker_id=3,
request=offset_commit.OffsetCommitV0Request(
group="work-group",
topics=[
offset_commit.TopicRequest(
name="test.topic",
partitions=[
offset_commit.PartitionRequest(
partition_id=0,
offset=81,
metadata="committed by %s" % c.name),
offset_commit.PartitionRequest(
partition_id=1,
offset=111,
metadata="committed by %s" % c.name),
])
]))
def test_max_bytes_at_partition_level(self):
self.add_topic("test.topic", leaders=(3, 3))
self.set_responses(
broker_id=3, api="fetch",
responses=[
fetch.FetchResponse(
topics=[
fetch.TopicResponse(
name="test.topic",
partitions=[
fetch.PartitionResponse(
partition_id=0,
error_code=errors.no_error,
highwater_mark_offset=2,
message_set=messages.MessageSet(
messages=[
(
0,
messages.Message(
magic=0, attributes=0,
key=None,
value='{"foo": "bar"}',
)
),
]
)
),
fetch.PartitionResponse(
partition_id=1,
error_code=errors.no_error,
highwater_mark_offset=2,
message_set=messages.MessageSet(
messages=[
(
0,
messages.Message(
magic=0, attributes=0,
key=None,
value='{"bwee": "bwoo"}',
)
),
]
)
),
]
),
]
),
]
)
c = FakeConsumer(["kafka01", "kafka02"], max_bytes=(1024 * 1024))
yield c.connect()
msgs = yield c.consume("test.topic")
self.assertEqual(msgs, [{"foo": "bar"}, {"bwee": "bwoo"}])
self.assert_sent(
broker_id=3,
request=fetch.FetchRequest(
replica_id=-1,
max_wait_time=1000,
min_bytes=1,
topics=[
fetch.TopicRequest(
name="test.topic",
partitions=[
fetch.PartitionRequest(
partition_id=0,
offset=0,
max_bytes=(512 * 1024),
),
fetch.PartitionRequest(
partition_id=1,
offset=0,
max_bytes=(512 * 1024),
),
]
)
]
)
)
def test_consumer_tracks_offsets(self):
self.add_topic("test.topic", leaders=(3, 8))
self.set_responses(
broker_id=3, api="fetch",
responses=[
fetch.FetchResponse(
topics=[
fetch.TopicResponse(
name="test.topic",
partitions=[
fetch.PartitionResponse(
partition_id=0,
error_code=errors.no_error,
highwater_mark_offset=2,
message_set=messages.MessageSet(
messages=[
(
0,
messages.Message(
magic=0, attributes=0,
key=None,
value='{"foo": "bar"}',
)
),
(
1,
messages.Message(
magic=0, attributes=0,
key=None,
value='{"bwee": "bwoo"}',
)
),
]
)
),
]
),
]
),
fetch.FetchResponse(
topics=[
fetch.TopicResponse(
name="test.topic",
partitions=[
fetch.PartitionResponse(
partition_id=0,
error_code=errors.no_error,
highwater_mark_offset=2,
message_set=messages.MessageSet([]),
),
]
)
]
)
]
)
self.set_responses(
broker_id=8, api="fetch",
responses=[
fetch.FetchResponse(
topics=[
fetch.TopicResponse(
name="test.topic",
partitions=[
fetch.PartitionResponse(
partition_id=1,
error_code=errors.no_error,
highwater_mark_offset=2,
message_set=messages.MessageSet(
messages=[
(
0,
messages.Message(
magic=0, attributes=0,
key=None,
value='{"meow": "bark"}',
)
),
]
)
),
]
),
]
),
fetch.FetchResponse(
topics=[
fetch.TopicResponse(
name="test.topic",
partitions=[
fetch.PartitionResponse(
partition_id=1,
error_code=errors.no_error,
highwater_mark_offset=2,
message_set=messages.MessageSet([])
),
]
),
]
),
]
)
c = FakeConsumer(["kafka01", "kafka02"])
yield c.connect()
msgs = yield c.consume("test.topic")
possible_orders = [
[{"meow": "bark"}, {"foo": "bar"}, {"bwee": "bwoo"}],
[{"foo": "bar"}, {"bwee": "bwoo"}, {"meow": "bark"}],
]
self.assertTrue(
any([msgs == possibility for possibility in possible_orders])
)
self.assert_sent(
broker_id=3,
request=fetch.FetchRequest(
replica_id=-1,
max_wait_time=1000,
min_bytes=1,
topics=[
fetch.TopicRequest(
name="test.topic",
partitions=[
fetch.PartitionRequest(
partition_id=0,
offset=0,
max_bytes=(1024 * 1024),
),
]
)
]
)
)
self.assert_sent(
broker_id=8,
request=fetch.FetchRequest(
replica_id=-1,
max_wait_time=1000,
min_bytes=1,
topics=[
fetch.TopicRequest(
name="test.topic",
partitions=[
fetch.PartitionRequest(
partition_id=1,
offset=0,
max_bytes=(1024 * 1024),
),
]
)
]
)
)
msgs = yield c.consume("test.topic")
self.assertEqual(msgs, [])
self.assert_sent(
broker_id=3,
request=fetch.FetchRequest(
replica_id=-1,
max_wait_time=1000,
min_bytes=1,
topics=[
fetch.TopicRequest(
name="test.topic",
partitions=[
fetch.PartitionRequest(
partition_id=0,
offset=2,
max_bytes=(1024 * 1024),
),
]
)
]
)
)
self.assert_sent(
broker_id=8,
request=fetch.FetchRequest(
replica_id=-1,
max_wait_time=1000,
min_bytes=1,
topics=[
fetch.TopicRequest(
name="test.topic",
partitions=[
fetch.PartitionRequest(
partition_id=1,
offset=1,
max_bytes=(1024 * 1024),
),
]
)
]
)
)
def test_offset_out_of_range_error(self):
self.add_topic("test.topic", leaders=(3,))
self.set_responses(
broker_id=3, api="fetch",
responses=[
fetch.FetchResponse(
topics=[
fetch.TopicResponse(
name="test.topic",
partitions=[
fetch.PartitionResponse(
partition_id=0,
error_code=errors.offset_out_of_range,
highwater_mark_offset=2,
message_set=messages.MessageSet([])
),
]
),
]
),
fetch.FetchResponse(
topics=[
fetch.TopicResponse(
name="test.topic",
partitions=[
fetch.PartitionResponse(
partition_id=0,
error_code=errors.no_error,
highwater_mark_offset=2,
message_set=messages.MessageSet(
messages=[
(
0,
messages.Message(
magic=0, attributes=0,
key=None,
value='{"cat": "dog"}',
)
),
]
)
),
]
),
]
),
]
)
c = FakeConsumer(["kafka01"])
yield c.connect()
c.offsets["test.topic"][0] = 80
c.synced_offsets.add("test.topic")
msgs = yield c.consume("test.topic")
self.assertEqual(msgs, [])
self.assert_sent(
broker_id=3,
request=fetch.FetchRequest(
replica_id=-1,
max_wait_time=1000,
min_bytes=1,
topics=[
fetch.TopicRequest(
name="test.topic",
partitions=[
fetch.PartitionRequest(
partition_id=0,
offset=80,
max_bytes=(1024 * 1024),
),
]
)
]
)
)
msgs = yield c.consume("test.topic")
self.assertEqual(msgs, [{"cat": "dog"}])
self.assert_sent(
broker_id=3,
request=fetch.FetchRequest(
replica_id=-1,
max_wait_time=1000,
min_bytes=1,
topics=[
fetch.TopicRequest(
name="test.topic",
partitions=[
fetch.PartitionRequest(
partition_id=0,
offset=0,
max_bytes=(1024 * 1024),
),
]
)
]
)
)
def test_retriable_code_when_consuming(self):
self.add_topic("test.topic", leaders=(3,))
self.set_responses(
broker_id=3, api="fetch",
responses=[
fetch.FetchResponse(
topics=[
fetch.TopicResponse(
name="test.topic",
partitions=[
fetch.PartitionResponse(
partition_id=0,
error_code=errors.leader_not_available,
highwater_mark_offset=2,
message_set=messages.MessageSet([])
),
]
),
]
),
fetch.FetchResponse(
topics=[
fetch.TopicResponse(
name="test.topic",
partitions=[
fetch.PartitionResponse(
partition_id=0,
error_code=errors.no_error,
highwater_mark_offset=2,
message_set=messages.MessageSet(
messages=[
(
0,
messages.Message(
magic=0, attributes=0,
key=None,
value='{"cat": "dog"}',
)
),
]
)
),
]
),
]
),
]
)
c = FakeConsumer(["kafka01"])
yield c.connect()
msgs = yield c.consume("test.topic")
c.cluster.heal.assert_called_once_with()
self.assertEqual(msgs, [])
self.assertEqual(len(self.requests_by_broker[3]), 1)
msgs = yield c.consume("test.topic")
self.assertEqual(msgs, [{"cat": "dog"}])
self.assertEqual(len(self.requests_by_broker[3]), 2)
def test_custom_deserializer_and_options(self):
self.add_topic("test.topic", leaders=(3,))
self.set_responses(
broker_id=3, api="fetch",
responses=[
fetch.FetchResponse(
topics=[
fetch.TopicResponse(
name="test.topic",
partitions=[
fetch.PartitionResponse(
partition_id=0,
error_code=errors.no_error,
highwater_mark_offset=2,
message_set=messages.MessageSet(
messages=[
(
0,
messages.Message(
magic=0, attributes=0,
key=None,
value='cat',
)
),
(
1,
messages.Message(
magic=0, attributes=0,
key=None,
value='dog',
)
),
]
)
),
]
),
]
),
]
)
results = [Exception(), "bark"]
def deserializer(val):
result = results.pop(0)
if isinstance(result, Exception):
raise result
return "%s: %s" % (val, result)
c = FakeConsumer(
["kafka01", "kafka02"],
deserializer=deserializer,
max_wait_time=500,
min_bytes=1024, max_bytes=1024
)
yield c.connect()
msgs = yield c.consume("test.topic")
self.assertEqual(msgs, ["dog: bark"])
self.assert_sent(
broker_id=3,
request=fetch.FetchRequest(
replica_id=-1,
max_wait_time=500,
min_bytes=1024,
topics=[
fetch.TopicRequest(
name="test.topic",
partitions=[
fetch.PartitionRequest(
partition_id=0,
offset=0,
max_bytes=1024,
),
]
)
]
)
)
def test_unretriable_error(self):
self.add_topic("test.topic", leaders=(1,))
self.set_responses(
broker_id=1, api="produce",
responses=[
produce.ProduceResponse(
topics=[
produce.TopicResponse(
name="test.topic",
partitions=[
produce.PartitionResponse(
partition_id=0,
error_code=errors.unknown,
offset=8001,
),
]
),
]
),
produce.ProduceResponse(
topics=[
produce.TopicResponse(
name="test.topic",
partitions=[
produce.PartitionResponse(
partition_id=0,
error_code=errors.no_error,
offset=8001,
),
]
),
]
),
]
)
msgs = ["foo", "bar"]
p = producer.Producer(["kafka01"], batch_size=1)
for msg in msgs:
yield p.produce("test.topic", msg)
self.assertEqual(len(self.requests_by_broker[1]), 2)
self.assert_sent(
broker_id=1,
request=produce.ProduceRequest(
required_acks=-1,
timeout=500,
topics=[
produce.TopicRequest(
name="test.topic",
partitions=[
produce.PartitionRequest(
partition_id=0,
message_set=messages.MessageSet.compressed(
compression=None,
msgs=[
messages.Message(
magic=0, attributes=0, key=None,
value=p.serializer("bar")
),
]
)
)
]
)
]
)
)
def test_commit_offset_with_large_metadata(self):
self.add_topic("test.topic", leaders=(1, 8))
self.allocator.allocation = {"test.topic": [1]}
self.set_responses(broker_id=3,
api="offset_fetch",
responses=[
offset_fetch.OffsetFetchResponse(topics=[
offset_fetch.TopicResponse(
name="test.topic",
partitions=[
offset_fetch.PartitionResponse(
error_code=errors.no_error,
partition_id=0,
offset=80,
metadata="committed, ok!")
])
]),
])
self.set_responses(
broker_id=3,
api="offset_commit",
responses=[
offset_commit.OffsetCommitResponse(topics=[
offset_commit.TopicResponse(
name="test.topic",
partitions=[
offset_commit.PartitionResponse(
error_code=(errors.offset_metadata_too_large),
partition_id=1,
)
]),
]),
offset_commit.OffsetCommitResponse(topics=[
offset_commit.TopicResponse(
name="test.topic",
partitions=[
offset_commit.PartitionResponse(
error_code=errors.no_error,
partition_id=1,
)
]),
]),
])
self.set_responses(
broker_id=8,
api="fetch",
responses=[
fetch.FetchResponse(topics=[
fetch.TopicResponse(
name="test.topic",
partitions=[
fetch.PartitionResponse(
partition_id=1,
error_code=errors.no_error,
highwater_mark_offset=2,
message_set=messages.MessageSet([
(80,
messages.Message(
magic=0,
attributes=0,
key=None,
value='{"cat": "meow"}',
)),
])),
]),
])
])
c = grouped.GroupedConsumer(["kafka01", "kafka02"],
"work-group",
zk_hosts=["zk01", "zk02", "zk03"])
yield c.connect()
msgs = yield c.consume("test.topic")
self.assertEqual(msgs, [{"cat": "meow"}])
self.assert_sent(
broker_id=3,
request=offset_commit.OffsetCommitV0Request(
group="work-group",
topics=[
offset_commit.TopicRequest(
name="test.topic",
partitions=[
offset_commit.PartitionRequest(
partition_id=1,
offset=81,
metadata="committed by %s" % c.name)
])
]))
self.assert_sent(broker_id=3,
request=offset_commit.OffsetCommitV0Request(
group="work-group",
topics=[
offset_commit.TopicRequest(
name="test.topic",
partitions=[
offset_commit.PartitionRequest(
partition_id=1,
offset=81,
metadata="")
])
]))