def test_defaults(self):
p = producer.Producer(["kafka01", "kafka02"])
self.assertEqual(p.batch_size, 1)
self.assertEqual(p.required_acks, -1)
self.assertEqual(p.ack_timeout, 500)
self.assertEqual(p.compression, None)
def test_producing_to_unknown_topic_never_sends(self):
self.add_topic("test.topic", leaders=(1,))
p = producer.Producer(["kafka01"], batch_size=1)
yield p.produce("other.topic", "foo")
self.assertEqual(self.requests_by_broker[1], [])
def test_flush_with_no_pending_sends(self):
self.add_topic("test.topic", leaders=(1,))
p = producer.Producer(
["kafka01"],
key_maker=attribute_key, partitioner=key_partitioner
)
yield p.flush()
self.assertEqual(self.requests_by_broker, {})
def test_leader_not_present_at_first(self):
self.add_topic("test.topic", leaders=(7,))
p = producer.Producer(["kafka01"], batch_size=1)
yield p.produce("test.topic", "foo")
self.add_topic("test.topic", leaders=(1,))
yield p.cluster.heal()
yield p.produce("test.topic", "bar")
self.assert_sent(
broker_id=1,
request=produce.ProduceRequest(
required_acks=-1,
timeout=500,
topics=[
produce.TopicRequest(
name="test.topic",
partitions=[
produce.PartitionRequest(
partition_id=0,
message_set=messages.MessageSet.compressed(
compression=None,
msgs=[
messages.Message(
magic=0, attributes=0, key=None,
value=p.serializer("foo")
),
messages.Message(
magic=0, attributes=0, key=None,
value=p.serializer("bar")
),
]
)
)
]
)
]
)
)
def test_unretriable_error(self):
self.add_topic("test.topic", leaders=(1,))
self.set_responses(
broker_id=1, api="produce",
responses=[
produce.ProduceResponse(
topics=[
produce.TopicResponse(
name="test.topic",
partitions=[
produce.PartitionResponse(
partition_id=0,
error_code=errors.unknown,
offset=8001,
),
]
),
]
),
produce.ProduceResponse(
topics=[
produce.TopicResponse(
name="test.topic",
partitions=[
produce.PartitionResponse(
partition_id=0,
error_code=errors.no_error,
offset=8001,
),
]
),
]
),
]
)
msgs = ["foo", "bar"]
p = producer.Producer(["kafka01"], batch_size=1)
for msg in msgs:
yield p.produce("test.topic", msg)
self.assertEqual(len(self.requests_by_broker[1]), 2)
self.assert_sent(
broker_id=1,
request=produce.ProduceRequest(
required_acks=-1,
timeout=500,
topics=[
produce.TopicRequest(
name="test.topic",
partitions=[
produce.PartitionRequest(
partition_id=0,
message_set=messages.MessageSet.compressed(
compression=None,
msgs=[
messages.Message(
magic=0, attributes=0, key=None,
value=p.serializer("bar")
),
]
)
)
]
)
]
)
)
def test_routing_to_partitions(self):
self.add_topic("test.topic", leaders=(1, 1, 8, 3))
self.set_responses(
broker_id=1, api="produce",
responses=[
produce.ProduceResponse(
topics=[
produce.TopicResponse(
name="test.topic",
partitions=[
produce.PartitionResponse(
partition_id=0,
error_code=errors.no_error,
offset=8000,
),
]
),
]
),
produce.ProduceResponse(
topics=[
produce.TopicResponse(
name="test.topic",
partitions=[
produce.PartitionResponse(
partition_id=1,
error_code=errors.no_error,
offset=8000,
),
]
),
]
),
produce.ProduceResponse(
topics=[
produce.TopicResponse(
name="test.topic",
partitions=[
produce.PartitionResponse(
partition_id=0,
error_code=errors.no_error,
offset=8001,
),
]
),
]
),
]
)
self.set_responses(
broker_id=3, api="produce",
responses=[
produce.ProduceResponse(
topics=[
produce.TopicResponse(
name="test.topic",
partitions=[
produce.PartitionResponse(
partition_id=3,
error_code=errors.no_error,
offset=8000,
),
]
),
]
),
]
)
msgs = [
{"key": 0, "msg": "foo"},
{"key": 1, "msg": "bar"},
{"key": 3, "msg": "bwee"},
{"key": 0, "msg": "bwoo"},
]
p = producer.Producer(
["kafka01"],
key_maker=attribute_key, partitioner=key_partitioner
)
for msg in msgs:
yield p.produce("test.topic", msg)
self.assert_sent(
broker_id=1,
request=produce.ProduceRequest(
required_acks=-1,
timeout=500,
topics=[
produce.TopicRequest(
name="test.topic",
partitions=[
produce.PartitionRequest(
partition_id=0,
message_set=messages.MessageSet.compressed(
compression=None,
msgs=[
messages.Message(
magic=0, attributes=0,
key=0,
value=p.serializer(
{"msg": "foo", "key": 0}
)
)
]
)
)
]
)
]
)
)
self.assert_sent(
broker_id=1,
request=produce.ProduceRequest(
required_acks=-1,
timeout=500,
topics=[
produce.TopicRequest(
name="test.topic",
partitions=[
produce.PartitionRequest(
partition_id=1,
message_set=messages.MessageSet.compressed(
compression=None,
msgs=[
messages.Message(
magic=0, attributes=0,
key=1,
value=p.serializer(
{"msg": "bar", "key": 1}
)
)
]
)
)
]
)
]
)
)
self.assert_sent(
broker_id=1,
request=produce.ProduceRequest(
required_acks=-1,
timeout=500,
topics=[
produce.TopicRequest(
name="test.topic",
partitions=[
produce.PartitionRequest(
partition_id=0,
message_set=messages.MessageSet.compressed(
compression=None,
msgs=[
messages.Message(
magic=0, attributes=0,
key=0,
value=p.serializer(
{"msg": "bwoo", "key": 0}
)
)
]
)
)
]
)
]
)
)
self.assert_sent(
broker_id=3,
request=produce.ProduceRequest(
required_acks=-1,
timeout=500,
topics=[
produce.TopicRequest(
name="test.topic",
partitions=[
produce.PartitionRequest(
partition_id=3,
message_set=messages.MessageSet.compressed(
compression=None,
msgs=[
messages.Message(
magic=0, attributes=0,
key=3,
value=p.serializer(
{"msg": "bwee", "key": 3}
)
)
]
)
)
]
)
]
)
)
def test_gzip_compression(self):
self.add_topic("test.topic", leaders=(1,))
self.set_responses(
broker_id=1, api="produce",
responses=[
produce.ProduceResponse(
topics=[
produce.TopicResponse(
name="test.topic",
partitions=[
produce.PartitionResponse(
partition_id=0,
error_code=errors.no_error,
offset=8003,
),
]
),
]
),
]
)
msgs = [u"foö", "bar"]
p = producer.Producer(
["kafka01"], batch_size=2, compression=constants.GZIP
)
for msg in msgs:
yield p.produce("test.topic", msg)
self.assert_sent(
broker_id=1,
request=produce.ProduceRequest(
required_acks=-1,
timeout=500,
topics=[
produce.TopicRequest(
name="test.topic",
partitions=[
produce.PartitionRequest(
partition_id=0,
message_set=messages.MessageSet.compressed(
compression=constants.GZIP,
msgs=[
messages.Message(
magic=0,
attributes=0,
key=None,
value=p.serializer(u"foö"),
),
messages.Message(
magic=0,
attributes=0,
key=None,
value=p.serializer("bar"),
)
]
)
)
]
)
]
)
)
def test_default_routing(self, mock_random):
self.add_topic("test.topic", leaders=(1, 8))
choices_to_make = [0, 1]
def get_next_choice(*args):
return choices_to_make.pop(0)
mock_random.choice.side_effect = get_next_choice
self.set_responses(
broker_id=1, api="produce",
responses=[
produce.ProduceResponse(
topics=[
produce.TopicResponse(
name="test.topic",
partitions=[
produce.PartitionResponse(
partition_id=0,
error_code=errors.no_error,
offset=8000,
),
]
),
]
),
]
)
self.set_responses(
broker_id=8, api="produce",
responses=[
produce.ProduceResponse(
topics=[
produce.TopicResponse(
name="test.topic",
partitions=[
produce.PartitionResponse(
partition_id=1,
error_code=errors.no_error,
offset=8000,
),
]
),
]
),
]
)
msgs = ["foo", "bar"]
p = producer.Producer(["kafka01"])
for msg in msgs:
yield p.produce("test.topic", msg)
self.assert_sent(
broker_id=1,
request=produce.ProduceRequest(
required_acks=-1,
timeout=500,
topics=[
produce.TopicRequest(
name="test.topic",
partitions=[
produce.PartitionRequest(
partition_id=0,
message_set=messages.MessageSet.compressed(
compression=None,
msgs=[
messages.Message(
magic=0, attributes=0, key=None,
value=p.serializer("foo")
)
]
)
)
]
)
]
)
)
self.assert_sent(
broker_id=8,
request=produce.ProduceRequest(
required_acks=-1,
timeout=500,
topics=[
produce.TopicRequest(
name="test.topic",
partitions=[
produce.PartitionRequest(
partition_id=1,
message_set=messages.MessageSet.compressed(
compression=None,
msgs=[
messages.Message(
magic=0, attributes=0, key=None,
value=p.serializer("bar")
)
]
)
)
]
)
]
)
)
