def test_owned_broker_flush_batching_by_max_request_size(self):
"""Test that producer batches messages into the batches no larger than
`max_request_size`
"""
large_payload = b''.join([uuid4().bytes for i in range(5000)])
producer = self._get_producer(
auto_start=False
)
# setup producer but do not start actually sending messages
partition = producer._topic.partitions[0]
owned_broker = OwnedBroker(producer, partition.leader, auto_start=False)
for i in range(100):
msg = Message(large_payload, partition_id=0)
owned_broker.enqueue(msg)
batch = owned_broker.flush(0, producer._max_request_size)
assert len(batch) < 100
assert sum([len(m.value) for m in batch]) < producer._max_request_size
# iterate through the rest of the batches and test the same invariant
while batch:
batch = owned_broker.flush(0, producer._max_request_size)
assert len(batch) < 100
assert sum([len(m.value) for m in batch]) < producer._max_request_size
def test_owned_broker_flush_message_larger_then_max_request_size(self):
"""Test that producer batches messages into the batches no larger than
`max_request_size`
"""
large_payload = b''.join([uuid4().bytes for i in range(50000)])
producer = self._get_producer(
auto_start=False
)
# setup producer but do not start actually sending messages
partition = producer._topic.partitions[0]
owned_broker = OwnedBroker(producer, partition.leader, auto_start=False)
delivery_report_queue = producer._cluster.handler.Queue()
msg = Message(
large_payload,
partition_id=0,
delivery_report_q=delivery_report_queue
)
owned_broker.enqueue(msg)
max_request_size = 1000
assert max_request_size < len(msg)
owned_broker.flush(0, max_request_size)
q_msg, exc = delivery_report_queue.get()
assert q_msg is msg
assert isinstance(exc, MessageSizeTooLarge)
def test_owned_broker_flush_batching_by_max_request_size(self):
"""Test that producer batches messages into the batches no larger than
`max_request_size`
"""
large_payload = b''.join([uuid4().bytes for i in range(5000)])
producer = self._get_producer(auto_start=False)
# setup producer but do not start actually sending messages
partition = producer._topic.partitions[0]
owned_broker = OwnedBroker(producer,
partition.leader,
auto_start=False)
for i in range(100):
msg = Message(large_payload, partition_id=0)
owned_broker.enqueue(msg)
batch = owned_broker.flush(0, producer._max_request_size)
assert len(batch) < 100
assert sum([len(m.value) for m in batch]) < producer._max_request_size
# iterate through the rest of the batches and test the same invariant
while batch:
batch = owned_broker.flush(0, producer._max_request_size)
assert len(batch) < 100
assert sum([len(m.value)
for m in batch]) < producer._max_request_size
def test_owned_broker_flush_message_larger_then_max_request_size(self):
"""Test that producer batches messages into the batches no larger than
`max_request_size`
"""
large_payload = b''.join([uuid4().bytes for i in range(50000)])
producer = self._get_producer(auto_start=False)
# setup producer but do not start actually sending messages
partition = producer._topic.partitions[0]
owned_broker = OwnedBroker(producer,
partition.leader,
auto_start=False)
delivery_report_queue = producer._cluster.handler.Queue()
msg = Message(large_payload,
partition_id=0,
delivery_report_q=delivery_report_queue)
owned_broker.enqueue(msg)
max_request_size = 1000
assert max_request_size < len(msg)
owned_broker.flush(0, max_request_size)
q_msg, exc = delivery_report_queue.get()
assert q_msg is msg
assert isinstance(exc, MessageSizeTooLarge)