def test_producer_send_messages(self):
start_offset0 = yield self.current_offset(self.topic, 0)
start_offset1 = yield self.current_offset(self.topic, 1)
producer = Producer(self.client)
# Goes to first partition
resp = yield producer.send_messages(
self.topic, msgs=[self.msg("one"), self.msg("two")])
self.assert_produce_response(resp, start_offset0)
# Goes to the 2nd partition
resp = yield producer.send_messages(self.topic,
msgs=[self.msg("three")])
self.assert_produce_response(resp, start_offset1)
# fetch the messages back and make sure they are as expected
yield self.assert_fetch_offset(
0, start_offset0, [self.msg("one"), self.msg("two")])
yield self.assert_fetch_offset(
1, start_offset1, [self.msg("three")])
# Goes back to the first partition because there's only two partitions
resp = yield producer.send_messages(
self.topic, msgs=[self.msg("four"), self.msg("five")])
self.assert_produce_response(resp, start_offset0+2)
yield self.assert_fetch_offset(
0, start_offset0, [self.msg("one"),
self.msg("two"),
self.msg("four"),
self.msg("five")])
yield producer.stop()
def test_producer_round_robin_partitioner(self):
start_offset0 = yield self.current_offset(self.topic, 0)
start_offset1 = yield self.current_offset(self.topic, 1)
producer = Producer(self.client,
partitioner_class=RoundRobinPartitioner)
resp1 = yield producer.send_messages(
self.topic, b"key1", [self.msg("one")])
resp2 = yield producer.send_messages(
self.topic, b"key2", [self.msg("two")])
resp3 = yield producer.send_messages(
self.topic, b"key3", [self.msg("three")])
resp4 = yield producer.send_messages(
self.topic, b"key4", [self.msg("four")])
self.assert_produce_response(resp1, start_offset0+0)
self.assert_produce_response(resp2, start_offset1+0)
self.assert_produce_response(resp3, start_offset0+1)
self.assert_produce_response(resp4, start_offset1+1)
yield self.assert_fetch_offset(
0, start_offset0, [self.msg("one"), self.msg("three")])
yield self.assert_fetch_offset(
1, start_offset1, [self.msg("two"), self.msg("four")])
yield producer.stop()
def test_switch_leader(self):
producer = Producer(self.client)
topic = self.topic
try:
for index in range(1, 3):
# cause the client to establish connections to all the brokers
log.debug("Pass: %d. Sending 10 random messages", index)
yield self._send_random_messages(producer, topic, 10)
# Ensure that the follower is in sync
log.debug("Ensuring topic/partition is replicated.")
part_meta = self.client.partition_meta[TopicAndPartition(
self.topic, 0)]
# Ensure the all the replicas are in-sync before proceeding
while len(part_meta.isr) != 2: # pragma: no cover
log.debug("Waiting for Kafka replica to become synced")
if len(part_meta.replicas) != 2:
log.error("Kafka replica 'disappeared'!"
"Partitition Meta: %r", part_meta)
yield async_delay(1.0)
yield self.client.load_metadata_for_topics(self.topic)
part_meta = self.client.partition_meta[TopicAndPartition(
self.topic, 0)]
# kill leader for partition 0
log.debug("Killing leader of partition 0")
broker, kill_time = self._kill_leader(topic, 0)
log.debug("Sending 1 more message: 'part 1'")
yield producer.send_messages(topic, msgs=['part 1'])
log.debug("Sending 1 more message: 'part 2'")
yield producer.send_messages(topic, msgs=['part 2'])
# send to new leader
log.debug("Sending 10 more messages")
yield self._send_random_messages(producer, topic, 10)
# Make sure the ZK ephemeral time (~6 seconds) has elapsed
wait_time = (kill_time + 6.5) - time.time()
if wait_time > 0:
log.debug("Waiting: %4.2f for ZK timeout", wait_time)
yield async_delay(wait_time)
# restart the kafka broker
log.debug("Restarting the broker")
broker.restart()
# count number of messages
log.debug("Getting message count")
count = yield self._count_messages(topic)
self.assertEqual(count, 22 * index)
finally:
log.debug("Stopping the producer")
yield producer.stop()
log.debug("Producer stopped")
log.debug("Test complete.")
def test_producer_batched_by_time(self):
start_offset0 = yield self.current_offset(self.topic, 0)
start_offset1 = yield self.current_offset(self.topic, 1)
# This needs to be big enough that the operations between starting the
# producer and the sleep take less time than this... I made
# it large enough that the test would still pass even with my Macbook's
# cores all pegged by a load generator.
batchtime = 5
try:
producer = Producer(self.client, batch_send=True,
batch_every_n=0, batch_every_t=batchtime)
startTime = time.time()
# Send 4 messages and do a fetch
send1D = producer.send_messages(
self.topic, msgs=[self.msg("one"), self.msg("two"),
self.msg("three"), self.msg("four")])
# set assert_fetch_offset() to wait for 0.1 secs on the server-side
# before returning no result. So, these calls should take 0.2sec
yield self.assert_fetch_offset(0, start_offset0, [], max_wait=0.1)
yield self.assert_fetch_offset(1, start_offset1, [], max_wait=0.1)
# Messages shouldn't have sent out yet, so we shouldn't have
# response from server yet on having received/responded to request
self.assertNoResult(send1D)
# Sending 3 more messages should NOT trigger the send, as less than
# 1 sec. elapsed by here, so send2D should still have no result.
send2D = producer.send_messages(
self.topic, msgs=[self.msg("five"), self.msg("six"),
self.msg("seven")])
# still no messages...
yield self.assert_fetch_offset(0, start_offset0, [], max_wait=0.1)
yield self.assert_fetch_offset(1, start_offset1, [], max_wait=0.1)
# Still no result on send, and send should NOT have gone out.
self.assertNoResult(send2D)
# Wait the timeout out. It'd be nicer to be able to just 'advance'
# the reactor, but since we need the network so...
yield async_delay(batchtime - (time.time() - startTime) + 0.05, clock=self.reactor)
# We need to yield to the reactor to have it process the response
# from the broker. Both send1D and send2D should then have results.
resp1 = yield send1D
resp2 = yield send2D
# ensure the 2 batches went into the proper partitions...
self.assert_produce_response(resp1, start_offset0)
self.assert_produce_response(resp2, start_offset1)
# Should be able to get messages now
yield self.assert_fetch_offset(
0, start_offset0, [self.msg("one"), self.msg("two"),
self.msg("three"), self.msg("four")])
yield self.assert_fetch_offset(
1, start_offset1, [self.msg("five"), self.msg("six"),
self.msg("seven")])
finally:
yield producer.stop()
def test_producer_batched_by_messages(self):
start_offset0 = yield self.current_offset(self.topic, 0)
start_offset1 = yield self.current_offset(self.topic, 1)
producer = Producer(self.client, batch_send=True, batch_every_n=10,
batch_every_b=0, batch_every_t=0)
# Send 4 messages and do a fetch. Fetch should timeout, and send
# deferred shouldn't have a result yet...
send1D = producer.send_messages(
self.topic, msgs=[self.msg("one"), self.msg("two"),
self.msg("three"), self.msg("four")])
# by default the assert_fetch_offset() waits for 0.5 secs on the server
# side before returning no result. So, these two calls should take 1sec
yield self.assert_fetch_offset(0, start_offset0, [])
yield self.assert_fetch_offset(1, start_offset1, [])
# Messages shouldn't have sent out yet, so we shouldn't have
# response from server yet on having received/responded to the request
self.assertNoResult(send1D)
# Sending 3 more messages won't trigger the send either (7 total)
send2D = producer.send_messages(
self.topic, msgs=[self.msg("five"), self.msg("six"),
self.msg("seven")])
# make sure no messages on server...
yield self.assert_fetch_offset(0, start_offset0, [])
yield self.assert_fetch_offset(1, start_offset1, [])
# Still no result on send
self.assertNoResult(send2D)
# send final batch which will be on partition 0 again...
send3D = producer.send_messages(
self.topic, msgs=[self.msg("eight"), self.msg("nine"),
self.msg("ten"), self.msg("eleven")])
# Now do a fetch, again waiting for up to 0.5 seconds for the response
# All four messages sent in first batch (to partition 0, given default
# R-R, start-at-zero partitioner), and 4 in 3rd batch
yield self.assert_fetch_offset(
0, start_offset0, [self.msg("one"), self.msg("two"),
self.msg("three"), self.msg("four"),
self.msg("eight"), self.msg("nine"),
self.msg("ten"), self.msg("eleven")],
fetch_size=2048)
# Fetch from partition:1 should have all messages in 2nd batch
yield self.assert_fetch_offset(
1, start_offset1, [self.msg("five"), self.msg("six"),
self.msg("seven")])
# make sure the deferreds fired with the proper result
resp1 = self.successResultOf(send1D)
resp2 = self.successResultOf(send2D)
resp3 = self.successResultOf(send3D)
self.assert_produce_response(resp1, start_offset0)
self.assert_produce_response(resp2, start_offset1)
self.assert_produce_response(resp3, start_offset0)
# cleanup
yield producer.stop()
def test_acks_none(self):
start_offset0 = yield self.current_offset(self.topic, 0)
yield self.current_offset(self.topic, 1)
producer = Producer(self.client, req_acks=PRODUCER_ACK_NOT_REQUIRED)
resp = yield producer.send_messages(self.topic, msgs=[self.msg("one")])
self.assertEqual(resp, None)
yield self.assert_fetch_offset(0, start_offset0, [self.msg("one")])
yield producer.stop()
def test_producer_hashed_partitioner(self):
start_offset0 = yield self.current_offset(self.topic, 0)
start_offset1 = yield self.current_offset(self.topic, 1)
producer = Producer(self.client, partitioner_class=HashedPartitioner)
resp1 = yield producer.send_messages(self.topic, b'1',
[self.msg("one")])
resp2 = yield producer.send_messages(self.topic, b'2',
[self.msg("two")])
resp3 = yield producer.send_messages(self.topic, b'1',
[self.msg("three")])
resp4 = yield producer.send_messages(self.topic, b'1',
[self.msg("four")])
resp5 = yield producer.send_messages(self.topic, b'2',
[self.msg("five")])
self.assert_produce_response(resp2, start_offset0 + 0)
self.assert_produce_response(resp5, start_offset0 + 1)
self.assert_produce_response(resp1, start_offset1 + 0)
self.assert_produce_response(resp3, start_offset1 + 1)
self.assert_produce_response(resp4, start_offset1 + 2)
yield self.assert_fetch_offset(
0,
start_offset0,
[
self.msg("two"),
self.msg("five"),
],
)
yield self.assert_fetch_offset(
1,
start_offset1,
[
self.msg("one"),
self.msg("three"),
self.msg("four"),
],
)
yield producer.stop()
def test_acks_local_write(self):
start_offset0 = yield self.current_offset(self.topic, 0)
yield self.current_offset(self.topic, 1)
producer = Producer(self.client, req_acks=PRODUCER_ACK_LOCAL_WRITE)
resp = yield producer.send_messages(self.topic, msgs=[self.msg("one")])
self.assert_produce_response(resp, start_offset0)
yield self.assert_fetch_offset(0, start_offset0, [self.msg("one")])
yield producer.stop()
def test_acks_none(self):
start_offset0 = yield self.current_offset(self.topic, 0)
yield self.current_offset(self.topic, 1)
producer = Producer(
self.client, req_acks=PRODUCER_ACK_NOT_REQUIRED)
resp = yield producer.send_messages(self.topic, msgs=[self.msg("one")])
self.assertEqual(resp, None)
yield self.assert_fetch_offset(0, start_offset0, [self.msg("one")])
yield producer.stop()
def test_acks_local_write(self):
start_offset0 = yield self.current_offset(self.topic, 0)
yield self.current_offset(self.topic, 1)
producer = Producer(
self.client, req_acks=PRODUCER_ACK_LOCAL_WRITE)
resp = yield producer.send_messages(self.topic, msgs=[self.msg("one")])
self.assert_produce_response(resp, start_offset0)
yield self.assert_fetch_offset(0, start_offset0, [self.msg("one")])
yield producer.stop()
def test_acks_all_replicas(self):
start_offset0 = yield self.current_offset(self.topic, 0)
start_offset1 = yield self.current_offset(self.topic, 1)
producer = Producer(self.client, req_acks=PRODUCER_ACK_ALL_REPLICAS)
resp = yield producer.send_messages(self.topic, msgs=[self.msg("one")])
self.assert_produce_response(resp, start_offset0)
yield self.assert_fetch_offset(0, start_offset0, [self.msg("one")])
yield self.assert_fetch_offset(1, start_offset1, [])
yield producer.stop()
def test_producer_round_robin_partitioner_random_start(self):
try:
RoundRobinPartitioner.set_random_start(True)
producer = Producer(self.client,
partitioner_class=RoundRobinPartitioner)
# Two partitions, so 1st and 3rd reqs should go to same part, but
# 2nd should go to different. Other than that, without statistical
# test, we can't say much... Partitioner tests should ensure that
# we really aren't always starting on a non-random partition...
resp1 = yield producer.send_messages(
self.topic, msgs=[self.msg("one"), self.msg("two")])
resp2 = yield producer.send_messages(
self.topic, msgs=[self.msg("three")])
resp3 = yield producer.send_messages(
self.topic, msgs=[self.msg("four"), self.msg("five")])
self.assertEqual(resp1.partition, resp3.partition)
self.assertNotEqual(resp1.partition, resp2.partition)
yield producer.stop()
finally:
RoundRobinPartitioner.set_random_start(False)
def test_producer_round_robin_partitioner_random_start(self):
try:
RoundRobinPartitioner.set_random_start(True)
producer = Producer(self.client,
partitioner_class=RoundRobinPartitioner)
# Two partitions, so 1st and 3rd reqs should go to same part, but
# 2nd should go to different. Other than that, without statistical
# test, we can't say much... Partitioner tests should ensure that
# we really aren't always starting on a non-random partition...
resp1 = yield producer.send_messages(
self.topic, msgs=[self.msg("one"), self.msg("two")])
resp2 = yield producer.send_messages(
self.topic, msgs=[self.msg("three")])
resp3 = yield producer.send_messages(
self.topic, msgs=[self.msg("four"), self.msg("five")])
self.assertEqual(resp1.partition, resp3.partition)
self.assertNotEqual(resp1.partition, resp2.partition)
yield producer.stop()
finally:
RoundRobinPartitioner.set_random_start(False)
def test_producer_hashed_partitioner(self):
start_offset0 = yield self.current_offset(self.topic, 0)
start_offset1 = yield self.current_offset(self.topic, 1)
producer = Producer(self.client,
partitioner_class=HashedPartitioner)
resp1 = yield producer.send_messages(
self.topic, b'1', [self.msg("one")])
resp2 = yield producer.send_messages(
self.topic, b'2', [self.msg("two")])
resp3 = yield producer.send_messages(
self.topic, b'1', [self.msg("three")])
resp4 = yield producer.send_messages(
self.topic, b'1', [self.msg("four")])
resp5 = yield producer.send_messages(
self.topic, b'2', [self.msg("five")])
self.assert_produce_response(resp2, start_offset0+0)
self.assert_produce_response(resp5, start_offset0+1)
self.assert_produce_response(resp1, start_offset1+0)
self.assert_produce_response(resp3, start_offset1+1)
self.assert_produce_response(resp4, start_offset1+2)
yield self.assert_fetch_offset(
0, start_offset0, [
self.msg("two"),
self.msg("five"),
])
yield self.assert_fetch_offset(
1, start_offset1, [
self.msg("one"),
self.msg("three"),
self.msg("four"),
])
yield producer.stop()
def test_acks_all_replicas(self):
start_offset0 = yield self.current_offset(self.topic, 0)
start_offset1 = yield self.current_offset(self.topic, 1)
producer = Producer(
self.client,
req_acks=PRODUCER_ACK_ALL_REPLICAS)
resp = yield producer.send_messages(self.topic, msgs=[self.msg("one")])
self.assert_produce_response(resp, start_offset0)
yield self.assert_fetch_offset(0, start_offset0, [self.msg("one")])
yield self.assert_fetch_offset(1, start_offset1, [])
yield producer.stop()
def test_producer_batched_gzipped_hashed_partitioner(self):
start_offset0 = yield self.current_offset(self.topic, 0)
start_offset1 = yield self.current_offset(self.topic, 1)
offsets = (start_offset0, start_offset1)
requests = []
msgs_by_partition = ([], [])
keys_by_partition = ([], [])
partitioner = HashedPartitioner(self.topic, [0, 1])
producer = Producer(self.client,
codec=CODEC_GZIP,
batch_send=True,
batch_every_n=100,
batch_every_t=None,
partitioner_class=HashedPartitioner)
# Send ten groups of messages, each with a different key
for i in range(10):
msg_group = []
key = 'Key: {}'.format(i)
part = partitioner.partition(key, [0, 1])
for j in range(10):
msg = self.msg('Group:{} Msg:{}'.format(i, j))
msg_group.append(msg)
msgs_by_partition[part].append(msg)
keys_by_partition[part].append(key)
request = producer.send_messages(self.topic,
key=key,
msgs=msg_group)
requests.append(request)
yield async_delay(.5) # Make the NoResult test have teeth...
if i < 9:
# This is to ensure we really are batching all the requests
self.assertNoResult(request)
# Now ensure we can retrieve the right messages from each partition
for part in [0, 1]:
yield self.assert_fetch_offset(part,
offsets[part],
msgs_by_partition[part],
keys_by_partition[part],
fetch_size=20480)
yield producer.stop()
def test_write_nonextant_topic(self):
"""
Test we can write to a non-extant topic (which will be auto-created)
simply by calling producer.send_messages with a long enough timeout.
"""
test_topics = ["{}-{}-{}".format(
self.id().split('.')[-1], i, random_string(10)) for i in range(10)]
producer = Producer(self.client, req_acks=PRODUCER_ACK_LOCAL_WRITE)
for topic in test_topics:
resp = yield producer.send_messages(topic, msgs=[self.msg(topic)])
# Make sure the send went ok
self.assert_produce_response(resp, 0)
# Make sure we can get the message back
yield self.assert_fetch_offset(
0, 0, [self.msg(topic)], topic=topic)
yield producer.stop()
def test_write_nonextant_topic(self):
"""
Test we can write to a non-extant topic (which will be auto-created)
simply by calling producer.send_messages with a long enough timeout.
"""
test_topics = ["{}-{}-{}".format(
self.id().split('.')[-1], i, random_string(10)) for i in range(10)]
producer = Producer(self.client, req_acks=PRODUCER_ACK_LOCAL_WRITE)
for topic in test_topics:
resp = yield producer.send_messages(topic, msgs=[self.msg(topic)])
# Make sure the send went ok
self.assert_produce_response(resp, 0)
# Make sure we can get the message back
yield self.assert_fetch_offset(
0, 0, [self.msg(topic)], topic=topic)
yield producer.stop()
def test_producer_batched_gzipped_hashed_partitioner(self):
start_offset0 = yield self.current_offset(self.topic, 0)
start_offset1 = yield self.current_offset(self.topic, 1)
offsets = (start_offset0, start_offset1)
requests = []
msgs_by_partition = ([], [])
keys_by_partition = ([], [])
partitioner = HashedPartitioner(self.topic, [0, 1])
producer = Producer(
self.client, codec=CODEC_GZIP, batch_send=True, batch_every_n=100,
batch_every_t=None, partitioner_class=HashedPartitioner)
# Send ten groups of messages, each with a different key
for i in range(10):
msg_group = []
key = 'Key: {}'.format(i).encode()
part = partitioner.partition(key, [0, 1])
for j in range(10):
msg = self.msg('Group:{} Msg:{}'.format(i, j))
msg_group.append(msg)
msgs_by_partition[part].append(msg)
keys_by_partition[part].append(key)
request = producer.send_messages(
self.topic, key=key, msgs=msg_group)
requests.append(request)
yield async_delay(.5) # Make the NoResult test have teeth...
if i < 9:
# This is to ensure we really are batching all the requests
self.assertNoResult(request)
# Now ensure we can retrieve the right messages from each partition
for part in [0, 1]:
yield self.assert_fetch_offset(
part, offsets[part], msgs_by_partition[part],
keys_by_partition[part], fetch_size=20480)
yield producer.stop()
def test_producer_batched_by_time(self):
start_offset0 = yield self.current_offset(self.topic, 0)
start_offset1 = yield self.current_offset(self.topic, 1)
# This needs to be big enough that the operations between starting the
# producer and the time.sleep() call take less time than this... I made
# it large enough that the test would still pass even with my Macbook's
# cores all pegged by a load generator.
batchtime = 5
try:
producer = Producer(self.client, batch_send=True,
batch_every_n=0, batch_every_t=batchtime)
startTime = time.time()
# Send 4 messages and do a fetch
send1D = producer.send_messages(
self.topic, msgs=[self.msg("one"), self.msg("two"),
self.msg("three"), self.msg("four")])
# set assert_fetch_offset() to wait for 0.1 secs on the server-side
# before returning no result. So, these calls should take 0.2sec
yield self.assert_fetch_offset(0, start_offset0, [], max_wait=0.1)
yield self.assert_fetch_offset(1, start_offset1, [], max_wait=0.1)
# Messages shouldn't have sent out yet, so we shouldn't have
# response from server yet on having received/responded to request
self.assertNoResult(send1D)
# Sending 3 more messages should NOT trigger the send, as less than
# 1 sec. elapsed by here, so send2D should still have no result.
send2D = producer.send_messages(
self.topic, msgs=[self.msg("five"), self.msg("six"),
self.msg("seven")])
# still no messages...
yield self.assert_fetch_offset(0, start_offset0, [], max_wait=0.1)
yield self.assert_fetch_offset(1, start_offset1, [], max_wait=0.1)
# Still no result on send, and send should NOT have gone out.
self.assertNoResult(send2D)
# Wait the timeout out. It'd be nicer to be able to just 'advance'
# the reactor, but since we need the network so...
time.sleep(batchtime - (time.time() - startTime) + 0.05)
# We need to yield to the reactor to have it process the response
# from the broker. Both send1D and send2D should then have results.
resp1 = yield send1D
resp2 = yield send2D
# ensure the 2 batches went into the proper partitions...
self.assert_produce_response(resp1, start_offset0)
self.assert_produce_response(resp2, start_offset1)
# Should be able to get messages now
yield self.assert_fetch_offset(
0, start_offset0, [self.msg("one"), self.msg("two"),
self.msg("three"), self.msg("four")])
yield self.assert_fetch_offset(
1, start_offset1, [self.msg("five"), self.msg("six"),
self.msg("seven")])
except IOError: # pragma: no cover
msg = "IOError: Probably time.sleep with negative value due to " \
"'too slow' system taking more than {0} seconds to do " \
"something that should take ~0.4 seconds.".format(batchtime)
log.exception(msg)
# In case of the IOError, we want to eat the CancelledError
send1D.addErrback(lambda _: None) # Eat any uncaught errors
send2D.addErrback(lambda _: None) # Eat any uncaught errors
self.fail(msg)
finally:
yield producer.stop()
def test_producer_batched_by_time(self):
start_offset0 = yield self.current_offset(self.topic, 0)
start_offset1 = yield self.current_offset(self.topic, 1)
# This needs to be big enough that the operations between starting the
# producer and the time.sleep() call take less time than this... I made
# it large enough that the test would still pass even with my Macbook's
# cores all pegged by a load generator.
batchtime = 5
try:
producer = Producer(self.client,
batch_send=True,
batch_every_n=0,
batch_every_t=batchtime)
startTime = time.time()
# Send 4 messages and do a fetch
send1D = producer.send_messages(self.topic,
msgs=[
self.msg("one"),
self.msg("two"),
self.msg("three"),
self.msg("four")
])
# set assert_fetch_offset() to wait for 0.1 secs on the server-side
# before returning no result. So, these calls should take 0.2sec
yield self.assert_fetch_offset(0, start_offset0, [], max_wait=0.1)
yield self.assert_fetch_offset(1, start_offset1, [], max_wait=0.1)
# Messages shouldn't have sent out yet, so we shouldn't have
# response from server yet on having received/responded to request
self.assertNoResult(send1D)
# Sending 3 more messages should NOT trigger the send, as less than
# 1 sec. elapsed by here, so send2D should still have no result.
send2D = producer.send_messages(
self.topic,
msgs=[self.msg("five"),
self.msg("six"),
self.msg("seven")])
# still no messages...
yield self.assert_fetch_offset(0, start_offset0, [], max_wait=0.1)
yield self.assert_fetch_offset(1, start_offset1, [], max_wait=0.1)
# Still no result on send, and send should NOT have gone out.
self.assertNoResult(send2D)
# Wait the timeout out. It'd be nicer to be able to just 'advance'
# the reactor, but since we need the network so...
time.sleep(batchtime - (time.time() - startTime) + 0.05)
# We need to yield to the reactor to have it process the response
# from the broker. Both send1D and send2D should then have results.
resp1 = yield send1D
resp2 = yield send2D
# ensure the 2 batches went into the proper partitions...
self.assert_produce_response(resp1, start_offset0)
self.assert_produce_response(resp2, start_offset1)
# Should be able to get messages now
yield self.assert_fetch_offset(0, start_offset0, [
self.msg("one"),
self.msg("two"),
self.msg("three"),
self.msg("four")
])
yield self.assert_fetch_offset(
1, start_offset1,
[self.msg("five"),
self.msg("six"),
self.msg("seven")])
except IOError: # pragma: no cover
msg = "IOError: Probably time.sleep with negative value due to " \
"'too slow' system taking more than {0} seconds to do " \
"something that should take ~0.4 seconds.".format(batchtime)
log.exception(msg)
# In case of the IOError, we want to eat the CancelledError
send1D.addErrback(lambda _: None) # Eat any uncaught errors
send2D.addErrback(lambda _: None) # Eat any uncaught errors
self.fail(msg)
finally:
yield producer.stop()
def test_producer_batched_by_bytes(self):
start_offset0 = yield self.current_offset(self.topic, 0)
start_offset1 = yield self.current_offset(self.topic, 1)
producer = Producer(self.client,
batch_send=True,
batch_every_b=4096,
batch_every_n=0,
batch_every_t=0)
# Send 4 messages and do a fetch. Fetch should timeout, and send
# deferred shouldn't have a result yet...
send1D = producer.send_messages(self.topic,
msgs=[
self.msg("one"),
self.msg("two"),
self.msg("three"),
self.msg("four")
])
# by default the assert_fetch_offset() waits for 0.5 secs on the server
# side before returning no result. So, these two calls should take 1sec
yield self.assert_fetch_offset(0, start_offset0, [])
yield self.assert_fetch_offset(1, start_offset1, [])
# Messages shouldn't have sent out yet, so we shouldn't have
# response from server yet on having received/responded to the request
self.assertNoResult(send1D)
# Sending 3 more messages should trigger the send, but we don't yield
# here, so we shouldn't have a response immediately after, and so
# send2D should still have no result.
send2D = producer.send_messages(
self.topic,
msgs=[self.msg("five"),
self.msg("six"),
self.msg("seven")])
# make sure no messages on server...
yield self.assert_fetch_offset(0, start_offset0, [])
yield self.assert_fetch_offset(1, start_offset1, [])
# Still no result on send
self.assertNoResult(send2D)
# send 3rd batch which will be on partition 0 again...
send3D = producer.send_messages(self.topic,
msgs=[
self.msg("eight"),
self.msg("nine"),
self.msg("ten"),
self.msg("eleven")
])
# make sure no messages on server...
yield self.assert_fetch_offset(0, start_offset0, [])
yield self.assert_fetch_offset(1, start_offset1, [])
# Still no result on send
self.assertNoResult(send3D)
# Finally, send a big message to trigger send
send4D = producer.send_messages(self.topic,
msgs=[self.msg("1234" * 1024)])
# Now do a fetch, again waiting for up to 0.5 seconds for the response
# All four messages sent in first batch (to partition 0, given default
# R-R, start-at-zero partitioner), and 4 in 3rd batch
yield self.assert_fetch_offset(0,
start_offset0, [
self.msg("one"),
self.msg("two"),
self.msg("three"),
self.msg("four"),
self.msg("eight"),
self.msg("nine"),
self.msg("ten"),
self.msg("eleven")
],
fetch_size=2048)
# Fetch from partition:1 should have all messages in 2nd batch, as
# send_messages() treats calls as groups and all/none are sent...
yield self.assert_fetch_offset(1,
start_offset1, [
self.msg("five"),
self.msg("six"),
self.msg("seven"),
self.msg("1234" * 1024)
],
fetch_size=5 * 1024)
# make sure the deferreds fired with the proper result
resp1 = self.successResultOf(send1D)
resp2 = self.successResultOf(send2D)
resp3 = self.successResultOf(send3D)
resp4 = self.successResultOf(send4D)
self.assert_produce_response(resp1, start_offset0)
self.assert_produce_response(resp2, start_offset1)
self.assert_produce_response(resp3, start_offset0)
self.assert_produce_response(resp4, start_offset1)
# cleanup
yield producer.stop()
def test_producer_batched_by_time(self):
start_offset0 = yield self.current_offset(self.topic, 0)
start_offset1 = yield self.current_offset(self.topic, 1)
# This needs to be big enough that the operations between starting the
# producer and the sleep take less time than this... I made
# it large enough that the test would still pass even with my Macbook's
# cores all pegged by a load generator.
batchtime = 5
try:
producer = Producer(self.client,
batch_send=True,
batch_every_n=0,
batch_every_t=batchtime)
startTime = time.time()
# Send 4 messages and do a fetch
send1D = producer.send_messages(self.topic,
msgs=[
self.msg("one"),
self.msg("two"),
self.msg("three"),
self.msg("four")
])
# set assert_fetch_offset() to wait for 0.1 secs on the server-side
# before returning no result. So, these calls should take 0.2sec
yield self.assert_fetch_offset(0, start_offset0, [], max_wait=0.1)
yield self.assert_fetch_offset(1, start_offset1, [], max_wait=0.1)
# Messages shouldn't have sent out yet, so we shouldn't have
# response from server yet on having received/responded to request
self.assertNoResult(send1D)
# Sending 3 more messages should NOT trigger the send, as less than
# 1 sec. elapsed by here, so send2D should still have no result.
send2D = producer.send_messages(
self.topic,
msgs=[self.msg("five"),
self.msg("six"),
self.msg("seven")])
# still no messages...
yield self.assert_fetch_offset(0, start_offset0, [], max_wait=0.1)
yield self.assert_fetch_offset(1, start_offset1, [], max_wait=0.1)
# Still no result on send, and send should NOT have gone out.
self.assertNoResult(send2D)
# Wait the timeout out. It'd be nicer to be able to just 'advance'
# the reactor, but since we need the network so...
yield async_delay(batchtime - (time.time() - startTime) + 0.05,
clock=self.reactor)
# We need to yield to the reactor to have it process the response
# from the broker. Both send1D and send2D should then have results.
resp1 = yield send1D
resp2 = yield send2D
# ensure the 2 batches went into the proper partitions...
self.assert_produce_response(resp1, start_offset0)
self.assert_produce_response(resp2, start_offset1)
# Should be able to get messages now
yield self.assert_fetch_offset(0, start_offset0, [
self.msg("one"),
self.msg("two"),
self.msg("three"),
self.msg("four")
])
yield self.assert_fetch_offset(
1, start_offset1,
[self.msg("five"),
self.msg("six"),
self.msg("seven")])
finally:
yield producer.stop()
