def test_producer_send_messages(self): start_offset0 = yield self.current_offset(self.topic, 0) start_offset1 = yield self.current_offset(self.topic, 1) producer = Producer(self.client) # Goes to first partition resp = yield producer.send_messages( self.topic, msgs=[self.msg("one"), self.msg("two")]) self.assert_produce_response(resp, start_offset0) # Goes to the 2nd partition resp = yield producer.send_messages(self.topic, msgs=[self.msg("three")]) self.assert_produce_response(resp, start_offset1) # fetch the messages back and make sure they are as expected yield self.assert_fetch_offset( 0, start_offset0, [self.msg("one"), self.msg("two")]) yield self.assert_fetch_offset( 1, start_offset1, [self.msg("three")]) # Goes back to the first partition because there's only two partitions resp = yield producer.send_messages( self.topic, msgs=[self.msg("four"), self.msg("five")]) self.assert_produce_response(resp, start_offset0+2) yield self.assert_fetch_offset( 0, start_offset0, [self.msg("one"), self.msg("two"), self.msg("four"), self.msg("five")]) yield producer.stop()
def test_switch_leader(self): producer = Producer(self.client) topic = self.topic try: for index in range(1, 3): # cause the client to establish connections to all the brokers log.debug("Pass: %d. Sending 10 random messages", index) yield self._send_random_messages(producer, topic, 10) # Ensure that the follower is in sync log.debug("Ensuring topic/partition is replicated.") part_meta = self.client.partition_meta[TopicAndPartition( self.topic, 0)] # Ensure the all the replicas are in-sync before proceeding while len(part_meta.isr) != 2: # pragma: no cover log.debug("Waiting for Kafka replica to become synced") if len(part_meta.replicas) != 2: log.error( "Kafka replica 'disappeared'!" "Partitition Meta: %r", part_meta) yield async_delay(1.0) yield self.client.load_metadata_for_topics(self.topic) part_meta = self.client.partition_meta[TopicAndPartition( self.topic, 0)] # kill leader for partition 0 log.debug("Killing leader of partition 0") broker, kill_time = self._kill_leader(topic, 0) log.debug("Sending 1 more message: 'part 1'") yield producer.send_messages(topic, msgs=['part 1']) log.debug("Sending 1 more message: 'part 2'") yield producer.send_messages(topic, msgs=['part 2']) # send to new leader log.debug("Sending 10 more messages") yield self._send_random_messages(producer, topic, 10) # Make sure the ZK ephemeral time (~6 seconds) has elapsed wait_time = (kill_time + 6.5) - time.time() if wait_time > 0: log.debug("Waiting: %4.2f for ZK timeout", wait_time) yield async_delay(wait_time) # restart the kafka broker log.debug("Restarting the broker") broker.restart() # count number of messages log.debug("Getting message count") count = yield self._count_messages(topic) self.assertEqual(count, 22 * index) finally: log.debug("Stopping the producer") yield producer.stop() log.debug("Producer stopped") log.debug("Test complete.")
def test_switch_leader(self): producer = Producer(self.client) topic = self.topic try: for index in range(1, 3): # cause the client to establish connections to all the brokers log.debug("Pass: %d. Sending 10 random messages", index) yield self._send_random_messages(producer, topic, 10) # Ensure that the follower is in sync log.debug("Ensuring topic/partition is replicated.") part_meta = self.client.partition_meta[TopicAndPartition( self.topic, 0)] # Ensure the all the replicas are in-sync before proceeding while len(part_meta.isr) != 2: # pragma: no cover log.debug("Waiting for Kafka replica to become synced") if len(part_meta.replicas) != 2: log.error("Kafka replica 'disappeared'!" "Partitition Meta: %r", part_meta) yield async_delay(1.0) yield self.client.load_metadata_for_topics(self.topic) part_meta = self.client.partition_meta[TopicAndPartition( self.topic, 0)] # kill leader for partition 0 log.debug("Killing leader of partition 0") broker, kill_time = self._kill_leader(topic, 0) log.debug("Sending 1 more message: 'part 1'") yield producer.send_messages(topic, msgs=['part 1']) log.debug("Sending 1 more message: 'part 2'") yield producer.send_messages(topic, msgs=['part 2']) # send to new leader log.debug("Sending 10 more messages") yield self._send_random_messages(producer, topic, 10) # Make sure the ZK ephemeral time (~6 seconds) has elapsed wait_time = (kill_time + 6.5) - time.time() if wait_time > 0: log.debug("Waiting: %4.2f for ZK timeout", wait_time) yield async_delay(wait_time) # restart the kafka broker log.debug("Restarting the broker") broker.restart() # count number of messages log.debug("Getting message count") count = yield self._count_messages(topic) self.assertEqual(count, 22 * index) finally: log.debug("Stopping the producer") yield producer.stop() log.debug("Producer stopped") log.debug("Test complete.")
def test_acks_none(self): start_offset0 = yield self.current_offset(self.topic, 0) yield self.current_offset(self.topic, 1) producer = Producer(self.client, req_acks=PRODUCER_ACK_NOT_REQUIRED) resp = yield producer.send_messages(self.topic, msgs=[self.msg("one")]) self.assertEqual(resp, None) yield self.assert_fetch_offset(0, start_offset0, [self.msg("one")]) yield producer.stop()
def test_producer_batched_by_time(self): start_offset0 = yield self.current_offset(self.topic, 0) start_offset1 = yield self.current_offset(self.topic, 1) # This needs to be big enough that the operations between starting the # producer and the sleep take less time than this... I made # it large enough that the test would still pass even with my Macbook's # cores all pegged by a load generator. batchtime = 5 try: producer = Producer(self.client, batch_send=True, batch_every_n=0, batch_every_t=batchtime) startTime = time.time() # Send 4 messages and do a fetch send1D = producer.send_messages( self.topic, msgs=[self.msg("one"), self.msg("two"), self.msg("three"), self.msg("four")]) # set assert_fetch_offset() to wait for 0.1 secs on the server-side # before returning no result. So, these calls should take 0.2sec yield self.assert_fetch_offset(0, start_offset0, [], max_wait=0.1) yield self.assert_fetch_offset(1, start_offset1, [], max_wait=0.1) # Messages shouldn't have sent out yet, so we shouldn't have # response from server yet on having received/responded to request self.assertNoResult(send1D) # Sending 3 more messages should NOT trigger the send, as less than # 1 sec. elapsed by here, so send2D should still have no result. send2D = producer.send_messages( self.topic, msgs=[self.msg("five"), self.msg("six"), self.msg("seven")]) # still no messages... yield self.assert_fetch_offset(0, start_offset0, [], max_wait=0.1) yield self.assert_fetch_offset(1, start_offset1, [], max_wait=0.1) # Still no result on send, and send should NOT have gone out. self.assertNoResult(send2D) # Wait the timeout out. It'd be nicer to be able to just 'advance' # the reactor, but since we need the network so... yield async_delay(batchtime - (time.time() - startTime) + 0.05, clock=self.reactor) # We need to yield to the reactor to have it process the response # from the broker. Both send1D and send2D should then have results. resp1 = yield send1D resp2 = yield send2D # ensure the 2 batches went into the proper partitions... self.assert_produce_response(resp1, start_offset0) self.assert_produce_response(resp2, start_offset1) # Should be able to get messages now yield self.assert_fetch_offset( 0, start_offset0, [self.msg("one"), self.msg("two"), self.msg("three"), self.msg("four")]) yield self.assert_fetch_offset( 1, start_offset1, [self.msg("five"), self.msg("six"), self.msg("seven")]) finally: yield producer.stop()
def test_acks_none(self): start_offset0 = yield self.current_offset(self.topic, 0) yield self.current_offset(self.topic, 1) producer = Producer( self.client, req_acks=PRODUCER_ACK_NOT_REQUIRED) resp = yield producer.send_messages(self.topic, msgs=[self.msg("one")]) self.assertEqual(resp, None) yield self.assert_fetch_offset(0, start_offset0, [self.msg("one")]) yield producer.stop()
def test_acks_local_write(self): start_offset0 = yield self.current_offset(self.topic, 0) yield self.current_offset(self.topic, 1) producer = Producer(self.client, req_acks=PRODUCER_ACK_LOCAL_WRITE) resp = yield producer.send_messages(self.topic, msgs=[self.msg("one")]) self.assert_produce_response(resp, start_offset0) yield self.assert_fetch_offset(0, start_offset0, [self.msg("one")]) yield producer.stop()
def test_producer_batched_by_messages(self): start_offset0 = yield self.current_offset(self.topic, 0) start_offset1 = yield self.current_offset(self.topic, 1) producer = Producer(self.client, batch_send=True, batch_every_n=10, batch_every_b=0, batch_every_t=0) # Send 4 messages and do a fetch. Fetch should timeout, and send # deferred shouldn't have a result yet... send1D = producer.send_messages( self.topic, msgs=[self.msg("one"), self.msg("two"), self.msg("three"), self.msg("four")]) # by default the assert_fetch_offset() waits for 0.5 secs on the server # side before returning no result. So, these two calls should take 1sec yield self.assert_fetch_offset(0, start_offset0, []) yield self.assert_fetch_offset(1, start_offset1, []) # Messages shouldn't have sent out yet, so we shouldn't have # response from server yet on having received/responded to the request self.assertNoResult(send1D) # Sending 3 more messages won't trigger the send either (7 total) send2D = producer.send_messages( self.topic, msgs=[self.msg("five"), self.msg("six"), self.msg("seven")]) # make sure no messages on server... yield self.assert_fetch_offset(0, start_offset0, []) yield self.assert_fetch_offset(1, start_offset1, []) # Still no result on send self.assertNoResult(send2D) # send final batch which will be on partition 0 again... send3D = producer.send_messages( self.topic, msgs=[self.msg("eight"), self.msg("nine"), self.msg("ten"), self.msg("eleven")]) # Now do a fetch, again waiting for up to 0.5 seconds for the response # All four messages sent in first batch (to partition 0, given default # R-R, start-at-zero partitioner), and 4 in 3rd batch yield self.assert_fetch_offset( 0, start_offset0, [self.msg("one"), self.msg("two"), self.msg("three"), self.msg("four"), self.msg("eight"), self.msg("nine"), self.msg("ten"), self.msg("eleven")], fetch_size=2048) # Fetch from partition:1 should have all messages in 2nd batch yield self.assert_fetch_offset( 1, start_offset1, [self.msg("five"), self.msg("six"), self.msg("seven")]) # make sure the deferreds fired with the proper result resp1 = self.successResultOf(send1D) resp2 = self.successResultOf(send2D) resp3 = self.successResultOf(send3D) self.assert_produce_response(resp1, start_offset0) self.assert_produce_response(resp2, start_offset1) self.assert_produce_response(resp3, start_offset0) # cleanup yield producer.stop()
def test_acks_local_write(self): start_offset0 = yield self.current_offset(self.topic, 0) yield self.current_offset(self.topic, 1) producer = Producer( self.client, req_acks=PRODUCER_ACK_LOCAL_WRITE) resp = yield producer.send_messages(self.topic, msgs=[self.msg("one")]) self.assert_produce_response(resp, start_offset0) yield self.assert_fetch_offset(0, start_offset0, [self.msg("one")]) yield producer.stop()
def test_acks_all_replicas(self): start_offset0 = yield self.current_offset(self.topic, 0) start_offset1 = yield self.current_offset(self.topic, 1) producer = Producer(self.client, req_acks=PRODUCER_ACK_ALL_REPLICAS) resp = yield producer.send_messages(self.topic, msgs=[self.msg("one")]) self.assert_produce_response(resp, start_offset0) yield self.assert_fetch_offset(0, start_offset0, [self.msg("one")]) yield self.assert_fetch_offset(1, start_offset1, []) yield producer.stop()
def test_switch_leader(self): """ Produce messages while killing the coordinator broker. Note that in order to avoid loss of acknowledged writes the producer must request acks of -1 (`afkak.common.PRODUCER_ACK_ALL_REPLICAS`). """ producer = Producer( self.client, req_acks=PRODUCER_ACK_ALL_REPLICAS, max_req_attempts=100, ) topic = self.topic try: for index in range(1, 3): # cause the client to establish connections to all the brokers log.debug("Pass: %d. Sending 10 random messages", index) yield self._send_random_messages(producer, topic, 10) # kill leader for partition 0 log.debug("Killing leader of partition 0") broker, kill_time = self._kill_leader(topic, 0) log.debug("Sending 1 more message: 'part 1'") yield producer.send_messages(topic, msgs=[b'part 1']) log.debug("Sending 1 more message: 'part 2'") yield producer.send_messages(topic, msgs=[b'part 2']) # send to new leader log.debug("Sending 10 more messages") yield self._send_random_messages(producer, topic, 10) # Make sure the ZK ephemeral time (~6 seconds) has elapsed wait_time = (kill_time + 6.5) - time.time() if wait_time > 0: log.debug("Waiting: %4.2f for ZK timeout", wait_time) yield async_delay(wait_time) # restart the kafka broker log.debug("Restarting leader broker %r", broker) broker.restart() # count number of messages log.debug("Getting message count") count = yield self._count_messages(topic) self.assertGreaterEqual(count, 22 * index) finally: log.debug("Stopping the producer") yield producer.stop() log.debug("Producer stopped") log.debug("Test complete.")
def test_producer_hashed_partitioner(self): start_offset0 = yield self.current_offset(self.topic, 0) start_offset1 = yield self.current_offset(self.topic, 1) producer = Producer(self.client, partitioner_class=HashedPartitioner) resp1 = yield producer.send_messages(self.topic, '1', [self.msg("one")]) resp2 = yield producer.send_messages(self.topic, '2', [self.msg("two")]) resp3 = yield producer.send_messages(self.topic, '1', [self.msg("three")]) resp4 = yield producer.send_messages(self.topic, '1', [self.msg("four")]) resp5 = yield producer.send_messages(self.topic, '2', [self.msg("five")]) self.assert_produce_response(resp2, start_offset0 + 0) self.assert_produce_response(resp5, start_offset0 + 1) self.assert_produce_response(resp1, start_offset1 + 0) self.assert_produce_response(resp3, start_offset1 + 1) self.assert_produce_response(resp4, start_offset1 + 2) yield self.assert_fetch_offset(0, start_offset0, [ self.msg("two"), self.msg("five"), ]) yield self.assert_fetch_offset(1, start_offset1, [ self.msg("one"), self.msg("three"), self.msg("four"), ]) yield producer.stop()
def test_acks_all_replicas(self): start_offset0 = yield self.current_offset(self.topic, 0) start_offset1 = yield self.current_offset(self.topic, 1) producer = Producer( self.client, req_acks=PRODUCER_ACK_ALL_REPLICAS) resp = yield producer.send_messages(self.topic, msgs=[self.msg("one")]) self.assert_produce_response(resp, start_offset0) yield self.assert_fetch_offset(0, start_offset0, [self.msg("one")]) yield self.assert_fetch_offset(1, start_offset1, []) yield producer.stop()
def test_producer_batched_gzipped_hashed_partitioner(self): start_offset0 = yield self.current_offset(self.topic, 0) start_offset1 = yield self.current_offset(self.topic, 1) offsets = (start_offset0, start_offset1) requests = [] msgs_by_partition = ([], []) keys_by_partition = ([], []) partitioner = HashedPartitioner(self.topic, [0, 1]) producer = Producer(self.client, codec=CODEC_GZIP, batch_send=True, batch_every_n=100, batch_every_t=None, partitioner_class=HashedPartitioner) # Send ten groups of messages, each with a different key for i in range(10): msg_group = [] key = 'Key: {}'.format(i) part = partitioner.partition(key, [0, 1]) for j in range(10): msg = self.msg('Group:{} Msg:{}'.format(i, j)) msg_group.append(msg) msgs_by_partition[part].append(msg) keys_by_partition[part].append(key) request = producer.send_messages(self.topic, key=key, msgs=msg_group) requests.append(request) yield async_delay(.5) # Make the NoResult test have teeth... if i < 9: # This is to ensure we really are batching all the requests self.assertNoResult(request) # Now ensure we can retrieve the right messages from each partition for part in [0, 1]: yield self.assert_fetch_offset(part, offsets[part], msgs_by_partition[part], keys_by_partition[part], fetch_size=20480) yield producer.stop()
def test_producer_round_robin_partitioner(self): start_offset0 = yield self.current_offset(self.topic, 0) start_offset1 = yield self.current_offset(self.topic, 1) producer = Producer(self.client, partitioner_class=RoundRobinPartitioner) resp1 = yield producer.send_messages(self.topic, "key1", [self.msg("one")]) resp2 = yield producer.send_messages(self.topic, "key2", [self.msg("two")]) resp3 = yield producer.send_messages(self.topic, "key3", [self.msg("three")]) resp4 = yield producer.send_messages(self.topic, "key4", [self.msg("four")]) self.assert_produce_response(resp1, start_offset0 + 0) self.assert_produce_response(resp2, start_offset1 + 0) self.assert_produce_response(resp3, start_offset0 + 1) self.assert_produce_response(resp4, start_offset1 + 1) yield self.assert_fetch_offset( 0, start_offset0, [self.msg("one"), self.msg("three")]) yield self.assert_fetch_offset( 1, start_offset1, [self.msg("two"), self.msg("four")]) yield producer.stop()
def test_write_nonextant_topic(self): """ Test we can write to a non-extant topic (which will be auto-created) simply by calling producer.send_messages with a long enough timeout. """ test_topics = ["{}-{}-{}".format( self.id().split('.')[-1], i, random_string(10)) for i in range(10)] producer = Producer(self.client, req_acks=PRODUCER_ACK_LOCAL_WRITE) for topic in test_topics: resp = yield producer.send_messages(topic, msgs=[self.msg(topic)]) # Make sure the send went ok self.assert_produce_response(resp, 0) # Make sure we can get the message back yield self.assert_fetch_offset( 0, 0, [self.msg(topic)], topic=topic) yield producer.stop()
def test_producer_send_messages(self): start_offset0 = yield self.current_offset(self.topic, 0) start_offset1 = yield self.current_offset(self.topic, 1) producer = Producer(self.client) # Goes to first partition resp = yield producer.send_messages( self.topic, msgs=[self.msg("one"), self.msg("two")]) self.assert_produce_response(resp, start_offset0) # Goes to the 2nd partition resp = yield producer.send_messages(self.topic, msgs=[self.msg("three")]) self.assert_produce_response(resp, start_offset1) # fetch the messages back and make sure they are as expected yield self.assert_fetch_offset( 0, start_offset0, [self.msg("one"), self.msg("two")]) yield self.assert_fetch_offset(1, start_offset1, [self.msg("three")]) # Goes back to the first partition because there's only two partitions resp = yield producer.send_messages( self.topic, msgs=[self.msg("four"), self.msg("five")]) self.assert_produce_response(resp, start_offset0 + 2) yield self.assert_fetch_offset(0, start_offset0, [ self.msg("one"), self.msg("two"), self.msg("four"), self.msg("five") ]) yield producer.stop()
def test_producer_round_robin_partitioner(self): start_offset0 = yield self.current_offset(self.topic, 0) start_offset1 = yield self.current_offset(self.topic, 1) producer = Producer(self.client, partitioner_class=RoundRobinPartitioner) resp1 = yield producer.send_messages( self.topic, b"key1", [self.msg("one")]) resp2 = yield producer.send_messages( self.topic, b"key2", [self.msg("two")]) resp3 = yield producer.send_messages( self.topic, b"key3", [self.msg("three")]) resp4 = yield producer.send_messages( self.topic, b"key4", [self.msg("four")]) self.assert_produce_response(resp1, start_offset0+0) self.assert_produce_response(resp2, start_offset1+0) self.assert_produce_response(resp3, start_offset0+1) self.assert_produce_response(resp4, start_offset1+1) yield self.assert_fetch_offset( 0, start_offset0, [self.msg("one"), self.msg("three")]) yield self.assert_fetch_offset( 1, start_offset1, [self.msg("two"), self.msg("four")]) yield producer.stop()
def test_producer_round_robin_partitioner_random_start(self): try: RoundRobinPartitioner.set_random_start(True) producer = Producer(self.client, partitioner_class=RoundRobinPartitioner) # Two partitions, so 1st and 3rd reqs should go to same part, but # 2nd should go to different. Other than that, without statistical # test, we can't say much... Partitioner tests should ensure that # we really aren't always starting on a non-random partition... resp1 = yield producer.send_messages( self.topic, msgs=[self.msg("one"), self.msg("two")]) resp2 = yield producer.send_messages( self.topic, msgs=[self.msg("three")]) resp3 = yield producer.send_messages( self.topic, msgs=[self.msg("four"), self.msg("five")]) self.assertEqual(resp1.partition, resp3.partition) self.assertNotEqual(resp1.partition, resp2.partition) yield producer.stop() finally: RoundRobinPartitioner.set_random_start(False)
def test_producer_batched_gzipped_hashed_partitioner(self): start_offset0 = yield self.current_offset(self.topic, 0) start_offset1 = yield self.current_offset(self.topic, 1) offsets = (start_offset0, start_offset1) requests = [] msgs_by_partition = ([], []) keys_by_partition = ([], []) partitioner = HashedPartitioner(self.topic, [0, 1]) producer = Producer( self.client, codec=CODEC_GZIP, batch_send=True, batch_every_n=100, batch_every_t=None, partitioner_class=HashedPartitioner) # Send ten groups of messages, each with a different key for i in range(10): msg_group = [] key = 'Key: {}'.format(i).encode() part = partitioner.partition(key, [0, 1]) for j in range(10): msg = self.msg('Group:{} Msg:{}'.format(i, j)) msg_group.append(msg) msgs_by_partition[part].append(msg) keys_by_partition[part].append(key) request = producer.send_messages( self.topic, key=key, msgs=msg_group) requests.append(request) yield async_delay(.5) # Make the NoResult test have teeth... if i < 9: # This is to ensure we really are batching all the requests self.assertNoResult(request) # Now ensure we can retrieve the right messages from each partition for part in [0, 1]: yield self.assert_fetch_offset( part, offsets[part], msgs_by_partition[part], keys_by_partition[part], fetch_size=20480) yield producer.stop()
def test_producer_hashed_partitioner(self): start_offset0 = yield self.current_offset(self.topic, 0) start_offset1 = yield self.current_offset(self.topic, 1) producer = Producer(self.client, partitioner_class=HashedPartitioner) resp1 = yield producer.send_messages( self.topic, b'1', [self.msg("one")]) resp2 = yield producer.send_messages( self.topic, b'2', [self.msg("two")]) resp3 = yield producer.send_messages( self.topic, b'1', [self.msg("three")]) resp4 = yield producer.send_messages( self.topic, b'1', [self.msg("four")]) resp5 = yield producer.send_messages( self.topic, b'2', [self.msg("five")]) self.assert_produce_response(resp2, start_offset0+0) self.assert_produce_response(resp5, start_offset0+1) self.assert_produce_response(resp1, start_offset1+0) self.assert_produce_response(resp3, start_offset1+1) self.assert_produce_response(resp4, start_offset1+2) yield self.assert_fetch_offset( 0, start_offset0, [ self.msg("two"), self.msg("five"), ]) yield self.assert_fetch_offset( 1, start_offset1, [ self.msg("one"), self.msg("three"), self.msg("four"), ]) yield producer.stop()
def test_producer_batched_by_time(self): start_offset0 = yield self.current_offset(self.topic, 0) start_offset1 = yield self.current_offset(self.topic, 1) # This needs to be big enough that the operations between starting the # producer and the time.sleep() call take less time than this... I made # it large enough that the test would still pass even with my Macbook's # cores all pegged by a load generator. batchtime = 5 try: producer = Producer(self.client, batch_send=True, batch_every_n=0, batch_every_t=batchtime) startTime = time.time() # Send 4 messages and do a fetch send1D = producer.send_messages(self.topic, msgs=[ self.msg("one"), self.msg("two"), self.msg("three"), self.msg("four") ]) # set assert_fetch_offset() to wait for 0.1 secs on the server-side # before returning no result. So, these calls should take 0.2sec yield self.assert_fetch_offset(0, start_offset0, [], max_wait=0.1) yield self.assert_fetch_offset(1, start_offset1, [], max_wait=0.1) # Messages shouldn't have sent out yet, so we shouldn't have # response from server yet on having received/responded to request self.assertNoResult(send1D) # Sending 3 more messages should NOT trigger the send, as less than # 1 sec. elapsed by here, so send2D should still have no result. send2D = producer.send_messages( self.topic, msgs=[self.msg("five"), self.msg("six"), self.msg("seven")]) # still no messages... yield self.assert_fetch_offset(0, start_offset0, [], max_wait=0.1) yield self.assert_fetch_offset(1, start_offset1, [], max_wait=0.1) # Still no result on send, and send should NOT have gone out. self.assertNoResult(send2D) # Wait the timeout out. It'd be nicer to be able to just 'advance' # the reactor, but since we need the network so... time.sleep(batchtime - (time.time() - startTime) + 0.05) # We need to yield to the reactor to have it process the response # from the broker. Both send1D and send2D should then have results. resp1 = yield send1D resp2 = yield send2D # ensure the 2 batches went into the proper partitions... self.assert_produce_response(resp1, start_offset0) self.assert_produce_response(resp2, start_offset1) # Should be able to get messages now yield self.assert_fetch_offset(0, start_offset0, [ self.msg("one"), self.msg("two"), self.msg("three"), self.msg("four") ]) yield self.assert_fetch_offset( 1, start_offset1, [self.msg("five"), self.msg("six"), self.msg("seven")]) except IOError: # pragma: no cover msg = "IOError: Probably time.sleep with negative value due to " \ "'too slow' system taking more than {0} seconds to do " \ "something that should take ~0.4 seconds.".format(batchtime) log.exception(msg) # In case of the IOError, we want to eat the CancelledError send1D.addErrback(lambda _: None) # Eat any uncaught errors send2D.addErrback(lambda _: None) # Eat any uncaught errors self.fail(msg) finally: yield producer.stop()
def test_throughput(self): yield async_delay(3) # 0.8.1.1 fails otherwise # Flag to shutdown keep_running = True # Count of messages sent sent_msgs_count = [0] total_messages_size = [0] # setup MESSAGE_BLOCK_SIZEx1024-ish byte messages to send over and over constant_messages = [ self.msg(s) for s in [random_string(1024) for x in range(MESSAGE_BLOCK_SIZE)] ] large_messages = [ self.msg(s) for s in [ random_string(FETCH_BUFFER_SIZE_BYTES * 3) for x in range(MESSAGE_BLOCK_SIZE) ] ] constant_messages_size = len(constant_messages[0]) * MESSAGE_BLOCK_SIZE large_messages_size = len(large_messages[0]) * MESSAGE_BLOCK_SIZE # Create a producer and send some messages producer = Producer(self.client) # Create consumers (1/partition) consumers = [ self.consumer(partition=p, fetch_max_wait_time=50) for p in range(PARTITION_COUNT) ] def log_error(failure): log.exception("Failure sending messages: %r", failure) # pragma: no cover def sent_msgs(resps): log.info("Messages Sent: %r", resps) sent_msgs_count[0] += MESSAGE_BLOCK_SIZE return resps def send_msgs(): # randomly, 1/20 of the time, send large messages if randint(0, 19): ## if True: messages = constant_messages large = '' total_messages_size[0] += constant_messages_size else: messages = large_messages large = ' large' total_messages_size[0] += large_messages_size log.info("Sending: %d%s messages", len(messages), large) d = producer.send_messages(self.topic, msgs=messages) # As soon as we get a response from the broker, count them # and if we're still supposed to, send more d.addCallback(sent_msgs) if keep_running: d.addCallback(lambda _: self.reactor.callLater(0, send_msgs)) ## d.addCallback(lambda _: send_msgs()) d.addErrback(log_error) # Start sending messages, MESSAGE_BLOCK_SIZE at a time, 1K or 384K each send_msgs() # Start the consumers from the beginning fetch_start = time.time() start_ds = [consumer.start(OFFSET_EARLIEST) for consumer in consumers] # Let them all run for awhile... log.info("Waiting %d seconds...", PRODUCE_TIME) yield async_delay(PRODUCE_TIME) # Tell the producer to stop keep_running = False # Wait up to PRODUCE_TIME for the consumers to catch up log.info( "Waiting up to %d seconds for " "consumers to finish consuming...", PRODUCE_TIME) deadline = time.time() + PRODUCE_TIME * 2 while time.time() < deadline: consumed = sum( [len(consumer.processor._messages) for consumer in consumers]) log.debug("Consumed %d messages.", consumed) if sent_msgs_count[0] == consumed: break yield async_delay(1) fetch_time = time.time() - fetch_start consumed_bytes = sum( [c.processor._messages_bytes[0] for c in consumers]) result_msg = ("Sent: {} messages ({:,} total bytes) in ~{} seconds" " ({}/sec), Consumed: {} in {:.2f} seconds.".format( sent_msgs_count[0], total_messages_size[0], PRODUCE_TIME, sent_msgs_count[0] / PRODUCE_TIME, consumed, fetch_time)) # Log the result, and print to stderr to get around nose capture log.info(result_msg) print("\n\t Performance Data: " + result_msg, file=sys.stderr) # And print data as stats stat('Production_Time', PRODUCE_TIME) stat('Consumption_Time', fetch_time) stat('Messages_Produced', sent_msgs_count[0]) stat('Messages_Consumed', consumed) stat('Messages_Bytes_Produced', total_messages_size[0]) stat('Messages_Bytes_Consumed', consumed_bytes) stat('Messages_Produced_Per_Second', sent_msgs_count[0] / PRODUCE_TIME) stat('Messages_Consumed_Per_Second', consumed / fetch_time) stat('Message_Bytes_Produced_Per_Second', total_messages_size[0] / PRODUCE_TIME) stat('Message_Bytes_Consumed_Per_Second', consumed_bytes / fetch_time) # Clean up log.debug('Stopping producer: %r', producer) yield producer.stop() log.debug('Stopping consumers: %r', consumers) for consumer in consumers: consumer.stop() [self.successResultOf(start_d) for start_d in start_ds] # make sure we got all the messages we sent self.assertEqual( sent_msgs_count[0], sum([len(consumer.processor._messages) for consumer in consumers]))
def test_producer_batched_by_time(self): start_offset0 = yield self.current_offset(self.topic, 0) start_offset1 = yield self.current_offset(self.topic, 1) # This needs to be big enough that the operations between starting the # producer and the time.sleep() call take less time than this... I made # it large enough that the test would still pass even with my Macbook's # cores all pegged by a load generator. batchtime = 5 try: producer = Producer(self.client, batch_send=True, batch_every_n=0, batch_every_t=batchtime) startTime = time.time() # Send 4 messages and do a fetch send1D = producer.send_messages( self.topic, msgs=[self.msg("one"), self.msg("two"), self.msg("three"), self.msg("four")]) # set assert_fetch_offset() to wait for 0.1 secs on the server-side # before returning no result. So, these calls should take 0.2sec yield self.assert_fetch_offset(0, start_offset0, [], max_wait=0.1) yield self.assert_fetch_offset(1, start_offset1, [], max_wait=0.1) # Messages shouldn't have sent out yet, so we shouldn't have # response from server yet on having received/responded to request self.assertNoResult(send1D) # Sending 3 more messages should NOT trigger the send, as less than # 1 sec. elapsed by here, so send2D should still have no result. send2D = producer.send_messages( self.topic, msgs=[self.msg("five"), self.msg("six"), self.msg("seven")]) # still no messages... yield self.assert_fetch_offset(0, start_offset0, [], max_wait=0.1) yield self.assert_fetch_offset(1, start_offset1, [], max_wait=0.1) # Still no result on send, and send should NOT have gone out. self.assertNoResult(send2D) # Wait the timeout out. It'd be nicer to be able to just 'advance' # the reactor, but since we need the network so... time.sleep(batchtime - (time.time() - startTime) + 0.05) # We need to yield to the reactor to have it process the response # from the broker. Both send1D and send2D should then have results. resp1 = yield send1D resp2 = yield send2D # ensure the 2 batches went into the proper partitions... self.assert_produce_response(resp1, start_offset0) self.assert_produce_response(resp2, start_offset1) # Should be able to get messages now yield self.assert_fetch_offset( 0, start_offset0, [self.msg("one"), self.msg("two"), self.msg("three"), self.msg("four")]) yield self.assert_fetch_offset( 1, start_offset1, [self.msg("five"), self.msg("six"), self.msg("seven")]) except IOError: # pragma: no cover msg = "IOError: Probably time.sleep with negative value due to " \ "'too slow' system taking more than {0} seconds to do " \ "something that should take ~0.4 seconds.".format(batchtime) log.exception(msg) # In case of the IOError, we want to eat the CancelledError send1D.addErrback(lambda _: None) # Eat any uncaught errors send2D.addErrback(lambda _: None) # Eat any uncaught errors self.fail(msg) finally: yield producer.stop()
def test_producer_batched_by_time(self): start_offset0 = yield self.current_offset(self.topic, 0) start_offset1 = yield self.current_offset(self.topic, 1) # This needs to be big enough that the operations between starting the # producer and the sleep take less time than this... I made # it large enough that the test would still pass even with my Macbook's # cores all pegged by a load generator. batchtime = 5 try: producer = Producer(self.client, batch_send=True, batch_every_n=0, batch_every_t=batchtime) startTime = time.time() # Send 4 messages and do a fetch send1D = producer.send_messages(self.topic, msgs=[ self.msg("one"), self.msg("two"), self.msg("three"), self.msg("four") ]) # set assert_fetch_offset() to wait for 0.1 secs on the server-side # before returning no result. So, these calls should take 0.2sec yield self.assert_fetch_offset(0, start_offset0, [], max_wait=0.1) yield self.assert_fetch_offset(1, start_offset1, [], max_wait=0.1) # Messages shouldn't have sent out yet, so we shouldn't have # response from server yet on having received/responded to request self.assertNoResult(send1D) # Sending 3 more messages should NOT trigger the send, as less than # 1 sec. elapsed by here, so send2D should still have no result. send2D = producer.send_messages( self.topic, msgs=[self.msg("five"), self.msg("six"), self.msg("seven")]) # still no messages... yield self.assert_fetch_offset(0, start_offset0, [], max_wait=0.1) yield self.assert_fetch_offset(1, start_offset1, [], max_wait=0.1) # Still no result on send, and send should NOT have gone out. self.assertNoResult(send2D) # Wait the timeout out. It'd be nicer to be able to just 'advance' # the reactor, but since we need the network so... yield async_delay(batchtime - (time.time() - startTime) + 0.05, clock=self.reactor) # We need to yield to the reactor to have it process the response # from the broker. Both send1D and send2D should then have results. resp1 = yield send1D resp2 = yield send2D # ensure the 2 batches went into the proper partitions... self.assert_produce_response(resp1, start_offset0) self.assert_produce_response(resp2, start_offset1) # Should be able to get messages now yield self.assert_fetch_offset(0, start_offset0, [ self.msg("one"), self.msg("two"), self.msg("three"), self.msg("four") ]) yield self.assert_fetch_offset( 1, start_offset1, [self.msg("five"), self.msg("six"), self.msg("seven")]) finally: yield producer.stop()