def setUpClass(cls): if not os.environ.get('KAFKA_VERSION'): # pragma: no cover return # Single zookeeper, 3 kafka brokers zk_chroot = random_string(10) replicas = 3 partitions = 2 max_bytes = 12 * 1048576 # 12 MB cls.zk = ZookeeperFixture.instance() kk_args = [ cls.zk.host, cls.zk.port, zk_chroot, replicas, partitions, max_bytes ] cls.kafka_brokers = [ KafkaFixture.instance(i, *kk_args) for i in range(replicas) ] hosts = ['%s:%d' % (b.host, b.port) for b in cls.kafka_brokers] cls.client = KafkaClient(hosts, timeout=2500, clientId=__name__) # Startup the twisted reactor in a thread. We need this before the # the KafkaClient can work, since KafkaBrokerClient relies on the # reactor for its TCP connection cls.reactor, cls.thread = threaded_reactor()
def test_large_messages(self): # Produce 10 "normal" size messages small_messages = yield self.send_messages(0, [str(x) for x in range(10)]) # Produce 10 messages that are large (bigger than default fetch size) large_messages = yield self.send_messages( 0, [random_string(FETCH_BUFFER_SIZE_BYTES * 3) for x in range(10)]) # Consumer should still get all of them consumer = self.consumer() # Start the consumer from the beginning d = consumer.start(OFFSET_EARLIEST) # Loop waiting for all the messages to show up while len(consumer.processor._messages) < 20: # Wait a bit for them to arrive yield async_delay() expected_messages = set(small_messages + large_messages) actual_messages = set( [x.message.value for x in consumer.processor._messages]) self.assertEqual(expected_messages, actual_messages) # Clean up consumer.stop() self.successResultOf(d)
def setUpClass(cls): if not os.environ.get('KAFKA_VERSION'): # pragma: no cover return DEBUGGING = True setDebugging(DEBUGGING) DelayedCall.debug = DEBUGGING zk_chroot = random_string(10) replicas = 2 partitions = 7 # mini zookeeper, 2 kafka brokers cls.zk = ZookeeperFixture.instance() kk_args = [cls.zk.host, cls.zk.port, zk_chroot, replicas, partitions] cls.kafka_brokers = [ KafkaFixture.instance(i, *kk_args) for i in range(replicas)] hosts = ['%s:%d' % (b.host, b.port) for b in cls.kafka_brokers] # We want a short timeout on message sending for this test, since # we are expecting failures when we take down the brokers cls.client = KafkaClient(hosts, timeout=1000, clientId=__name__) # Startup the twisted reactor in a thread. We need this before the # the KafkaClient can work, since KafkaBrokerClient relies on the # reactor for its TCP connection cls.reactor, cls.thread = threaded_reactor()
def setUpClass(cls): if not os.environ.get('KAFKA_VERSION'): # pragma: no cover return DEBUGGING = True setDebugging(DEBUGGING) DelayedCall.debug = DEBUGGING zk_chroot = random_string(10) replicas = 2 partitions = 7 # mini zookeeper, 2 kafka brokers cls.zk = ZookeeperFixture.instance() kk_args = [cls.zk.host, cls.zk.port, zk_chroot, replicas, partitions] cls.kafka_brokers = [ KafkaFixture.instance(i, *kk_args) for i in range(replicas) ] hosts = ['%s:%d' % (b.host, b.port) for b in cls.kafka_brokers] # We want a short timeout on message sending for this test, since # we are expecting failures when we take down the brokers cls.client = KafkaClient(hosts, timeout=1000, clientId=__name__) # Startup the twisted reactor in a thread. We need this before the # the KafkaClient can work, since KafkaBrokerClient relies on the # reactor for its TCP connection cls.reactor, cls.thread = threaded_reactor()
def setUpClass(cls): if not os.environ.get('KAFKA_VERSION'): # pragma: no cover log.warning("WARNING: KAFKA_VERSION not found in environment") return DEBUGGING = True setDebugging(DEBUGGING) DelayedCall.debug = DEBUGGING # Single zookeeper, 3 kafka brokers zk_chroot = random_string(10) replicas = 3 partitions = 2 cls.zk = ZookeeperFixture.instance() kk_args = [cls.zk.host, cls.zk.port, zk_chroot, replicas, partitions] cls.kafka_brokers = [ KafkaFixture.instance(i, *kk_args) for i in range(replicas)] # server is used by our superclass when creating the client... cls.server = cls.kafka_brokers[0] # Startup the twisted reactor in a thread. We need this before the # the KafkaClient can work, since KafkaBrokerClient relies on the # reactor for its TCP connection cls.reactor, cls.thread = threaded_reactor()
def test_large_messages(self): # Produce 10 "normal" size messages small_messages = yield self.send_messages( 0, [str(x) for x in range(10)]) # Produce 10 messages that are large (bigger than default fetch size) large_messages = yield self.send_messages( 0, [random_string(FETCH_BUFFER_SIZE_BYTES * 3) for x in range(10)]) # Consumer should still get all of them consumer = self.consumer() # Start the consumer from the beginning d = consumer.start(OFFSET_EARLIEST) # Loop waiting for all the messages to show up while len(consumer.processor._messages) < 20: # Wait a bit for them to arrive yield async_delay() expected_messages = set(small_messages + large_messages) actual_messages = set([x.message.value for x in consumer.processor._messages]) self.assertEqual(expected_messages, actual_messages) # Clean up consumer.stop() self.successResultOf(d)
def setUpClass(cls): if not os.environ.get('KAFKA_VERSION'): # pragma: no cover log.warning("WARNING: KAFKA_VERSION not found in environment") return DEBUGGING = True setDebugging(DEBUGGING) DelayedCall.debug = DEBUGGING # Single zookeeper, 3 kafka brokers zk_chroot = random_string(10) replicas = 3 partitions = PARTITION_COUNT cls.zk = ZookeeperFixture.instance() kk_args = [cls.zk.host, cls.zk.port, zk_chroot, replicas, partitions] cls.kafka_brokers = [ KafkaFixture.instance(i, *kk_args) for i in range(replicas) ] # server is used by our superclass when creating the client... cls.server = cls.kafka_brokers[0] # Startup the twisted reactor in a thread. We need this before the # the KafkaClient can work, since KafkaBrokerClient relies on the # reactor for its TCP connection cls.reactor, cls.thread = threaded_reactor()
def _send_random_messages(self, producer, topic, n): for j in range(n): resp = yield producer.send_messages( topic, msgs=[random_string(10)]) self.assertFalse(isinstance(resp, Exception)) if resp: self.assertEqual(resp.error, 0)
def _send_random_messages(self, producer, topic, n): for j in range(n): resp = yield producer.send_messages(topic, msgs=[random_string(10)]) self.assertFalse(isinstance(resp, Exception)) if resp: self.assertEqual(resp.error, 0)
def test_huge_messages(self): # Produce 10 "normal" size messages yield self.send_messages(0, [str(x) for x in range(10)]) # Setup a max buffer size for the consumer, and put a message in # Kafka that's bigger than that MAX_FETCH_BUFFER_SIZE_BYTES = (256 * 1024) - 10 huge_message, = yield self.send_messages( 0, [random_string(MAX_FETCH_BUFFER_SIZE_BYTES + 10)]) # Create a consumer with the (smallish) max buffer size consumer = self.consumer(max_buffer_size=MAX_FETCH_BUFFER_SIZE_BYTES) # This consumer fails to get the message, and errbacks the start # deferred d = consumer.start(OFFSET_EARLIEST) # Loop waiting for the errback to be called while not d.called: # Wait a bit for them to arrive yield async_delay() # Make sure the failure is as expected self.failureResultOf(d, ConsumerFetchSizeTooSmall) # Make sure the smaller, earlier messages were delivered self.assert_message_count(consumer.processor._messages, 10) # last offset seen last_offset = consumer.processor._messages[-1].offset # Stop the consumer: d already errbacked, but stop still must be called consumer.stop() # Create a consumer with no fetch size limit big_consumer = self.consumer() # Start just past the last message processed d = big_consumer.start(last_offset + 1) # Consume giant message successfully while not big_consumer.processor._messages: # Wait a bit for it to arrive yield async_delay() self.assertEqual(big_consumer.processor._messages[0].message.value, huge_message) # Clean up big_consumer.stop() self.successResultOf(d)
def test_commit_fetch_offsets(self): """test_commit_fetch_offsets RANT: https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol implies that the metadata supplied with the commit will be returned by the fetch, but under 0.8.2.1 with a API_version of 0, it's not. Switch to using the V1 API and it works. """ # noqa resp = {} c_group = "CG_1" metadata = "My_Metadata_{}".format(random_string(10)) offset = random.randint(0, 1024) log.debug("Commiting offset: %d metadata: %s for topic: %s part: 0", offset, metadata, self.topic) req = OffsetCommitRequest(self.topic, 0, offset, -1, metadata) # We have to retry, since the client doesn't, and Kafka will # create the topic on the fly, but the first request will fail for attempt in range(20): log.debug("test_commit_fetch_offsets: Commit Attempt: %d", attempt) try: (resp, ) = yield self.client.send_offset_commit_request( c_group, [req]) except ConsumerCoordinatorNotAvailableError: log.info( "No Coordinator for Consumer Group: %s Attempt: %d of 20", c_group, attempt) time.sleep(0.5) continue except NotCoordinatorForConsumerError: # pragma: no cover # Kafka seems to have a timing issue: If we ask broker 'A' who # the ConsumerCoordinator is for a auto-created, not extant # topic, the assigned broker may not realize it's been so # designated by the time we find out and make our request. log.info( "Coordinator is not coordinator!!: %s Attempt: %d of 20", c_group, attempt) time.sleep(0.5) continue break self.assertEqual(getattr(resp, 'error', -1), 0) req = OffsetFetchRequest(self.topic, 0) (resp, ) = yield self.client.send_offset_fetch_request(c_group, [req]) self.assertEqual(resp.error, 0) self.assertEqual(resp.offset, offset) # Check we received the proper metadata in the response self.assertEqual(resp.metadata, metadata) log.debug("test_commit_fetch_offsets: Test Complete.")
def test_huge_messages(self): # Produce 10 "normal" size messages yield self.send_messages(0, [str(x) for x in range(10)]) # Setup a max buffer size for the consumer, and put a message in # Kafka that's bigger than that MAX_FETCH_BUFFER_SIZE_BYTES = (256 * 1024) - 10 huge_message, = yield self.send_messages( 0, [random_string(MAX_FETCH_BUFFER_SIZE_BYTES + 10)]) # Create a consumer with the (smallish) max buffer size consumer = self.consumer(max_buffer_size=MAX_FETCH_BUFFER_SIZE_BYTES) # This consumer fails to get the message, and errbacks the start # deferred d = consumer.start(OFFSET_EARLIEST) # Loop waiting for the errback to be called while not d.called: # Wait a bit for them to arrive yield async_delay() # Make sure the failure is as expected self.failureResultOf(d, ConsumerFetchSizeTooSmall) # Make sure the smaller, earlier messages were delivered self.assert_message_count(consumer.processor._messages, 10) # last offset seen last_offset = consumer.processor._messages[-1].offset # Stop the consumer: d already errbacked, but stop still must be called consumer.stop() # Create a consumer with no fetch size limit big_consumer = self.consumer() # Start just past the last message processed d = big_consumer.start(last_offset + 1) # Consume giant message successfully while not big_consumer.processor._messages: # Wait a bit for it to arrive yield async_delay() self.assertEqual(big_consumer.processor._messages[0].message.value, huge_message) # Clean up big_consumer.stop() self.successResultOf(d)
def setUpClass(cls): if not os.environ.get('KAFKA_VERSION'): # pragma: no cover return # Single zookeeper, 3 kafka brokers zk_chroot = random_string(10) replicas = 3 partitions = 2 cls.zk = ZookeeperFixture.instance() kk_args = [cls.zk.host, cls.zk.port, zk_chroot, replicas, partitions] cls.kafka_brokers = [ KafkaFixture.instance(i, *kk_args) for i in range(replicas)] hosts = ['%s:%d' % (b.host, b.port) for b in cls.kafka_brokers] cls.client = KafkaClient(hosts, timeout=1500, clientId=__name__) # Startup the twisted reactor in a thread. We need this before the # the KafkaClient can work, since KafkaBrokerClient relies on the # reactor for its TCP connection cls.reactor, cls.thread = threaded_reactor()
def test_write_nonextant_topic(self): """test_write_nonextant_topic Test we can write to a non-extant topic (which will be auto-created) simply by calling producer.send_messages with a long enough timeout. """ test_topics = ["{}-{}-{}".format( self.id().split('.')[-1], i, random_string(10)) for i in range(10)] producer = Producer( self.client, req_acks=PRODUCER_ACK_LOCAL_WRITE) for topic in test_topics: resp = yield producer.send_messages(topic, msgs=[self.msg(topic)]) # Make sure the send went ok self.assert_produce_response(resp, 0) # Make sure we can get the message back yield self.assert_fetch_offset( 0, 0, [self.msg(topic)], topic=topic) yield producer.stop()
def test_commit_fetch_offsets(self): """test_commit_fetch_offsets RANT: https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol implies that the metadata supplied with the commit will be returned by the fetch, but under 0.8.2.1 with a API_version of 0, it's not. Switch to using the V1 API and it works. """ # noqa resp = {} c_group = "CG_1" metadata = "My_Metadata_{}".format(random_string(10)) offset = random.randint(0, 1024) log.debug("Commiting offset: %d metadata: %s for topic: %s part: 0", offset, metadata, self.topic) req = OffsetCommitRequest(self.topic, 0, offset, -1, metadata) # We have to retry, since the client doesn't, and Kafka will # create the topic on the fly, but the first request will fail for attempt in range(20): log.debug("test_commit_fetch_offsets: Commit Attempt: %d", attempt) try: (resp,) = yield self.client.send_offset_commit_request( c_group, [req]) except ConsumerCoordinatorNotAvailableError: log.info( "No Coordinator for Consumer Group: %s Attempt: %d of 20", c_group, attempt) time.sleep(0.5) continue break self.assertEqual(getattr(resp, 'error', -1), 0) req = OffsetFetchRequest(self.topic, 0) (resp,) = yield self.client.send_offset_fetch_request(c_group, [req]) self.assertEqual(resp.error, 0) self.assertEqual(resp.offset, offset) # broker doesn't seem to return proper metadata self.assertEqual(resp.metadata, metadata) log.debug("test_commit_fetch_offsets: Test Complete.")
def test_write_nonextant_topic(self): """test_write_nonextant_topic Test we can write to a non-extant topic (which will be auto-created) simply by calling producer.send_messages with a long enough timeout. """ test_topics = [ "{}-{}-{}".format(self.id().split('.')[-1], i, random_string(10)) for i in range(10) ] producer = Producer(self.client, req_acks=PRODUCER_ACK_LOCAL_WRITE) for topic in test_topics: resp = yield producer.send_messages(topic, msgs=[self.msg(topic)]) # Make sure the send went ok self.assert_produce_response(resp, 0) # Make sure we can get the message back yield self.assert_fetch_offset(0, 0, [self.msg(topic)], topic=topic) yield producer.stop()
def test_throughput(self): yield async_delay(3) # 0.8.1.1 fails otherwise # Flag to shutdown keep_running = True # Count of messages sent sent_msgs_count = [0] total_messages_size = [0] # setup MESSAGE_BLOCK_SIZEx1024-ish byte messages to send over and over constant_messages = [ self.msg(s) for s in [random_string(1024) for x in range(MESSAGE_BLOCK_SIZE)] ] large_messages = [ self.msg(s) for s in [ random_string(FETCH_BUFFER_SIZE_BYTES * 3) for x in range(MESSAGE_BLOCK_SIZE) ] ] constant_messages_size = len(constant_messages[0]) * MESSAGE_BLOCK_SIZE large_messages_size = len(large_messages[0]) * MESSAGE_BLOCK_SIZE # Create a producer and send some messages producer = Producer(self.client) # Create consumers (1/partition) consumers = [ self.consumer(partition=p, fetch_max_wait_time=50) for p in range(PARTITION_COUNT) ] def log_error(failure): log.exception("Failure sending messages: %r", failure) # pragma: no cover def sent_msgs(resps): log.info("Messages Sent: %r", resps) sent_msgs_count[0] += MESSAGE_BLOCK_SIZE return resps def send_msgs(): # randomly, 1/20 of the time, send large messages if randint(0, 19): ## if True: messages = constant_messages large = '' total_messages_size[0] += constant_messages_size else: messages = large_messages large = ' large' total_messages_size[0] += large_messages_size log.info("Sending: %d%s messages", len(messages), large) d = producer.send_messages(self.topic, msgs=messages) # As soon as we get a response from the broker, count them # and if we're still supposed to, send more d.addCallback(sent_msgs) if keep_running: d.addCallback(lambda _: self.reactor.callLater(0, send_msgs)) ## d.addCallback(lambda _: send_msgs()) d.addErrback(log_error) # Start sending messages, MESSAGE_BLOCK_SIZE at a time, 1K or 384K each send_msgs() # Start the consumers from the beginning fetch_start = time.time() start_ds = [consumer.start(OFFSET_EARLIEST) for consumer in consumers] # Let them all run for awhile... log.info("Waiting %d seconds...", PRODUCE_TIME) yield async_delay(PRODUCE_TIME) # Tell the producer to stop keep_running = False # Wait up to PRODUCE_TIME for the consumers to catch up log.info( "Waiting up to %d seconds for " "consumers to finish consuming...", PRODUCE_TIME) deadline = time.time() + PRODUCE_TIME * 2 while time.time() < deadline: consumed = sum( [len(consumer.processor._messages) for consumer in consumers]) log.debug("Consumed %d messages.", consumed) if sent_msgs_count[0] == consumed: break yield async_delay(1) fetch_time = time.time() - fetch_start consumed_bytes = sum( [c.processor._messages_bytes[0] for c in consumers]) result_msg = ("Sent: {} messages ({:,} total bytes) in ~{} seconds" " ({}/sec), Consumed: {} in {:.2f} seconds.".format( sent_msgs_count[0], total_messages_size[0], PRODUCE_TIME, sent_msgs_count[0] / PRODUCE_TIME, consumed, fetch_time)) # Log the result, and print to stderr to get around nose capture log.info(result_msg) print("\n\t Performance Data: " + result_msg, file=sys.stderr) # And print data as stats stat('Production_Time', PRODUCE_TIME) stat('Consumption_Time', fetch_time) stat('Messages_Produced', sent_msgs_count[0]) stat('Messages_Consumed', consumed) stat('Messages_Bytes_Produced', total_messages_size[0]) stat('Messages_Bytes_Consumed', consumed_bytes) stat('Messages_Produced_Per_Second', sent_msgs_count[0] / PRODUCE_TIME) stat('Messages_Consumed_Per_Second', consumed / fetch_time) stat('Message_Bytes_Produced_Per_Second', total_messages_size[0] / PRODUCE_TIME) stat('Message_Bytes_Consumed_Per_Second', consumed_bytes / fetch_time) # Clean up log.debug('Stopping producer: %r', producer) yield producer.stop() log.debug('Stopping consumers: %r', consumers) for consumer in consumers: consumer.stop() [self.successResultOf(start_d) for start_d in start_ds] # make sure we got all the messages we sent self.assertEqual( sent_msgs_count[0], sum([len(consumer.processor._messages) for consumer in consumers]))
def test_gzip(self): for i in xrange(100): s1 = random_string(100) s2 = gzip_decode(gzip_encode(s1)) self.assertEqual(s1, s2)
def test_snappy(self): for i in xrange(100): s1 = random_string(120) s2 = snappy_decode(snappy_encode(s1)) self.assertEqual(s1, s2)
def setUp(self): super(unittest.TestCase, self).setUp() if not self.topic: self.topic = "%s-%s" % (self.id()[self.id().rindex(".") + 1:], random_string(10))
def setUp(self): super(unittest.TestCase, self).setUp() if not self.topic: self.topic = "%s-%s" % ( self.id()[self.id().rindex(".") + 1:], random_string(10))