def test_produce_no_autocreate(self): with mocket.mocket() as m: # We're going to be responding as follows: # 1) I have a topic 'test' (metadata request) # 2) What's a 'foo'? [kafka autocreates, client retries] # 2') What's a 'foo'? [kafka autocreates, client retries] # 2'') What's a 'foo'? [kafka autocreates, client retries] # 3) client aborts. topic_name = add_one_topic_test_response(m) add_no_such_topic_foo_response(m) add_no_such_topic_foo_response(m) add_no_such_topic_foo_response(m) client = KafkaClient(KAFKA_HOST) # connect is async, so give it a chance to run. yield client.start() self.assertTrue(client._ready) # there should be a single topic self.assert_topics(client, [topic_name]) try: yield client.send_message("foo", "test") except: pass else: assert False, "Should have raised a KafkaError" # that shouldn't have broken the client self.assertTrue(client._ready) # There should now be self.assert_topics(client, [topic_name])
def test_multiproduce_bad_unicode(self): with mocket.mocket() as m: topic_name = add_one_topic_test_response(m) unicode_topic = u"touché" client = KafkaClient(KAFKA_HOST) # connect is async, so give it a chance to run. yield client.start() self.assertTrue(client._ready) # there should be a single topic self.assert_topics(client, [topic_name]) try: yield client.send_message(unicode_topic, "test") except TopicError: pass else: assert False, "Should have raised on non-ascii topic" # the broken message shouldn't affect subsequent messages yield client.send_message(topic_name, "test") # that shouldn't have broken the client self.assertTrue(client._ready) self.assert_topics(client, [topic_name]) # also we should have blacklisted this topic name self.assertTrue(client.bad_topic_names, set([unicode_topic]))
def live_fire_test(self): with mocket.mocket(socket.AF_INET, socket.SOCK_STREAM, 0) as m: client = KafkaClient(KAFKA_HOST) yield client.start() yield client.send_message("foo", "bar") print client.topic_to_partitions import pprint pprint.pprint(m.get_transcript()) assert False
def test_no_topics(self): with mocket.mocket() as m: # This says "oh hai, I don't have any topics" add_no_topics_response(m) client = KafkaClient(KAFKA_HOST) # connect is async, so give it a chance to run. yield client.start() self.assertTrue(client._ready) self.assertEqual(client.topic_to_partitions, {})
def test_topics(self): with mocket.mocket() as m: # This says "I have one topic named 'test'" with a # replication-factor" of 1 and 1 partition topic_name = add_one_topic_test_response(m) client = KafkaClient(KAFKA_HOST) # connect is async, so give it a chance to run. yield client.start() self.assertTrue(client._ready) # there should be a single topic self.assert_topics(client, [topic_name])
def test_no_topics_multi_start(self): with mocket.mocket() as m: # This says "oh hai, I don't have any topics" add_no_topics_response(m) client = KafkaClient(KAFKA_HOST) # start should be able to be run as many times as # we want for _ in range(3): yield client.start() yield gen.sleep(.5) self.assertTrue(client._ready) self.assertEqual(client.topic_to_partitions, {})
def test_topics_unicode(self): with mocket.mocket() as m: # This says "I have one topic named 'test'" with a # replication-factor" of 1 and 1 partition topic_name = add_one_topic_test_response(m) # Kafkaclient uses struct, and struct expects strings client = KafkaClient(KAFKA_HOST, topic_names=[unicode(topic_name)], timeout=1) # connect is async, so give it a chance to run. yield client.start() self.assertTrue(client._ready) # there should be a single topic self.assert_topics(client, [topic_name])
def test_topics_unicode_fail(self): with mocket.mocket() as m: # This says "I have one topic named 'test'" with a # replication-factor" of 1 and 1 partition topic_name = add_one_topic_test_response(m) # Kafkaclient uses struct, and struct expects strings try: client = KafkaClient(KAFKA_HOST, topic_names=[u"touché"], timeout=1) yield client.start() except TopicError: pass else: assert False, "Should have raised on non-ascii topic" self.assertFalse(client._ready)
def test_produce(self): with mocket.mocket() as m: topic_name = add_one_topic_test_response(m) client = KafkaClient(KAFKA_HOST) # connect is async, so give it a chance to run. yield client.start() self.assertTrue(client._ready) # there should be a single topic self.assert_topics(client, [topic_name]) yield client.send_message(topic_name, topic_name) # that shouldn't have broken the client self.assertTrue(client._ready) self.assert_topics(client, [topic_name])
def test_topics_unicode_fail(self): with mocket.mocket() as m: # This says "I have one topic named 'test'" with a # replication-factor" of 1 and 1 partition topic_name = add_one_topic_test_response(m) # Kafkaclient uses struct, and struct expects strings try: client = KafkaClient( KAFKA_HOST, topic_names=[u"touché"], timeout=1 ) yield client.start() except TopicError: pass else: assert False, "Should have raised on non-ascii topic" self.assertFalse(client._ready)
def test_topics_unicode(self): with mocket.mocket() as m: # This says "I have one topic named 'test'" with a # replication-factor" of 1 and 1 partition topic_name = add_one_topic_test_response(m) # Kafkaclient uses struct, and struct expects strings client = KafkaClient( KAFKA_HOST, topic_names=[unicode(topic_name)], timeout=1 ) # connect is async, so give it a chance to run. yield client.start() self.assertTrue(client._ready) # there should be a single topic self.assert_topics(client, [topic_name])
def test_produce_autocreate(self): with mocket.mocket() as m: # We're going to be responding as follows: # 1) I have a topic 'test' (metadata request) # 2) What's a 'foo'? [kafka autocreates, client retries] # 3) Oh right! 'foo'! Here you go [success] topic_name = add_one_topic_test_response(m) add_no_such_topic_foo_response(m) new_topic = add_one_topic_foo_response(m) client = KafkaClient(KAFKA_HOST) # connect is async, so give it a chance to run. yield client.start() self.assertTrue(client._ready) # there should be a single topic self.assert_topics(client, [topic_name]) yield client.send_message(new_topic, topic_name) # that shouldn't have broken the client self.assertTrue(client._ready) # There should now be 2 topics self.assert_topics(client, [topic_name, new_topic])
# coding: utf8 from kafkaka.tornado_patch import KafkaClient import tornado.ioloop import time if __name__ == "__main__": c = KafkaClient("t-storm1:9092", topic_names=['im-msg']) start = time.time() print '' for i in xrange(500): c.send_message('im-msg', u'你好'.encode('utf8'), str(time.time()), str(i)) c.send_message('im-msg', 'hi', str(time.time()), str(i)) for i in xrange(500): c.send_message('im-msg', u'你好'.encode('utf8'), str(time.time()), str(i)) c.send_message('im-msg', 'hi', str(time.time()), str(i)) print time.time() - start print 'this will not block' tornado.ioloop.IOLoop.instance().start()
def test_connection_fail(self): with mocket.mocket() as m: client = KafkaClient(KAFKA_HOST) # connect is async, so give it a chance to run. yield client.start()