def run(self): consumer = FeedConsumer(server, offset='latest', group ='LoadTestGroup') consumer.add_topic(topic, Consumer.dummy) Consumer.valid = 0 Consumer.invalid = 0 consumer.start() while 1 : if Consumer.cons_done == 1: print "Consumer done!" break else: time.sleep(5)
def run(self): consumer = FeedConsumer(server, offset='latest', group='LoadTestGroup') consumer.add_topic(topic, Consumer.dummy) Consumer.valid = 0 Consumer.invalid = 0 consumer.start() while 1: if Consumer.cons_done == 1: print "Consumer done!" break else: time.sleep(5)
def part0(key,value): print "[PARTITION 0] Value %s"%( str(value)) def part1(key,value): print "[PARTITION 1] Value %s"%( str(value)) def part2(key,value): print "[PARTITION 2] Value %s"%( str(value)) #fc = FeedConsumer(srv, group ='TestGroup', offset= 'smallest') fc1 = FeedConsumer(srv, group ='TestGroup') fc1.add_topic(topic,part0, (0,)) # Or could have given # fc.add_topic(topic, callback_for_1_and_2, (1,2)) fc2 = FeedConsumer(srv, group ='TestGroup') fc2.add_topic(topic,part1, (1,)) fc3 = FeedConsumer(srv, group ='TestGroup') fc3.add_topic(topic,part2, (2,)) print "listening.. " fc1.start() fc2.start()
else : topic = sys.argv[1] logging.basicConfig( format='%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s', filename='/tmp/kafkalog', level=logging.INFO ) logger = logging.getLogger('kafka') logger.setLevel(logging.INFO) def dummy(key,value): print "[x] Key %s :: Value %s"%(str(key),str(value)) def new_start(key,value): print "\txxx-- [x] Key %s :: Value %s"%(str(key),str(value)) print " Adding topic " + topic fc = FeedConsumer("localhost:9092", group='testg') fc.add_topic(topic,dummy) fc.start() fc.add_topic("test1",new_start) time.sleep(40) print " Removing topic " + topic fc.remove_topic(topic) time.sleep(40)
def dummy(key,value): print "[%s] Value %s"%(str(time.time()), str(value)) def print_start(key,value): print "[x] FROM START %s"%(str(value)) def testxx_start(key,value): print "[x]FROM NEWER TOPIC | %s"%(str(value)) #fc = FeedConsumer(srv) fc = FeedConsumer(srv, offset= 'smallest', group ='TestGroup') print "listing to topic %s"%(topic) fc.add_topic(topic,dummy) fc.start() time.sleep(15) """ fc_from_start = FeedConsumer(srv,offset= 'smallest') fc_from_start.add_topic("test_start",print_start) fc_from_start.start() time.sleep(30) print "listing to topic text xx" fc_from_start.add_topic("testxx",testxx_start) """
filename='/tmp/wooflog', level=logging.INFO) logger = logging.getLogger('kafka') logger.setLevel(logging.INFO) srv = os.getenv("GOMSG_SRV", "localhost:9092") def dummy(key, value): print "[%s] Value %s" % (str(time.time()), str(value)) def print_start(key, value): print "[x] FROM START %s" % (str(value)) def testxx_start(key, value): print "[x]FROM NEWER TOPIC | %s" % (str(value)) # to continue to use offsets from zk #fc = FeedConsumer(srv, offset= 'latest', group ='TestGroup',use_zk=True) fc = FeedConsumer(srv, offset='latest', group='TestGroup') print "listing to topic %s" % (topic) fc.add_topic(topic, dummy) fc.start() time.sleep(15) time.sleep(60)
def part0(key, value): print "[PARTITION 0] Value %s" % (str(value)) def part1(key, value): print "[PARTITION 1] Value %s" % (str(value)) def part2(key, value): print "[PARTITION 2] Value %s" % (str(value)) #fc = FeedConsumer(srv, group ='TestGroup', offset= 'smallest') fc1 = FeedConsumer(srv, group='TestGroup') fc1.add_topic(topic, part0, (0, )) # Or could have given # fc.add_topic(topic, callback_for_1_and_2, (1,2)) fc2 = FeedConsumer(srv, group='TestGroup') fc2.add_topic(topic, part1, (1, )) fc3 = FeedConsumer(srv, group='TestGroup') fc3.add_topic(topic, part2, (2, )) print "listening.. " fc1.start() fc2.start() fc3.start()