def run(self):
consumer = FeedConsumer(server, offset='latest', group ='LoadTestGroup')
consumer.add_topic(topic, Consumer.dummy)
Consumer.valid = 0
Consumer.invalid = 0
consumer.start()
while 1 :
if Consumer.cons_done == 1:
print "Consumer done!"
break
else:
time.sleep(5)
def run(self):
consumer = FeedConsumer(server, offset='latest', group='LoadTestGroup')
consumer.add_topic(topic, Consumer.dummy)
Consumer.valid = 0
Consumer.invalid = 0
consumer.start()
while 1:
if Consumer.cons_done == 1:
print "Consumer done!"
break
else:
time.sleep(5)
srv = os.getenv("GOMSG_SRV","localhost:9092")
def part0(key,value):
print "[PARTITION 0] Value %s"%( str(value))
def part1(key,value):
print "[PARTITION 1] Value %s"%( str(value))
def part2(key,value):
print "[PARTITION 2] Value %s"%( str(value))
#fc = FeedConsumer(srv, group ='TestGroup', offset= 'smallest')
fc1 = FeedConsumer(srv, group ='TestGroup')
fc1.add_topic(topic,part0, (0,))
# Or could have given
# fc.add_topic(topic, callback_for_1_and_2, (1,2))
fc2 = FeedConsumer(srv, group ='TestGroup')
fc2.add_topic(topic,part1, (1,))
fc3 = FeedConsumer(srv, group ='TestGroup')
fc3.add_topic(topic,part2, (2,))
print "listening.. "
fc1.start()
else :
topic = sys.argv[1]
logging.basicConfig(
format='%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s',
filename='/tmp/kafkalog',
level=logging.INFO
)
logger = logging.getLogger('kafka')
logger.setLevel(logging.INFO)
def dummy(key,value):
print "[x] Key %s :: Value %s"%(str(key),str(value))
def new_start(key,value):
print "\txxx-- [x] Key %s :: Value %s"%(str(key),str(value))
print " Adding topic " + topic
fc = FeedConsumer("localhost:9092", group='testg')
fc.add_topic(topic,dummy)
fc.start()
fc.add_topic("test1",new_start)
time.sleep(40)
print " Removing topic " + topic
fc.remove_topic(topic)
time.sleep(40)
srv = os.getenv("GOMSG_SRV","localhost:9092")
def dummy(key,value):
print "[%s] Value %s"%(str(time.time()), str(value))
def print_start(key,value):
print "[x] FROM START %s"%(str(value))
def testxx_start(key,value):
print "[x]FROM NEWER TOPIC | %s"%(str(value))
#fc = FeedConsumer(srv)
fc = FeedConsumer(srv, offset= 'smallest', group ='TestGroup')
print "listing to topic %s"%(topic)
fc.add_topic(topic,dummy)
fc.start()
time.sleep(15)
"""
fc_from_start = FeedConsumer(srv,offset= 'smallest')
fc_from_start.add_topic("test_start",print_start)
fc_from_start.start()
time.sleep(30)
print "listing to topic text xx"
fc_from_start.add_topic("testxx",testxx_start)
"""
filename='/tmp/wooflog',
level=logging.INFO)
logger = logging.getLogger('kafka')
logger.setLevel(logging.INFO)
srv = os.getenv("GOMSG_SRV", "localhost:9092")
def dummy(key, value):
print "[%s] Value %s" % (str(time.time()), str(value))
def print_start(key, value):
print "[x] FROM START %s" % (str(value))
def testxx_start(key, value):
print "[x]FROM NEWER TOPIC | %s" % (str(value))
# to continue to use offsets from zk
#fc = FeedConsumer(srv, offset= 'latest', group ='TestGroup',use_zk=True)
fc = FeedConsumer(srv, offset='latest', group='TestGroup')
print "listing to topic %s" % (topic)
fc.add_topic(topic, dummy)
fc.start()
time.sleep(15)
time.sleep(60)
def part0(key, value):
print "[PARTITION 0] Value %s" % (str(value))
def part1(key, value):
print "[PARTITION 1] Value %s" % (str(value))
def part2(key, value):
print "[PARTITION 2] Value %s" % (str(value))
#fc = FeedConsumer(srv, group ='TestGroup', offset= 'smallest')
fc1 = FeedConsumer(srv, group='TestGroup')
fc1.add_topic(topic, part0, (0, ))
# Or could have given
# fc.add_topic(topic, callback_for_1_and_2, (1,2))
fc2 = FeedConsumer(srv, group='TestGroup')
fc2.add_topic(topic, part1, (1, ))
fc3 = FeedConsumer(srv, group='TestGroup')
fc3.add_topic(topic, part2, (2, ))
print "listening.. "
fc1.start()
fc2.start()
fc3.start()
