Python FeedConsumer.start Examples

Example #1

File: load_tests.py Project: whereXiaowei/woof

    def run(self):
        consumer = FeedConsumer(server, offset='latest', group='LoadTestGroup')
        consumer.add_topic(topic, Consumer.dummy)
        Consumer.valid = 0
        Consumer.invalid = 0
        consumer.start()

        while 1:
            if Consumer.cons_done == 1:
                print "Consumer done!"
                break
            else:
                time.sleep(5)

Example #2

File: load_tests.py Project: goibibo/woof

    def run(self):
        consumer = FeedConsumer(server, offset='latest', group ='LoadTestGroup')
        consumer.add_topic(topic, Consumer.dummy)
        Consumer.valid = 0
        Consumer.invalid = 0
        consumer.start()

        while 1 :
            if Consumer.cons_done == 1:
                print "Consumer done!"
                break
            else:
                time.sleep(5)

Example #3

File: test_partitioned_cons.py Project: cookingkode/woof

def part1(key,value):
    print "[PARTITION 1]  Value %s"%( str(value))


def part2(key,value):
    print "[PARTITION 2]  Value %s"%( str(value))



#fc =  FeedConsumer(srv, group ='TestGroup', offset= 'smallest')
fc1 =  FeedConsumer(srv, group ='TestGroup')
fc1.add_topic(topic,part0, (0,))
# Or could have given
# fc.add_topic(topic, callback_for_1_and_2, (1,2))

fc2 =  FeedConsumer(srv, group ='TestGroup')
fc2.add_topic(topic,part1, (1,))

fc3 =  FeedConsumer(srv, group ='TestGroup')
fc3.add_topic(topic,part2, (2,))

print "listening..  "



fc1.start()
fc2.start()
fc3.start()

time.sleep(120)

Example #4

File: test_cons_add_remove.py Project: whereXiaowei/woof

else :
	topic = sys.argv[1]

logging.basicConfig(
        format='%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s',
        filename='/tmp/kafkalog',
        level=logging.INFO
        )

logger = logging.getLogger('kafka')
logger.setLevel(logging.INFO)

def dummy(key,value):
	print "[x] Key %s :: Value %s"%(str(key),str(value)) 


def new_start(key,value):
	print "\txxx--  [x] Key %s :: Value %s"%(str(key),str(value)) 

print " Adding topic " + topic

fc = FeedConsumer("localhost:9092", group='testg')
fc.add_topic(topic,dummy)
fc.start()
fc.add_topic("test1",new_start)
time.sleep(40)
print " Removing topic " + topic
fc.remove_topic(topic)
time.sleep(40)

Example #5

    print "[PARTITION 0]  Value %s" % (str(value))


def part1(key, value):
    print "[PARTITION 1]  Value %s" % (str(value))


def part2(key, value):
    print "[PARTITION 2]  Value %s" % (str(value))


#fc =  FeedConsumer(srv, group ='TestGroup', offset= 'smallest')
fc1 = FeedConsumer(srv, group='TestGroup')
fc1.add_topic(topic, part0, (0, ))
# Or could have given
# fc.add_topic(topic, callback_for_1_and_2, (1,2))

fc2 = FeedConsumer(srv, group='TestGroup')
fc2.add_topic(topic, part1, (1, ))

fc3 = FeedConsumer(srv, group='TestGroup')
fc3.add_topic(topic, part2, (2, ))

print "listening..  "

fc1.start()
fc2.start()
fc3.start()

time.sleep(120)