def consume(kafka_host):
kafka = KafkaClient(kafka_host)
consumer = SimpleConsumer(kafka, 'fetcher', cfg['kafka']['pages'])
producer = SimpleProducer(kafka)
consumer.max_buffer_size=20*1024*1024
for msg in consumer:
page = json.loads(msg.message.value)
process(page, producer)
kafka.close()
from kafka import KafkaClient, SimpleConsumer
from sys import argv
MIN_NUM_ARGS = 4
if (len(argv) < MIN_NUM_ARGS):
print "Wrong number of arguments: ", len(argv)
print "Usage: python simple-consumer <broker-address> <topic-name> <consumer-group-id>"
exit(1)
kafkaClientAddress = argv[1]
topicName = argv[2]
consumerGroupId = argv[3]
kafkaClient = KafkaClient(kafkaClientAddress)
consumer = SimpleConsumer(kafkaClient, consumerGroupId, topicName)
consumer.max_buffer_size=0
consumer.seek(0,2)
for message in consumer:
print("OFFSET: "+str(message[0])+"\t MSG: "+str(message[1][3]))
def simple_consumer():
consumer = SimpleConsumer(kafka, 'group', sys.argv[1])
consumer.max_buffer_size = 0
consumer.seek(0, 2) # Most recent, not stream begin
return consumer
if args.offset:
zk.set(
'/consumers/{0}/offsets/{1}/{2}'.format(args.consumer, args.topic,
args.partition),
b"%s" % (args.offset))
try:
if args.broker:
kclient = KafkaClient("%s" % (args.broker))
# add support for more than 1 parititon
consumer = SimpleConsumer(kclient,
args.consumer,
args.topic,
partitions=[0])
consumer.max_buffer_size = None
if args.offset:
consumer.seek(0, 1)
message = consumer.get_message()
if message:
print "DEBUG: restoring"
print("MSG: " + str(message[1][3]) + "\tOFFSET: " +
str(message[0]) + "\t KEY: " + str(message.message.key))
if not args.set:
zk.set(
'/consumers/{0}/offsets/{1}/{2}'.format(
args.consumer, args.topic, args.partition),
b"%s" % (old_offset))
else:
