def get_offsets(offsets_after_time_millis,
conn_params=config.DEFAULT_CONN_PARAMS):
curr_time = long(time.time() * 1000)
for host in config.bagheera_nodes:
for topic in config.topics:
for partition in config.partitions:
consumer = SimpleConsumer(host, conn_params['port'],
conn_params['nrecs'],
conn_params['bufsize'])
offset = long(
consumer.getOffsetsBefore(topic, partition,
offsets_after_time_millis, 1)[0])
consumer.close()
System.out.println(
json.dumps({
'time_millis': curr_time,
'hostname': host,
'topic': topic,
'partition': partition,
'offset': offset
}))
def process_messages_forever(self):
consumer = SimpleConsumer(self.hostname, self.conn_params['port'],
self.conn_params['nrecs'], self.conn_params['bufsize'])
if self.offset is None:
offset = long(consumer.getOffsetsBefore(self.topic,
self.partition,
long(time.time() * 1000), 3)[0]) #TODO
else:
offset = self.offset
nrecs = 0
while True:
req = FetchRequest(self.topic, self.partition, offset, 1024 * 1024 * 64)
messageset = consumer.fetch(req)
for msg in messageset.elements():
self.processor(msg)
offset = msg.offset()
nrecs += 1
if nrecs >= self.offset_update_freq:
self.update_offset(offset)
nrecs = 0
