def get_offsets(offsets_after_time_millis, conn_params=config.DEFAULT_CONN_PARAMS): curr_time = long(time.time() * 1000) for host in config.bagheera_nodes: for topic in config.topics: for partition in config.partitions: consumer = SimpleConsumer(host, conn_params['port'], conn_params['nrecs'], conn_params['bufsize']) offset = long( consumer.getOffsetsBefore(topic, partition, offsets_after_time_millis, 1)[0]) consumer.close() System.out.println( json.dumps({ 'time_millis': curr_time, 'hostname': host, 'topic': topic, 'partition': partition, 'offset': offset }))
def process_messages_forever(self): consumer = SimpleConsumer(self.hostname, self.conn_params['port'], self.conn_params['nrecs'], self.conn_params['bufsize']) if self.offset is None: offset = long(consumer.getOffsetsBefore(self.topic, self.partition, long(time.time() * 1000), 3)[0]) #TODO else: offset = self.offset nrecs = 0 while True: req = FetchRequest(self.topic, self.partition, offset, 1024 * 1024 * 64) messageset = consumer.fetch(req) for msg in messageset.elements(): self.processor(msg) offset = msg.offset() nrecs += 1 if nrecs >= self.offset_update_freq: self.update_offset(offset) nrecs = 0