def __init__(self, data_dir, sink_conf): self.data_dir = data_dir self.running = False self.tfcache = TimedFdCache(self.WAIT_TIMEOUT) self.sinks = CollectSinks(sink_conf) self.queue = Queue()
class CollectQueue(object): LOG = logging.getLogger('collector-queue') WAIT_TIMEOUT = 1 def __init__(self, data_dir, sink_conf): self.data_dir = data_dir self.running = False self.tfcache = TimedFdCache(self.WAIT_TIMEOUT) self.sinks = CollectSinks(sink_conf) self.queue = Queue() def run(self): self.running = True while self.running: self._process_data(self.WAIT_TIMEOUT) while not self.queue.empty(): self._process_data(1) self.tfcache.close() def stop(self): self.running = False print 'Queue Killed', self.queue.qsize() def put(self, data, data_dir=None): self.queue.put(data) def _process_data(self, timeout): try: key, timestamp, value = self.queue.get(True, timeout) except: self.tfcache.flush() else: self._store_data(key, timestamp, value) self._sink_store_data(key, timestamp, value) @debug_time def _store_data(self, key, timestamp, value): try: fd = self.tfcache.open(key, tsfile.Writer, self.data_dir) fd.write('%s %s\n' % (timestamp, value)) except Exception, e: self.LOG.warn('WAL failure: %s' % e)