def __init__(self, data_dir, sink_conf):
self.data_dir = data_dir
self.running = False
self.tfcache = TimedFdCache(self.WAIT_TIMEOUT)
self.sinks = CollectSinks(sink_conf)
self.queue = Queue()
class CollectQueue(object):
LOG = logging.getLogger('collector-queue')
WAIT_TIMEOUT = 1
def __init__(self, data_dir, sink_conf):
self.data_dir = data_dir
self.running = False
self.tfcache = TimedFdCache(self.WAIT_TIMEOUT)
self.sinks = CollectSinks(sink_conf)
self.queue = Queue()
def run(self):
self.running = True
while self.running:
self._process_data(self.WAIT_TIMEOUT)
while not self.queue.empty():
self._process_data(1)
self.tfcache.close()
def stop(self):
self.running = False
print 'Queue Killed', self.queue.qsize()
def put(self, data, data_dir=None):
self.queue.put(data)
def _process_data(self, timeout):
try:
key, timestamp, value = self.queue.get(True, timeout)
except:
self.tfcache.flush()
else:
self._store_data(key, timestamp, value)
self._sink_store_data(key, timestamp, value)
@debug_time
def _store_data(self, key, timestamp, value):
try:
fd = self.tfcache.open(key, tsfile.Writer, self.data_dir)
fd.write('%s %s\n' % (timestamp, value))
except Exception, e:
self.LOG.warn('WAL failure: %s' % e)
