def pullData(self):
'''
consume messages from each datapoint's queue,
format and push to opentsdb queue
'''
start = time.time()
#self.messages = []
for datapoint in self._taskConfig.datapoints:
zqueue = QueueHandler()
results = zqueue.get(datapoint['path'])
for r in results:
msg = self.tsdbFormat(datapoint['metadata'], eval(r))
self.messages.append(msg)
end = time.time() - start
avg = len(self._taskConfig.datapoints)/(end+.001)
log.info("pullData %s queues for %s in %0.1f seconds at %0.1f queues/s" % (len(self._taskConfig.datapoints),self.configId, end, avg))
def consumeDatapoints(self):
'''
push messages from opentsdb queue to the TSDB server
'''
log.debug("consumeDatapoints")
start = time.time()
try:
q = QueueHandler()
q._delete = False
messages = q.get('opentsdb')
for message in messages:
result = self._tsd.send(message)
if result != len(message):
log.warn('failed TSDB delivery for %s...requeuing' % message)
q.messages = messages
q.put("opentsdb")
end = time.time() - start
avg = len(messages)/(end+.01)
log.info("pushed %s datapoints in %0.1f seconds at %0.1f msg/s" % (len(messages),end,avg))
except:
log.warn("problem collecting data")
