def reQueue(self, result):
log.debug("reQueue %s messages" % len(self.messages))
start = time.time()
tsdbqueue = QueueHandler()
tsdbqueue.messages = self.messages
tsdbqueue.put('opentsdb')
#self.messages = []
end = time.time() - start
avg = len(self.messages)/(end+.001)
log.info("reQueue %s messages for %s in %0.1f seconds at %0.1f queues/s" % (len(self._taskConfig.datapoints),self.configId, end, avg))
return defer.succeed(None)
def pullData(self):
'''
consume messages from each datapoint's queue,
format and push to opentsdb queue
'''
start = time.time()
#self.messages = []
for datapoint in self._taskConfig.datapoints:
zqueue = QueueHandler()
results = zqueue.get(datapoint['path'])
for r in results:
msg = self.tsdbFormat(datapoint['metadata'], eval(r))
self.messages.append(msg)
end = time.time() - start
avg = len(self._taskConfig.datapoints)/(end+.001)
log.info("pullData %s queues for %s in %0.1f seconds at %0.1f queues/s" % (len(self._taskConfig.datapoints),self.configId, end, avg))
def consumeDatapoints(self):
'''
push messages from opentsdb queue to the TSDB server
'''
log.debug("consumeDatapoints")
start = time.time()
try:
q = QueueHandler()
q._delete = False
messages = q.get('opentsdb')
for message in messages:
result = self._tsd.send(message)
if result != len(message):
log.warn('failed TSDB delivery for %s...requeuing' % message)
q.messages = messages
q.put("opentsdb")
end = time.time() - start
avg = len(messages)/(end+.01)
log.info("pushed %s datapoints in %0.1f seconds at %0.1f msg/s" % (len(messages),end,avg))
except:
log.warn("problem collecting data")
def putMQ(self, queue, msg):
from ZenPacks.community.OpenTSDB.QueueHandler import QueueHandler
q = QueueHandler()
q.messages = [ msg ]
try: q.put(queue)
except: pass
