def pullData(self): ''' consume messages from each datapoint's queue, format and push to opentsdb queue ''' start = time.time() #self.messages = [] for datapoint in self._taskConfig.datapoints: zqueue = QueueHandler() results = zqueue.get(datapoint['path']) for r in results: msg = self.tsdbFormat(datapoint['metadata'], eval(r)) self.messages.append(msg) end = time.time() - start avg = len(self._taskConfig.datapoints)/(end+.001) log.info("pullData %s queues for %s in %0.1f seconds at %0.1f queues/s" % (len(self._taskConfig.datapoints),self.configId, end, avg))
def consumeDatapoints(self): ''' push messages from opentsdb queue to the TSDB server ''' log.debug("consumeDatapoints") start = time.time() try: q = QueueHandler() q._delete = False messages = q.get('opentsdb') for message in messages: result = self._tsd.send(message) if result != len(message): log.warn('failed TSDB delivery for %s...requeuing' % message) q.messages = messages q.put("opentsdb") end = time.time() - start avg = len(messages)/(end+.01) log.info("pushed %s datapoints in %0.1f seconds at %0.1f msg/s" % (len(messages),end,avg)) except: log.warn("problem collecting data")