class InsertTSDB(object): def __init__(self, **conf): self.alertr = RedisClient(db='alert', **conf['database']) self.stamp = dict() def CheckData(self): while True: self.save = list() batch = dict() start = time() print start keys = self.alertr.keys('result:collector:system*') for key in keys: tmp = key.split(':') tmp = tmp[2:] name = '.'.join(tmp) if key not in self.stamp: self.stamp[key] = dict() self.stamp[key]['timestamp'] = None data = json.loads(self.alertr.get(key)) stamp = data.pop('timestamp') if not self.stamp[key]['timestamp'] or self.stamp[key]['timestamp'] != stamp: for host,value in data.items(): if isinstance(value,dict): for k,v in value.items(): self.save.append( 'put %s %s %s host=%s key=%s' % (name,stamp,v,host,k)) else: self.save.append( 'put %s %s %s host=%s' % (name,stamp,value,host)) self.stamp[key]['timestamp'] = stamp MAX = (len(self.save) / 1000 + 1) print "max:%s" % MAX for i,value in enumerate(self.save): mod = i % MAX if mod not in batch: batch[mod] = list() batch[mod].append(value) for i in batch: f = open('/tmp/ok','w') f.write('\n'.join(batch[i])) f.close() p1 = subprocess.Popen(['cat', '/tmp/ok'], stdout=subprocess.PIPE) p2 = subprocess.Popen(['nc','10.0.29.54', '4242'], stdin=p1.stdout, stdout=subprocess.PIPE) p1.stdout.close() sleep(0.1) p1.join() p2.join() spend = 60 - time() + start print time() if spend:sleep(spend)
def __init__(self, **conf): self.alertr = RedisClient(db='alert', **conf['database']) self.stamp = dict()