def collect():
#创建子进程执行sflowtool命令
sp = subprocess.Popen(SFLOWTOOL, stdout=subprocess.PIPE)
while True:
#循环逐行读取标准输出
line = sp.stdout.readline()
symbol = line.split()[0]
if symbol == 'startDatagram':
header = {}
samples = []
i = 0
elif symbol == 'endDatagram':
for item in samples:
tmp = header.copy()
tmp.update(item)
db['origin'].insert(tmp)
print tmp['unixSecondsUTC']
else:
if symbol == 'startSample':
samples.append({'status':0})
elif symbol == 'endSample':
i += 1
else:
[key,value] = line.rstrip('\r\n').replace(' ','\n',1).split('\n')[:2]
if METRICS.has_key(key):
if METRICS[key]['group'] == 'common':
header[key] = data_convert(key, value)
else:
samples[i][key] = data_convert(key, value)
def process():
db_origin = db['origin']
db_host = db['host']
key = {'UUID': 1}
now = int(time.time())
condition = {'unixSecondsUTC': {"$gte": now - 10, "$lt": now}}
initial = {'unixSecondsUTC': 0}
reduce = 'function (doc, prev) {'
reduce += 'if (doc.unixSecondsUTC > prev.unixSecondsUTC) {'
reduce += 'for (var key in doc) {'
reduce += 'prev[key] = doc[key]; }}}'
documents_1 = db_origin.group(key, condition, initial, reduce)
initial = {'unixSecondsUTC': float('inf')}
reduce = reduce.replace('>', '<')
documents_2 = db_origin.group(key, condition, initial, reduce)
for i in range(len(documents_1)):
if documents_1[i]['unixSecondsUTC'] == documents_2[i][
'unixSecondsUTC']:
continue
documents_1[i] = cpu_util(documents_1[i], documents_2[i])
del documents_1[i]['status']
interval = documents_1[i]['unixSecondsUTC'] - documents_2[i][
'unixSecondsUTC']
hostdoc = {
'host': {},
'cpu': {},
'memory': {},
'disk': {},
'network': {}
}
for key in documents_1[i].keys():
if METRICS.has_key(key):
if METRICS[key]['dst'] == 'COUNTER':
documents_1[i][key] = round(
(documents_1[i][key] - documents_2[i][key]) / interval,
2)
if documents_1[i][key] < 0:
documents_1[i][key] = 0
if METRICS[key]['group'] != 'common':
if not documents_1[i].has_key(METRICS[key]['group']):
documents_1[i][METRICS[key]['group']] = {}
hostdoc[METRICS[key]['group']][key] = documents_1[i][key]
documents_1[i][METRICS[key]
['group']][key] = documents_1[i][key]
del documents_1[i][key]
else:
hostdoc[key] = documents_1[i][key]
db_origin.update({'_id': documents_1[i]['_id']},
{'$set': {
'status': 1
}})
db_host.update({'_id': documents_1[i]['host']['UUID']},
{'$set': hostdoc},
upsert=True)
db['day_5s'].insert(documents_1[i])
# if documents_1:
# db['day_5s'].insert(documents_1)
# print documents_1[0]['_id']
db_origin.remove({'unixSecondsUTC': {"$lt": now - 3 * 24 * 60 * 60}})
db['day_5s'].remove({'unixSecondsUTC': {"$lt": now - 30 * 24 * 60 * 60}})
def process():
db_origin = db['origin']
db_host = db['host']
key = {'UUID':1}
now = int(time.time())
condition = {'unixSecondsUTC': {"$gte": now-10, "$lt": now}}
initial = {'unixSecondsUTC': 0}
reduce = 'function (doc, prev) {'
reduce += 'if (doc.unixSecondsUTC > prev.unixSecondsUTC) {'
reduce += 'for (var key in doc) {'
reduce += 'prev[key] = doc[key]; }}}'
documents_1 = db_origin.group(key, condition, initial, reduce)
initial = {'unixSecondsUTC': float('inf')}
reduce = reduce.replace('>', '<')
documents_2 = db_origin.group(key, condition, initial, reduce)
for i in range(len(documents_1)):
if documents_1[i]['unixSecondsUTC'] == documents_2[i]['unixSecondsUTC']:
continue
documents_1[i] = cpu_util(documents_1[i], documents_2[i])
del documents_1[i]['status']
interval = documents_1[i]['unixSecondsUTC'] - documents_2[i]['unixSecondsUTC']
hostdoc = {'host':{},'cpu':{},'memory':{},'disk':{},'network':{}}
for key in documents_1[i].keys():
if METRICS.has_key(key):
if METRICS[key]['dst'] == 'COUNTER':
documents_1[i][key] = round((documents_1[i][key] - documents_2[i][key])/interval, 2)
if documents_1[i][key] < 0:
documents_1[i][key] = 0
if METRICS[key]['group'] != 'common':
if not documents_1[i].has_key(METRICS[key]['group']):
documents_1[i][METRICS[key]['group']] = {}
hostdoc[METRICS[key]['group']][key] = documents_1[i][key]
documents_1[i][METRICS[key]['group']][key] = documents_1[i][key]
del documents_1[i][key]
else:
hostdoc[key] = documents_1[i][key]
db_origin.update({'_id':documents_1[i]['_id']},{'$set':{'status':1}})
db_host.update({'_id':documents_1[i]['host']['UUID']},{'$set':hostdoc},upsert=True)
db['day_5s'].insert(documents_1[i])
# if documents_1:
# db['day_5s'].insert(documents_1)
# print documents_1[0]['_id']
db_origin.remove({'unixSecondsUTC': {"$lt": now-3*24*60*60}})
db['day_5s'].remove({'unixSecondsUTC': {"$lt": now-30*24*60*60}})