def add_task(self):
dt = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
hosts = host.objects.all()
networks = device.objects.all()
t_uuid = [p.name for p in PeriodicTask.objects.all()]
i = IntervalSchedule.objects.filter(every=5, period="minutes")
if not i:
i = IntervalSchedule.objects.create(every=5, period="minutes")
interval = IntervalSchedule.objects.get(every=5, period="minutes")
if hosts:
for h in hosts:
#主机状态0, 报警状态0
if h.status == 0 and h.alarm == 0:
# 报警
if h.uuid not in t_uuid:
PeriodicTask.objects.create(name=h.uuid, task="task.curlmulti.curlmulti_tsdb", interval=interval, args="[\"%s\"]" % (h.uuid))
ports = h.port_set.filter(status=1, alarm=0)
if ports:
for p in ports:
content = "critical: %s 主机:%s 进程名:%s 端口:%s down" % (dt, h.ip, p.alias if p.alias else p.proc_name, p.port)
for g in h.group.all():
alarm(content, Group.objects.filter(name=g.name))
elif h.status == 0 and h.alarm == 1:
if h.uuid in t_uuid:
PeriodicTask.objects.get(name=h.uuid).delete()
elif h.status == 1 and h.alarm == 0:
#报警
if h.uuid in t_uuid:
PeriodicTask.objects.get(name=h.uuid).delete()
content = "critical: %s 主机:%s down" % (dt, h.ip)
#print content
for g in h.group.all():
alarm(content, Group.objects.filter(name=g.name))
elif h.status == 1 and h.alarm == 1:
if h.uuid in t_uuid:
PeriodicTask.objects.get(name=h.uuid).delete()
if networks:
for n in networks:
if n.alarm == 0:
if n.uuid not in t_uuid:
PeriodicTask.objects.create(name=n.uuid, task="task.curlmulti.network_curlmulti_tsdb", interval=interval, args="[\"%s\"]" % (n.uuid))
else:
if n.uuid in t_uuid:
PeriodicTask.objects.get(name=n.uuid).delete()
print self.request
def tsdb_ratio(h, url, url_ago, name, key, method, symbol, threshold,
floatingthreshold, floatingvalue, counter, attempt, groups,
alert):
filename = "/tmp/%s_ratio_url_%s.%s" % (h.uuid, name, key)
filename_ago = "/tmp/%s_ratio_url_ago_%s.%s" % (h.uuid, name, key)
data = pycurl_data(filename, quote(url, ':/=&()?,>.'))
data_ago = pycurl_data(filename_ago, quote(url_ago, ':/=&()?,>.'))
metric = "%s.%s" % (name, key)
if data == 0 and data_ago == 0:
dt = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
symbols = {
'>': 'gt',
'>=': 'ge',
'<': 'lt',
'<=': 'le',
'=': 'eq',
'!=': 'ne',
'<>': 'ne'
}
comparator = operator.__dict__[symbols[symbol]]
with open(filename, "rb") as f:
bfb_val = tsdb_ratio_data(f.readline(), method)
with open(filename_ago, "rb") as f:
bfb_val_ago = tsdb_ratio_data(f.readline(), method)
rv = 0
if bfb_val and bfb_val_ago:
val = ratio(bfb_val_ago, bfb_val)
if comparator(abs(val), threshold):
rv = 1
if rv == 0:
floatingvalue = 0
alert = 0
if counter > 0 and alert == 0:
counter = 0
content = "状态:ok %s 主机:%s metric:%s 阀值百分比:%s 方法:环比 %s 结果百分比:|%s|" % (
dt, h.ip, metric, bytes2human(threshold), symbol,
bytes2human(val))
print content
alarm(content, groups)
item_update(h, name, key, val, counter, floatingvalue, alert)
else:
item_update(h, name, key, val, counter, floatingvalue, alert)
elif rv == 1:
if floatingthreshold == 0:
floatingvalue = 0
content = "状态:critical %s 主机:%s metric:%s 结果百分比:|%s| 方法:环比 %s 阀值百分比:%s" % (
dt, h.ip, metric, bytes2human(val), symbol,
bytes2human(threshold))
counter += 1
if attempt == 0 and alert == 0:
alarm(content, groups)
elif counter < attempt and alert == 0:
alarm(content, groups)
item_update(h, name, key, val, counter, floatingvalue, alert)
else:
if counter == 0:
floatingvalue = val + floatingthreshold
content = "状态:critical %s 主机:%s metric:%s 结果百分比:|%s| 方法:环比 %s 阀值百分比:%s" % (
dt, h.ip, metric, bytes2human(val), symbol,
bytes2human(threshold))
print content
counter += 1
if attempt == 0 and alert == 0:
alarm(content, groups)
elif counter < attempt and alert == 0:
alarm(content, groups)
item_update(h, name, key, val, counter, floatingvalue,
alert)
elif counter > 0:
fv = 0
if comparator(val, floatingvalue):
fv = 1
if fv == 0:
while not comparator(val, floatingvalue):
floatingvalue -= floatingthreshold
print "floatingvalue:%s" % (floatingvalue)
floatingvalue += floatingthreshold
item_update(h, name, key, val, counter, floatingvalue,
alert)
elif fv == 1:
content = "状态:critical %s 主机:%s metric:%s 结果百分比:|%s| 方法:环比 %s 浮动值百分比:%s" % (
dt, h.ip, metric, bytes2human(val), symbol,
bytes2human(floatingvalue))
print content
if attempt == 0 and alert == 0:
alarm(content, groups)
elif counter < attempt and alert == 0:
alarm(content, groups)
floatingvalue = val + floatingthreshold
counter += 1
item_update(h, name, key, val, counter, floatingvalue,
alert)
def tsdb_data(data, url, h, service, item, method, symbol, threshold, floatingthreshold, floatingvalue, counter, attempt, groups):
dt = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
_data = json.loads(data)
symbols = {
'>' : 'gt',
'>=' : 'ge',
'<' : 'lt',
'<=' : 'le',
'=' : 'eq',
'!=' : 'ne',
'<>' : 'ne'
}
if len(_data) and type(_data) == list:
data_dict = _data[0]
metric = "%s.%s" % (service, item)
keys = data_dict.keys()
comparator = operator.__dict__[symbols[symbol]]
val = 0
rv = 0
if 'dps' in keys:
if len(data_dict['dps']):
points = data_dict['dps'].values()
stats = Stats(points)
if method == "max":
val = stats.max()
elif method == "min":
val = stats.min()
elif method == "avg":
val = stats.avg()
elif method == "sum":
val = stats.sum()
elif method == "count":
val = stats.count()
else:
val = stats.avg()
if comparator(val, threshold):
rv = 1
if rv == 0:
floatingvalue = 0
if counter > 0:
counter = 0
content = "状态:ok %s 主机:%s metric:%s 阀值:%s 方法:%s %s 结果:%s" % (dt, h.ip, metric, bytes2human(threshold), method, symbol, bytes2human(val))
alarm(content, groups)
item_update(h, service, item, val, counter, floatingvalue)
else:
item_update(h, service, item, val, counter, floatingvalue)
elif rv == 1:
if floatingthreshold == 0:
floatingvalue = 0
content = "状态:critical %s 主机:%s metric:%s 结果:%s 方法:%s %s 阀值:%s" % (dt, h.ip, metric, bytes2human(val), method, symbol, bytes2human(threshold))
counter += 1
if attempt == 0:
alarm(content, groups)
elif counter < attempt:
alarm(content, groups)
item_update(h, service, item, val, counter, floatingvalue)
else:
if counter == 0:
floatingvalue = val + floatingthreshold
content = "状态:critical %s 主机:%s metric:%s 结果:%s 方法:%s %s 阀值:%s" % (dt, h.ip, metric, bytes2human(val), method, symbol, bytes2human(threshold))
counter += 1
if attempt == 0:
alarm(content, groups)
elif counter < attempt:
alarm(content, groups)
item_update(h, service, item, val, counter, floatingvalue)
elif counter > 0:
fv = 0
if comparator(val, floatingvalue):
fv = 1
if fv == 0:
while not comparator(val, floatingvalue):
floatingvalue -= floatingthreshold
floatingvalue += floatingthreshold
item_update(h, service, item, val, counter, floatingvalue)
elif fv == 1:
content = "状态:critical %s 主机:%s metric:%s 结果:%s 方法:%s %s 浮动值:%s" % (dt, h.ip, metric, bytes2human(val), method, symbol, bytes2human(floatingvalue))
if attempt == 0:
alarm(content, groups)
elif counter < attempt:
alarm(content, groups)
floatingvalue = val + floatingthreshold
counter += 1
item_update(h, service, item, val, counter, floatingvalue)
def tsdb_data(data, url, h, service, item, method, symbol, threshold, floatingthreshold, floatingvalue, counter, attempt, groups, alert):
dt = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
_data = json.loads(data)
if len(_data) and type(_data) == list:
data_dict = _data[0]
metric = "%s.%s" % (service, item)
keys = data_dict.keys()
comparator = operator.__dict__[symbols[symbol]]
val = 0
rv = 0
if 'dps' in keys:
if len(data_dict['dps']):
points = data_dict['dps'].values()
stats = Stats(points)
if method == "max":
val = stats.max()
elif method == "min":
val = stats.min()
elif method == "avg":
val = stats.avg()
elif method == "sum":
val = stats.sum()
elif method == "count":
val = stats.count()
else:
val = stats.avg()
if comparator(val, threshold):
rv = 1
if rv == 0:
floatingvalue = 0
alert = 0
if counter > 0:
counter = 0
content = "状态:ok %s 主机:%s metric:%s 阀值:%s 方法:%s %s 结果:%s" % (dt, h.ip, metric, bytes2human(threshold), method, symbol, bytes2human(val))
alarm(content, groups)
item_update(h, service, item, val, counter, floatingvalue, alert)
else:
item_update(h, service, item, val, counter, floatingvalue, alert)
elif rv == 1:
if floatingthreshold == 0:
floatingvalue = 0
content = "状态:critical %s 主机:%s metric:%s 结果:%s 方法:%s %s 阀值:%s" % (dt, h.ip, metric, bytes2human(val), method, symbol, bytes2human(threshold))
counter += 1
if attempt == 0 and alert == 0:
alarm(content, groups)
elif counter < attempt and alert == 0:
alarm(content, groups)
item_update(h, service, item, val, counter, floatingvalue, alert)
else:
if counter == 0:
floatingvalue = val + floatingthreshold
content = "状态:critical %s 主机:%s metric:%s 结果:%s 方法:%s %s 阀值:%s" % (dt, h.ip, metric, bytes2human(val), method, symbol, bytes2human(threshold))
counter += 1
if attempt == 0 and alert == 0:
alarm(content, groups)
elif counter < attempt and alert == 0:
alarm(content, groups)
item_update(h, service, item, val, counter, floatingvalue, alert)
elif counter > 0:
fv = 0
if comparator(val, floatingvalue):
fv = 1
if fv == 0:
while not comparator(val, floatingvalue):
floatingvalue -= floatingthreshold
floatingvalue += floatingthreshold
item_update(h, service, item, val, counter, floatingvalue, alert)
elif fv == 1:
content = "状态:critical %s 主机:%s metric:%s 结果:%s 方法:%s %s 浮动值:%s" % (dt, h.ip, metric, bytes2human(val), method, symbol, bytes2human(floatingvalue))
if attempt == 0 and alert == 0:
alarm(content, groups)
elif counter < attempt and alert == 0:
alarm(content, groups)
floatingvalue = val + floatingthreshold
counter += 1
item_update(h, service, item, val, counter, floatingvalue, alert)
def tsdb_ratio(h, url, url_ago, name, key, method, symbol, threshold, floatingthreshold, floatingvalue, counter, attempt, groups, alert):
filename = "/tmp/%s_ratio_url_%s.%s" % (h.uuid, name, key)
filename_ago = "/tmp/%s_ratio_url_ago_%s.%s" % (h.uuid, name, key)
data = pycurl_data(filename, quote(url, ':/=&()?,>.'))
data_ago = pycurl_data(filename_ago, quote(url_ago, ':/=&()?,>.'))
metric = "%s.%s" % (name, key)
if data == 0 and data_ago == 0:
dt = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
symbols = {
'>' : 'gt',
'>=' : 'ge',
'<' : 'lt',
'<=' : 'le',
'=' : 'eq',
'!=' : 'ne',
'<>' : 'ne'
}
comparator = operator.__dict__[symbols[symbol]]
with open(filename, "rb") as f:
bfb_val = tsdb_ratio_data(f.readline(), method)
with open(filename_ago, "rb") as f:
bfb_val_ago = tsdb_ratio_data(f.readline(), method)
rv = 0
if bfb_val and bfb_val_ago:
val = ratio(bfb_val_ago, bfb_val)
if comparator(abs(val), threshold):
rv = 1
if rv == 0:
floatingvalue = 0
alert = 0
if counter > 0 and alert == 0:
counter = 0
content = "状态:ok %s 主机:%s metric:%s 阀值百分比:%s 方法:环比 %s 结果百分比:|%s|" % (dt, h.ip, metric, bytes2human(threshold), symbol, bytes2human(val))
print content
alarm(content, groups)
item_update(h, name, key, val, counter, floatingvalue, alert)
else:
item_update(h, name, key, val, counter, floatingvalue, alert)
elif rv == 1:
if floatingthreshold == 0:
floatingvalue = 0
content = "状态:critical %s 主机:%s metric:%s 结果百分比:|%s| 方法:环比 %s 阀值百分比:%s" % (dt, h.ip, metric, bytes2human(val), symbol, bytes2human(threshold))
counter += 1
if attempt == 0 and alert== 0:
alarm(content, groups)
elif counter < attempt and alert == 0:
alarm(content, groups)
item_update(h, name, key, val, counter, floatingvalue, alert)
else:
if counter == 0:
floatingvalue = val + floatingthreshold
content = "状态:critical %s 主机:%s metric:%s 结果百分比:|%s| 方法:环比 %s 阀值百分比:%s" % (dt, h.ip, metric, bytes2human(val), symbol, bytes2human(threshold))
print content
counter += 1
if attempt == 0 and alert == 0:
alarm(content, groups)
elif counter < attempt and alert == 0:
alarm(content, groups)
item_update(h, name, key, val, counter, floatingvalue, alert)
elif counter > 0:
fv = 0
if comparator(val, floatingvalue):
fv = 1
if fv == 0:
while not comparator(val, floatingvalue):
floatingvalue -= floatingthreshold
print "floatingvalue:%s" % (floatingvalue)
floatingvalue += floatingthreshold
item_update(h, name, key, val, counter, floatingvalue, alert)
elif fv == 1:
content = "状态:critical %s 主机:%s metric:%s 结果百分比:|%s| 方法:环比 %s 浮动值百分比:%s" % (dt, h.ip, metric, bytes2human(val), symbol, bytes2human(floatingvalue))
print content
if attempt == 0 and alert == 0:
alarm(content, groups)
elif counter < attempt and alert == 0:
alarm(content, groups)
floatingvalue = val + floatingthreshold
counter += 1
item_update(h, name, key, val, counter, floatingvalue, alert)
