def check_result_submit(request):
launcher = Redis_Queue(Webmoni_Send_Mail_Queue)
if request.method == 'POST':
submitData = json.loads(request.POST.get('submitData'))
client_ip = request.META['REMOTE_ADDR']
node_obj = API_verify(client_ip)
if node_obj is not None:
try:
MonitorData.objects.create(
http_code=submitData.get('http_code'),
total_time=submitData.get('total_time'),
datetime=submitData.get('datetime'),
node_id=submitData.get('node'),
url_id=submitData.get('url_id'))
if not submitData.get('status') == 100:
Event_Log.objects.create(
datetime=submitData.get('datetime'),
event_type_id=submitData.get('status'),
node_id=submitData.get('node'),
url_id=submitData.get('url_id'),
)
launcher.publish(submitData)
success_data['data'] = '数据存储成功'
return HttpResponse(json.dumps(success_data))
except IntegrityError:
except_data['data'] = 'IntegrityError 数据存储异常'
return HttpResponse(json.dumps(except_data))
else:
except_data['data'] = 'API验证失败'
return HttpResponse(json.dumps(except_data))
if request.method == 'GET':
except_data['data'] = '拒绝GET请求'
return HttpResponse(json.dumps(except_data))
def playbook_exe_sls(request):
if request.method == "POST":
minion_id_list = json.loads(request.POST.get('minion_id_list'))
playbook_id = request.POST.get('playbook_id')
# 生成任务编号number=yyyymmdd+000
last = Async_jobs.objects.last()
today = datetime.date.today().strftime('%Y%m%d')
try:
number = str(int(last.number) +
1) if last.number[0:8] == today else today + '001'
except AttributeError:
number = today + '001'
# 发布消息
state_execute = Redis_Queue('state_execute')
state_param = {
'number': number,
'minion_id_list': minion_id_list,
'playbook_id': playbook_id
}
state_execute.publish(state_param)
# 执行记录写入数据库
create_time = datetime.datetime.fromtimestamp(time.time())
description = PlayBook.objects.get(id=playbook_id)
jobs_info = Async_jobs(number=number,
description=description,
project=description.project,
create_time=create_time,
status=0,
targets_total=len(minion_id_list))
jobs_info.save()
try:
jobs_info.minion.add(*minion_id_list)
# 更新主机任务计数
for salt_id in minion_id_list:
Accepted_minion.objects.filter(salt_id=salt_id).update(
jobs_count=F('jobs_count') + 1)
except Exception as error:
# 异常返回
EXCEPT_DATA['data'] = {
'number': number,
'description': str(description),
'create_time': create_time.strftime("%H:%M:%S"),
'error': error,
}
return HttpResponse(json.dumps(EXCEPT_DATA))
# 成功返回
SUCCESS_DATA['data'] = {
'number': number,
'description': str(description),
'create_time': create_time.strftime("%H:%M:%S"),
'success': 'Join the queue',
}
return HttpResponse(json.dumps(SUCCESS_DATA))
def tables_update_cert(request):
if request.method == 'POST':
url_id = request.POST.get('url_id')
domain = DomainName.objects.get(id=url_id)
print(11,domain)
msg = {
'pattern':1,
'domain':domain.url
}
launcher = Redis_Queue(Webmoni_Check_Cert_Queue)
launcher.publish(msg)
success_data['data'] = '已加入检测队列'
return HttpResponse(json.dumps(success_data))
class Check_Cert_Worker(object):
def __init__(self):
# 存储检查过的域名最近一次检查时间
self.worker = Redis_Queue(Webmoni_Check_Cert_Queue)
self.radio = self.worker.subscribe()
def start(self):
"""
收听来自webmoni_check_cert频道的节目
:return:
"""
print('Check Cert Worker 已启动')
while True:
msg = self.radio.parse_response()
program = eval(msg[2].decode())
pattern = program.get('pattern')
if pattern == 0: # 节目模式=0 就检测所有域名
domain_list = DomainName.objects.filter(check_id=0).values('url')
for row in domain_list:
domain = row.get('url')
c = Cert_check(domain)
cert_info = c.py_check()
DomainName.objects.filter(url=domain).update(cert_valid_date=cert_info['endDate'],
cert_valid_days=cert_info['expire'])
elif pattern == 1: # 节目模式=1 检测单个域名
domain = program.get('domain')
c = Cert_check(domain)
cert_info = c.py_check()
DomainName.objects.filter(url=domain).update(cert_valid_date=cert_info['endDate'],
cert_valid_days=cert_info['expire'])
def tables_update_all_cert(request):
if request.method == 'POST':
now = request.POST.get('now')
last_check_all_cert_time = 0
if redis_conn.get('last_check_all_cert_time') is not None:
last_check_all_cert_time = redis_conn.get('last_check_all_cert_time').decode()
now_time = int(time.time())
if now_time - int(last_check_all_cert_time) > 7200:
msg = {
'pattern': 0,
}
launcher = Redis_Queue(Webmoni_Check_Cert_Queue)
launcher.publish(msg)
redis_conn.set('last_check_all_cert_time',now_time)
success_data['data'] = '开始更新'
return HttpResponse(json.dumps(success_data))
else:
except_data['data'] = '请勿频繁更新所有证书信息,冷却时间2小时!'
return HttpResponse(json.dumps(except_data))
def minion_add(request):
if request.method == "POST":
id = request.POST.get('id')
ipv4 = request.POST.get('ipv4')
city = request.POST.get('city')
now_time = datetime.datetime.fromtimestamp(time.time())
status = 2 #检测状态：0=离线，1=正常，2=检测中
# salt-key允许加入
key_manage = Key_manage()
key_manage.accept_key(minion_id=id)
# 主机信息入库
Accepted_minion.objects.create(id=id,
ipv4=ipv4,
city=city,
datetime=now_time,
status=status)
# 发送检测任务到redis队列
monion_check = Redis_Queue('check_minion')
# 模式1=test.ping,2=grains.items
test = {'pattern': 1, 'id': [id], 'add': True}
monion_check.publish(test)
grains = {'pattern': 2, 'id': [id], 'add': True}
monion_check.publish(grains)
SUCCESS_DATA['data'] = '添加成功'
msg = SUCCESS_DATA
# 返回结果
return HttpResponse(json.dumps(msg))
def minion_test(request):
if request.method == "POST":
# 接收id
id = request.POST.get('id')
minion_id_list = []
# 实例化
monion_check = Redis_Queue('check_minion')
key_manage = Key_manage()
# 检测全部主机在线状态
if id == '*':
Accepted_minion.objects.update(status=2)
minion_id_obj = Accepted_minion.objects.values('id')
for minion_id in minion_id_obj:
minion_id_list.append(minion_id['id'])
# 发送检测任务到redis队列 # pattern模式1=test.ping,2=grains.items
test = {'pattern': 1, 'id': minion_id_list, 'add': False}
monion_check.publish(test)
# 返回结果
SUCCESS_DATA['data'] = '检测中'
msg = SUCCESS_DATA
return HttpResponse(json.dumps(msg))
# 批量检测部分主机信息
else:
salt_id_list = json.loads(request.POST.get('id'))
minion_id_obj = Accepted_minion.objects.in_bulk(
salt_id_list).values()
# 已经允许的salt-key
accepted_list = key_manage.accepted_minion()
for minion_id in minion_id_obj:
# 判断主机有效性
if str(minion_id) in accepted_list:
# 更新数据库
Accepted_minion.objects.filter(id=minion_id).update(
status=2)
# 添加到检测列表
minion_id_list.append(str(minion_id))
else:
# salt-key已不存在，标记为异常:status=3
now_time = datetime.datetime.fromtimestamp(time.time())
errinfo = {
'datetime': now_time,
'status': 3,
'cpu_model': '',
'osfinger': '不存在的salt-key',
'mem_gib': 0,
'mem_total': 0,
'num_cpus': 0,
}
Accepted_minion.objects.filter(id=minion_id).update(
**errinfo)
# 发送检测任务到redis队列 # pattern模式1=test.ping,2=grains.items
grains = {'pattern': 2, 'id': minion_id_list, 'add': False}
monion_check.publish(grains)
# 返回页面
SUCCESS_DATA['data'] = '检测中'
msg = SUCCESS_DATA
return HttpResponse(json.dumps(msg))
class Send_Mail_Worker(object):
def __init__(self):
self.current = 0
self.now_data = {}
self.worker = Redis_Queue(Webmoni_Send_Mail_Queue)
self.radio = self.worker.subscribe()
def start(self):
print('Send Mail Worker 已启动')
while True:
msg = self.radio.parse_response()
program = eval(msg[2].decode())
current_Timestamp = program.get('time') - program.get('time') % 300
if self.current == 0:
self.current = current_Timestamp
status = program.get('status')
url_id = program.get('url_id')
if self.current != current_Timestamp:
self.send_mail(self.now_data)
self.now_data.clear()
self.current = current_Timestamp
# 本次数据没有超过5分钟的情况
if self.current == current_Timestamp:
if not self.now_data.get(program['domain']):
self.now_data[program['domain']] = {'failtag': 0, 'areas': {}}
if status != 100:
self.now_data[program['domain']]['failtag'] += 1
self.now_data[program['domain']]['areas'][str(program.get('node'))] = program.get('total_time')
def send_mail(self,last_data):
node_info = {}
for node in Node.objects.all():
node_info[str(node.id)] = node.node
for domain, y in last_data.items():
domain_obj = DomainName.objects.get(url=domain)
if domain_obj.warning == 0:
if y['failtag'] >= int(len(y['areas']) / 2) + 1:
DomainName.objects.filter(url=domain).update(status=99)
content = '域名:{}<br>CDN:{}<br>项目:{}<br>'.format(domain, domain_obj.cdn, domain_obj.project_name)
for area, val in y['areas'].items():
content += '{}:{}<br>'.format(node_info[area], val)
content += '<a href="http://47.91.239.10:80/webmoni/Nowarning/{}/">点击不警告</a><br>'.format(domain)
send_mail(domain, [Ming, Lin], content)
else:
DomainName.objects.filter(url=domain).update(status=100)
class Minion_Check_Worker(object):
def __init__(self):
self.worker = Redis_Queue('check_minion')
self.radio = self.worker.subscribe()
# 检测主机队列
def start(self):
print('Minion Check Worker 已启动')
while True:
# 接收队列消息
msg = self.radio.parse_response()
# 解析消息内容
msg = eval(msg[2])
time.sleep(10) if msg['add'] else None
# 调用test.ping测试
minion_check = Test_ping()
# 检测主机配置
grains = Grains()
if msg['pattern'] == 1: # 检测主机
minion_check.get_status(msg['id'])
if msg['pattern'] == 2: # 获取主机信息
grains.get_minion_items(msg['id'])
def __init__(self):
# 设定自动执行的间隔时间
self.interval = 200
# 实例化检测队列
self.check_minion = Redis_Queue('check_minion')
class Saltstack_Auto_Worker(object):
def __init__(self):
# 设定自动执行的间隔时间
self.interval = 200
# 实例化检测队列
self.check_minion = Redis_Queue('check_minion')
def start(self):
print('Saltstack Auto Worker 已启动')
while True:
# -----------------定时统计主机执行剧本的次数---开始-------------------------
minion_id_list = Accepted_minion.objects.all()
for minion_obj in minion_id_list:
jobs_count = Async_jobs.objects.filter(
minion__id=minion_obj.id).count()
Accepted_minion.objects.filter(id=minion_obj.id).update(
jobs_count=jobs_count)
# -----------------定时统计主机执行剧本的次数---结束-------------------------
time.sleep(self.interval)
# -----------------自动检测执行剧本排队异常---开始---------------------------
# 0=排队 1=执行中 2=执行完成 3=异常
hours_ago = datetime.datetime.fromtimestamp(time.time() - 3600 * 3)
except_jobs = Async_jobs.objects.defer('information').filter(
Q(status=0) | Q(status=1) & Q(create_time__lte=hours_ago))
for job in except_jobs:
if job.status == 0:
Async_jobs.objects.filter(id=job.id).update(
status=3, success_total=0)
elif job.status == 1 and job.information is not None:
try:
Async_jobs.objects.filter(id=job.id).update(
status=2,
success_total=eval(job.information)['success'])
except Exception:
Async_jobs.objects.filter(id=job.id).update(
status=2, success_total=0)
else:
Async_jobs.objects.filter(id=job.id).update(
status=3, success_total=0)
# -----------------自动检测执行剧本排队异常---结束---------------------------
time.sleep(self.interval)
# -----------------自动检测在线离线---开始----------------------------------
minion_id_list = []
# 从salt-key获取所有minion
key_manage = Key_manage()
accepted_list = key_manage.accepted_minion()
# 从数据库获取所有minion
minion_id_dict = Accepted_minion.objects.values('id')
# 判断数据库中的minion有效性，添加到minion_id_list
for minion_item in minion_id_dict:
if minion_item['id'] in accepted_list:
minion_id_list.append(minion_item['id'])
else:
# salt-key已不存在，标记为异常:status=3
now_time = datetime.datetime.fromtimestamp(time.time())
errinfo = {
'datetime': now_time,
'status': 3,
'cpu_model': '',
'osfinger': '不存在的salt-key',
'mem_gib': 0,
'mem_total': 0,
'num_cpus': 0,
}
Accepted_minion.objects.filter(
id=minion_item['id']).update(**errinfo)
# 发送检测消息到队列
test = {'pattern': 1, 'id': minion_id_list, 'add': False}
self.check_minion.publish(test)
# -----------------自动检测在线离线---结束----------------------------------
# 等待进入下一回合
time.sleep(self.interval)
def __init__(self):
self.worker = Redis_Queue('check_minion')
self.radio = self.worker.subscribe()
def __init__(self):
self.worker = Redis_Queue('state_execute')
self.radio = self.worker.subscribe()
self.client = client.LocalClient()
class Execute_PlayBook_Worker(object):
def __init__(self):
self.worker = Redis_Queue('state_execute')
self.radio = self.worker.subscribe()
self.client = client.LocalClient()
def start(self):
print('Execute PlayBook Worker 已启动')
while True:
# 接收队列消息
msg = self.radio.parse_response()
# 解析消息内容
msg = eval(msg[2])
print('state_execute频道接收消息：', msg)
# 执行状态
minion_state = Minion_state()
result = minion_state.exe_sls(msg['number'], msg['minion_id_list'],
msg['playbook_id'])
jid = result[0]
number = result[1]
t = 1
# 判断异常
if result[0] not in msg['minion_id_list'] and type(
jid) == int and jid > 0:
# 持续查询，直到出现结果，或者连接超时
while not self.client.get_cache_returns(jid):
time.sleep(1)
if t == 1200:
information = {'ERROR': 'Timeout'}
break
else:
t += 1
else:
information = self.client.get_cache_returns(jid)
# 继续查询，直到全部主机执行结果全部得到
if len(information) != len(
msg['minion_id_list']) and 'ERROR' not in information:
while len(self.client.get_cache_returns(jid)) != len(
msg['minion_id_list']):
time.sleep(1)
if t == 1200:
information = self.client.get_cache_returns(jid)
minion_id_obj = Accepted_minion.objects.in_bulk(
msg['minion_id_list']).values()
for minion_id in minion_id_obj:
if str(minion_id) not in information:
information[str(minion_id)] = {
'ret': ['Timeout']
}
break
else:
t += 1
else:
information = self.client.get_cache_returns(jid)
minion_state.save_sls(number=number, information=information)
print('任务完成，state_execute频道继续接收消息')
else:
information = {
'ERROR': '未选择主机或剧本'
} if jid == 0 else {
'不存在的主机': result[0]
}
minion_state.save_sls(number=number, information=information)
print('任务异常，state_execute频道继续接收消息')
continue
def __init__(self):
self.current = 0
self.now_data = {}
self.worker = Redis_Queue(Webmoni_Send_Mail_Queue)
self.radio = self.worker.subscribe()
def __init__(self):
# 存储检查过的域名最近一次检查时间
self.worker = Redis_Queue(Webmoni_Check_Cert_Queue)
self.radio = self.worker.subscribe()
