def sync_nodes_exception(self):
"""同步节点异常（性能指标）"""
system_settings = SystemSettingsModel.get_settings()
# Master
for key, value in self.get_master_performance().items():
threshold = system_settings.get("threshold_{}".format(key))
if not threshold or threshold > value:
continue
NodesExceptionsModel.merge_one(
host_port="Master",
node_type="master",
exc_time=datetime.datetime.now(),
exc_level="WARNING",
exc_message="Current {} is {}(>={})!".format(
key, value, threshold))
pass
# Slave
for host_port in self.slaves:
for key, value in self.get_salve_performance(host_port).items():
threshold = system_settings.get("threshold_{}".format(key))
if not threshold or threshold > value:
continue
NodesExceptionsModel.merge_one(
host_port=host_port,
node_type="slave",
exc_time=datetime.datetime.now(),
exc_level="WARNING",
exc_message="Current {} is {}(>={})!".format(
key, value, threshold))
pass
pass
def node_exception_update():
vc_md5 = request.form.get('id', request.form.get('vc_md5'))
act = request.form.get('act')
# 更新
kwargs = request.form.to_dict()
# 处理checkBox值
is_closed = True if 'is_closed' in kwargs else False
NodesExceptionsModel.update_one(vc_md5=vc_md5,
is_closed=is_closed,
remark=kwargs.get('remark', ''))
flash('{} success!'.format(act))
return redirect('/client/exception/manage', code=302)
pass
def sync_email_exception_job():
"""
email exception by scheduler
通过调度发送异常邮件
:param self:
:return:
"""
logger.info('Start email exception job')
eoh = EmailOperationHelper(**settings.MAIL_KWARGS)
sys_conf = SystemSettingsModel.get_settings()
if not sys_conf.get("use_email_alert"):
logger.info('End email exception job, the email alert is closed')
return
default_recipients = sys_conf.get("default_recipients")
for node in NodesModel.get_list():
recipients = node.recipients or default_recipients
if not recipients:
continue
recipients = recipients.split(";")
subject = "Node({}-{}) Exception of SpiderPlatform".format(
node.node_type, node.host_port)
models = NodesExceptionsModel.get_limit(
is_closed=False, is_emailed=False, node_md5=node.vc_md5)
if not models:
continue
content = "\r\n\r\n".join([m.get_email_content() for m in models])
eoh.send_mail(subject, content, recipients=recipients)
for model in models:
NodesExceptionsModel.update_one(model=model, is_emailed=True)
pass
for project in ProjectsModel.get_list():
recipients = project.recipients or default_recipients
if not recipients:
continue
recipients = recipients.split(";")
subject = "Project({}) Exception of SpiderPlatform".format(
project.project_name)
models = JobsExceptionsModel.get_limit(
is_closed=False, is_emailed=False,
project_name=project.project_name)
if not models:
continue
content = "\r\n\r\n".join([m.get_email_content() for m in models])
eoh.send_mail(subject, content, recipients=recipients)
for model in models:
JobsExceptionsModel.update_one(model=model, is_emailed=True)
pass
logger.info('End email exception job')
pass
def node_exception_search():
page_index = request.form.get('pageNum', 1)
page_size = request.form.get('pageSize', 10)
node_md5 = request.form.get('dataID')
is_closed = request.form.get('dataType', False)
keywords = request.form.get('keywords')
# 查询分页结果集
if node_md5:
pagination = NodesExceptionsModel.get_pagination(page_index=page_index,
page_size=page_size,
is_closed=is_closed,
node_md5=node_md5,
keywords=keywords)
else:
pagination = NodesExceptionsModel.get_pagination(page_index=page_index,
page_size=page_size,
is_closed=is_closed,
node_type="slave",
keywords=keywords)
return jsonify(pagination)
def handle_error(e):
code = 500
if isinstance(e, HTTPException):
code = e.code
# app.logger.error(traceback.print_exc())
msg = '{}==>{}\r\n{}'.format(type(e), e, traceback.print_exc())
current_app.logger.error(msg) # 写入日志
# NodesExceptionsModel.merge_exception(msg) # 写入数据库
NodesExceptionsModel.merge_one(
host_port="Master",
node_type="master",
exc_time=datetime.datetime.now(),
exc_level="ERROR",
exc_message=msg
)
return jsonify({
'code': code,
'success': False,
'msg': str(e),
'data': None
})
def my_listener(event):
"""
监听Scheduler发出的事件并作出处理，如任务执行完、任务出错等
:param event: Scheduler 事件
:return:
"""
if event.exception:
# or logger.fatal
logger.critical('The job of {} crashed :('.format(event.job_id))
msg = '{}==>{}\r\n{}'.format(type(event.exception), event.exception,
event.traceback)
logger.error(msg) # 写入日志
# NodesExceptionsModel.merge_exception(msg) # 写入数据库
NodesExceptionsModel.merge_one(
host_port="Master",
node_type="master",
exc_time=datetime.datetime.now(),
exc_level="ERROR",
exc_message=msg
)
else:
logger.info('The job of {} worked :)'.format(event.job_id))
def node_exception_send():
sys_conf = SystemSettingsModel.get_settings()
if not sys_conf.get("use_email_alert"):
print('End email exception job, the email alert is closed')
return
default_recipients = sys_conf.get("default_recipients")
for node in NodesModel.get_list():
recipients = node.recipients or default_recipients
if not recipients:
continue
recipients = recipients.split(";")
subject = "Node({}-{}) Exception".format(node.node_type,
node.host_port)
template = "nodes/email/exceptions"
exceptions = [
m.to_dict() for m in NodesExceptionsModel.get_limit(
is_closed=False, is_emailed=False, node_md5=node.vc_md5)
]
send_emails(subject, template, *recipients, exceptions=exceptions)
pass
pass
def node_exception_delete(exception_md5):
NodesExceptionsModel.del_one(vc_md5=exception_md5)
flash('Delete success!')
return redirect(request.referrer)
pass