def _handle_event(self, event):
close_old_connections()
obj = SimpleLazyObject(
lambda: Detection.objects.filter(pk=event.job_id).first())
if event.code == EVENT_SCHEDULER_SHUTDOWN:
logger.info(f'EVENT_SCHEDULER_SHUTDOWN: {event}')
Notify.make_notify('monitor', '1', '调度器已关闭',
'调度器意外关闭，你可以在github上提交issue', False)
elif event.code == EVENT_JOB_MAX_INSTANCES:
logger.info(f'EVENT_JOB_MAX_INSTANCES: {event}')
Notify.make_notify('monitor', '1', f'{obj.name} - 达到调度实例上限',
'一般为上个周期的执行任务还未结束，请增加调度间隔或减少任务执行耗时')
elif event.code == EVENT_JOB_ERROR:
logger.info(
f'EVENT_JOB_ERROR: job_id {event.job_id} exception: {event.exception}'
)
Notify.make_notify('monitor', '1', f'{obj.name} - 执行异常',
f'{event.exception}')
elif event.code == EVENT_JOB_EXECUTED:
is_ok, out = event.retval
obj = Detection.objects.filter(pk=event.job_id).first()
is_notified = True if obj.latest_notify_time else False
if obj.latest_status in [0, None] and is_ok is False:
obj.latest_fault_time = int(time.time())
if is_ok:
obj.latest_notify_time = 0
obj.fault_times = 0
else:
obj.fault_times += 1
obj.latest_status = 0 if is_ok else 1
obj.latest_run_time = human_datetime(event.scheduled_run_time)
obj.save()
self._handle_notify(obj, is_notified, out)
def notify_by_wx(event, obj):
spug_key, u_ids = _parse_args(obj.grp)
if not spug_key:
Notify.make_notify(notify_source, '1', '发送报警信息失败',
'未配置报警服务调用凭据，请在系统管理/系统设置/报警服务设置中配置。')
return
users = set(
x.wx_token
for x in Contact.objects.filter(id__in=u_ids, wx_token__isnull=False))
if users:
data = {
'token': spug_key,
'event': event,
'subject': obj.name,
'desc': obj.out,
'remark': f'故障持续{obj.duration}' if event == '2' else None,
'users': list(users)
}
requests.post(f'{spug_server}/apis/notify/wx/', json=data)
else:
Notify.make_notify(notify_source, '1', '发送报警信息失败',
'未找到可用的通知对象，请确保设置了相关报警联系人的微信Token。')
def notify_by_email(event, subject, grp):
spug_key, u_ids = _parse_args(grp)
users = set(x.email for x in Contact.objects.filter(id__in=u_ids, email__isnull=False))
if users:
mail_service = json.loads(AppSetting.get_default('mail_service', '{}'))
if mail_service.get('server'):
event_map = {'1': '告警', '2': '恢复'}
subject = f'{event_map[event]}-{subject}'
mail = Mail(**mail_service)
mail.send_text_mail(users, subject, f'{subject}\r\n\r\n自动发送，请勿回复。')
elif spug_key:
data = {
'token': spug_key,
'event': event,
'subject': subject,
'users': list(users)
}
requests.post(f'{spug_server}/apis/notify/mail/', json=data)
else:
Notify.make_notify(notify_source, '1', '发送报警信息失败', '未配置报警服务调用凭据，请在系统管理/系统设置/报警服务设置中配置。')
else:
Notify.make_notify(notify_source, '1', '发送报警信息失败', '未找到可用的通知对象，请确保设置了相关报警联系人的邮件地址。')
def _handle_response(res, mode):
if res.status_code != 200:
Notify.make_notify(notify_source, '1', '告警通知发送失败',
f'返回状态码：{res.status_code}, 请求URL：{res.url}')
if mode in ['dd', 'wx']:
res = res.json()
if res.get('errcode') != 0:
Notify.make_notify(notify_source, '1', '告警通知发送失败', f'返回数据：{res}')
if mode == 'spug':
res = res.json()
if res.get('error'):
Notify.make_notify(notify_source, '1', '告警通知发送失败', f'错误信息：{res}')
def _handle_event(self, event):
close_old_connections()
obj = SimpleLazyObject(
lambda: Task.objects.filter(pk=event.job_id).first())
if event.code == EVENT_SCHEDULER_SHUTDOWN:
logger.info(f'EVENT_SCHEDULER_SHUTDOWN: {event}')
Notify.make_notify('schedule', '1', '调度器已关闭',
'调度器意外关闭，你可以在github上提交issue')
elif event.code == EVENT_JOB_MAX_INSTANCES:
logger.info(f'EVENT_JOB_MAX_INSTANCES: {event}')
Notify.make_notify('schedule', '1', f'{obj.name} - 达到调度实例上限',
'一般为上个周期的执行任务还未结束，请增加调度间隔或减少任务执行耗时')
elif event.code == EVENT_JOB_ERROR:
logger.info(
f'EVENT_JOB_ERROR: job_id {event.job_id} exception: {event.exception}'
)
Notify.make_notify('schedule', '1', f'{obj.name} - 执行异常',
f'{event.exception}')
elif event.code == EVENT_JOB_EXECUTED:
job = self.scheduler.get_job('3')
print(job.id, job.name, job.next_run_time)
if event.retval:
score = 0
for item in event.retval:
score += 1 if item[1] else 0
history = History.objects.create(
task_id=event.job_id,
status=2
if score == len(event.retval) else 1 if score else 0,
run_time=human_datetime(event.scheduled_run_time),
output=json.dumps(event.retval))
Task.objects.filter(pk=event.job_id).update(latest=history)
if score != 0 and time.time() - counter.get(event.job_id,
0) > 3600:
counter[event.job_id] = time.time()
Notify.make_notify('schedule', '1', f'{obj.name} - 执行失败',
'请在任务计划中查看失败详情')