def _handle_event(self, event): close_old_connections() obj = SimpleLazyObject( lambda: Detection.objects.filter(pk=event.job_id).first()) if event.code == EVENT_SCHEDULER_SHUTDOWN: logger.info(f'EVENT_SCHEDULER_SHUTDOWN: {event}') Notify.make_notify('monitor', '1', '调度器已关闭', '调度器意外关闭,你可以在github上提交issue', False) elif event.code == EVENT_JOB_MAX_INSTANCES: logger.info(f'EVENT_JOB_MAX_INSTANCES: {event}') Notify.make_notify('monitor', '1', f'{obj.name} - 达到调度实例上限', '一般为上个周期的执行任务还未结束,请增加调度间隔或减少任务执行耗时') elif event.code == EVENT_JOB_ERROR: logger.info( f'EVENT_JOB_ERROR: job_id {event.job_id} exception: {event.exception}' ) Notify.make_notify('monitor', '1', f'{obj.name} - 执行异常', f'{event.exception}') elif event.code == EVENT_JOB_EXECUTED: is_ok, out = event.retval obj = Detection.objects.filter(pk=event.job_id).first() is_notified = True if obj.latest_notify_time else False if obj.latest_status in [0, None] and is_ok is False: obj.latest_fault_time = int(time.time()) if is_ok: obj.latest_notify_time = 0 obj.fault_times = 0 else: obj.fault_times += 1 obj.latest_status = 0 if is_ok else 1 obj.latest_run_time = human_datetime(event.scheduled_run_time) obj.save() self._handle_notify(obj, is_notified, out)
def notify_by_wx(event, obj): spug_key, u_ids = _parse_args(obj.grp) if not spug_key: Notify.make_notify(notify_source, '1', '发送报警信息失败', '未配置报警服务调用凭据,请在系统管理/系统设置/报警服务设置中配置。') return users = set( x.wx_token for x in Contact.objects.filter(id__in=u_ids, wx_token__isnull=False)) if users: data = { 'token': spug_key, 'event': event, 'subject': obj.name, 'desc': obj.out, 'remark': f'故障持续{obj.duration}' if event == '2' else None, 'users': list(users) } requests.post(f'{spug_server}/apis/notify/wx/', json=data) else: Notify.make_notify(notify_source, '1', '发送报警信息失败', '未找到可用的通知对象,请确保设置了相关报警联系人的微信Token。')
def notify_by_email(event, subject, grp): spug_key, u_ids = _parse_args(grp) users = set(x.email for x in Contact.objects.filter(id__in=u_ids, email__isnull=False)) if users: mail_service = json.loads(AppSetting.get_default('mail_service', '{}')) if mail_service.get('server'): event_map = {'1': '告警', '2': '恢复'} subject = f'{event_map[event]}-{subject}' mail = Mail(**mail_service) mail.send_text_mail(users, subject, f'{subject}\r\n\r\n自动发送,请勿回复。') elif spug_key: data = { 'token': spug_key, 'event': event, 'subject': subject, 'users': list(users) } requests.post(f'{spug_server}/apis/notify/mail/', json=data) else: Notify.make_notify(notify_source, '1', '发送报警信息失败', '未配置报警服务调用凭据,请在系统管理/系统设置/报警服务设置中配置。') else: Notify.make_notify(notify_source, '1', '发送报警信息失败', '未找到可用的通知对象,请确保设置了相关报警联系人的邮件地址。')
def _handle_response(res, mode): if res.status_code != 200: Notify.make_notify(notify_source, '1', '告警通知发送失败', f'返回状态码:{res.status_code}, 请求URL:{res.url}') if mode in ['dd', 'wx']: res = res.json() if res.get('errcode') != 0: Notify.make_notify(notify_source, '1', '告警通知发送失败', f'返回数据:{res}') if mode == 'spug': res = res.json() if res.get('error'): Notify.make_notify(notify_source, '1', '告警通知发送失败', f'错误信息:{res}')
def _handle_event(self, event): close_old_connections() obj = SimpleLazyObject( lambda: Task.objects.filter(pk=event.job_id).first()) if event.code == EVENT_SCHEDULER_SHUTDOWN: logger.info(f'EVENT_SCHEDULER_SHUTDOWN: {event}') Notify.make_notify('schedule', '1', '调度器已关闭', '调度器意外关闭,你可以在github上提交issue') elif event.code == EVENT_JOB_MAX_INSTANCES: logger.info(f'EVENT_JOB_MAX_INSTANCES: {event}') Notify.make_notify('schedule', '1', f'{obj.name} - 达到调度实例上限', '一般为上个周期的执行任务还未结束,请增加调度间隔或减少任务执行耗时') elif event.code == EVENT_JOB_ERROR: logger.info( f'EVENT_JOB_ERROR: job_id {event.job_id} exception: {event.exception}' ) Notify.make_notify('schedule', '1', f'{obj.name} - 执行异常', f'{event.exception}') elif event.code == EVENT_JOB_EXECUTED: job = self.scheduler.get_job('3') print(job.id, job.name, job.next_run_time) if event.retval: score = 0 for item in event.retval: score += 1 if item[1] else 0 history = History.objects.create( task_id=event.job_id, status=2 if score == len(event.retval) else 1 if score else 0, run_time=human_datetime(event.scheduled_run_time), output=json.dumps(event.retval)) Task.objects.filter(pk=event.job_id).update(latest=history) if score != 0 and time.time() - counter.get(event.job_id, 0) > 3600: counter[event.job_id] = time.time() Notify.make_notify('schedule', '1', f'{obj.name} - 执行失败', '请在任务计划中查看失败详情')