class MonitorServerIpHandler(RequestHandler): def __init__(self, application, request, **kwargs): super(MonitorServerIpHandler, self).__init__(application, request, **kwargs) self.db_session = Session() def __del__(self): self.db_session.close() @except_handler() def get(self, req_data): data_list = self.db_session.query(MonitorServerIp).filter( MonitorServerIp._region_id == None) serialized = [ AlchemyEncoder().parse_sqlalchemy_object(item) for item in data_list ] return {'instance': serialized, 'result': 0, 'message': 'success'} @except_handler() def post(self, req_data): form = MonitorServerIpForm(ImmutableMultiDict(req_data)) if not form.validate(): raise Status400(form.errors) result = MonitorServerIp._validate(self.db_session, **req_data) instance = result['instance'] if instance is None: raise Status403(result['message']) instance._save(self.db_session) return { 'instance': AlchemyEncoder().parse_sqlalchemy_object(instance), 'result': 0, 'message': 'success' } ''' 支持多个ServerIp删除 ''' @except_handler() def delete(self, req_data): if not req_data.has_key('id'): raise Status400('id is required') failed = [] success = [] for ip_id in req_data['id']: result = MonitorServerIp._validate(self.db_session, ip_id) instance = result['instance'] if instance is None: failed.append(ip_id) else: success.append(ip_id) instance._delete(self.db_session) return { 'result': 0, 'message': 'success', 'success': success, 'failed': failed }
class MonitorAlertLogDetailHandler(RequestHandler): def initialize(self): self.db_session = Session() def on_finish(self): self.db_session.close() @except_handler() def get(self, req_data): if not req_data.has_key('id'): raise Status400('id is required') instance = self.db_session.query(MonitorAlertLog).get( req_data['id'][0]) if not instance: raise Status403('alert log id is invalid') origin_data = AlchemyEncoder().parse_sqlalchemy_object(instance) key_list = [ 'start_time', 'region', 'state', 'content_detail', 'rule_name', 'broadband', 'id' ] ret_data = { key: value for key, value in origin_data.iteritems() if key in key_list } if origin_data['state'] == 'close': stop_time = datetime.datetime.strptime(origin_data['stop_time'], '%Y-%m-%d %H:%M:%S') start_time = datetime.datetime.strptime(origin_data['start_time'], '%Y-%m-%d %H:%M:%S') if stop_time < start_time: time_delta = start_time - stop_time else: time_delta = stop_time - start_time ret_data['duration'] = time_delta.seconds / 60 else: now = datetime.datetime.now() start_time = datetime.datetime.strptime(origin_data['start_time'], '%Y-%m-%d %H:%M:%S') time_delta = now - start_time ret_data['duration'] = time_delta.seconds / 60 if origin_data['county'] == '*': ret_data['county'] = origin_data['province'] else: ret_data['county'] = '{0}{1}'.format(origin_data['province'], origin_data['county']) return { 'result': 0, 'message': 'success', 'instance': ret_data, }
class MonitorAlertLogHandler(RequestHandler): def initialize(self): self.db_session = Session() def on_finish(self): self.db_session.close() @except_handler() def get(self, req_data): form = MonitorAlertLogGetForm(ImmutableMultiDict(req_data)) if not form.validate(): raise Status400(form.errors) if form.data['state'] == 'all': data_list = self.db_session.query(MonitorAlertLog).filter( MonitorAlertLog.start_time > form.data['starttime'], MonitorAlertLog.start_time < form.data['stoptime'], ).all() else: data_list = self.db_session.query(MonitorAlertLog).filter( MonitorAlertLog.start_time > form.data['starttime'], MonitorAlertLog.start_time < form.data['stoptime'], MonitorAlertLog.state == form.data['state'], ).all() serialized = [] for item in data_list: rec = { 'uuid': item.id, 'state': item.state, 'starttime': item.start_time, 'content': item.content, 'region': item.region, 'dest': u'{0}{1}'.format(item.province, item.county), } start_time = datetime.datetime.strptime(item.start_time, '%Y-%m-%d %H:%M:%S') stop_time = datetime.datetime.strptime(item.stop_time, '%Y-%m-%d %H:%M:%S') if item.state == 'close': if stop_time > start_time: rec['duration'] = (stop_time - start_time).seconds / 60 else: rec['duration'] = (start_time - stop_time).seconds / 60 else: rec['duration'] = (datetime.datetime.now() - start_time).seconds / 60 serialized.append(rec) return {'instance': serialized, 'result': 0, 'message': 'success'}
class MonitorItemHandler(RequestHandler): def __init__(self, application, request, **kwargs): super(MonitorItemHandler, self).__init__(application, request, **kwargs) self.db_session = Session() def __del__(self): self.db_session.close() @except_handler() def get(self, req_data): data_list = self.db_session.query(MonitorItem).all() serialized = [ AlchemyEncoder().parse_sqlalchemy_object(item) for item in data_list ] return {'instance': serialized, 'result': 0, 'message': 'success'}
class MonitorAlertConfirmHandler(RequestHandler): def initialize(self): self.db_session = Session() self.redis = RedisCache() def on_finish(self): self.db_session.close() ''' trig_id: MonitorTrigger id ''' def make_confirm(self, trig_id, ip, province, county, broadband, appid, state): flag = 'cfm:{0};{1};{2};{3};{4};{5}'.format(trig_id, ip, province, county, broadband, appid) if state == 'confirm': ret = self.redis.set_data(flag, True) if state == 'cansel': ret = self.redis.del_data(flag) if not ret: return False return True @except_handler() def post(self, req_data): if not req_data.has_key('id'): raise Status400('id is required') instance = self.db_session.query(MonitorAlertLog).get( req_data['id'][0]) if not instance: raise Status400('id not exists') if instance.state != 'open': raise Status400('state is not open') ret = self.make_confirm(instance.trigger, instance.serverip, instance.province, instance.county, instance.broadband, instance.appid, 'confirm') if not ret: return {'result': -1, 'message': 'success', 'id': req_data['id']} instance.state = 'confirm' instance.content_detail = u'{}<br/>告警已经确认。'.format( instance.content_detail).encode('utf8') instance._save(self.db_session, update=True) return {'result': 0, 'message': 'success', 'id': req_data['id']}
class MonitorAlertDashDetailHandler(RequestHandler): def initialize(self): self.db_session = Session() def on_finish(self): self.db_session.close() @except_handler() def get(self, req_data): form = MonitorAlertDashDetailForm(ImmutableMultiDict(req_data)) if not form.validate(): raise Status400(form.errors) form_data = form.data.copy() region = form_data.pop('region') ser_instance = self.db_session.query(MonitorRegion).get(region) if not ser_instance: raise Status403('region id is invalid') server_list = [i.ip for i in ser_instance._serverips] # logger.debug('{0}'.format(form_data)) form_data['broadband'] = form_data['broadband'].split(',') origin_data = es_client.get_province_record( settings.opt['ELASTICSEARCH']['INDEX'], servers=server_list, **form_data) # logger.debug('{0}'.format(origin_data)) ip_total = 0 avg_delay, avg_loss = 0, 0 for item in origin_data['county_net']: ip_total = ip_total + item['counter'] for item in origin_data['county_net']: avg_delay = avg_delay + item['delay'] * item['counter'] / ip_total avg_loss = avg_loss + item['loss_rate'] * item['counter'] / ip_total item['ip_perc'] = item['counter'] / ip_total * 100 return { 'return': 0, 'message': 'success', 'ip_total': ip_total, 'avg_delay': avg_delay, 'avg_loss': avg_loss, 'update_time': origin_data['updatetime'], 'instance': origin_data['county_net'] }
class MonitorRegionServerIpHandler(RequestHandler): def __init__(self, application, request, **kwargs): super(MonitorRegionServerIpHandler, self).__init__(application, request, **kwargs) self.db_session = Session() def __del__(self): self.db_session.close() @except_handler() def get(self, req_data): if not req_data.has_key('id'): raise Status400('id is required') result = MonitorRegion._validate(self.db_session, req_data['id']) if result['instance'] is None: raise Status403(result['message']) serialized = [ AlchemyEncoder().parse_sqlalchemy_object(item) for item in result['instance']._serverips ] return {'instance': serialized, 'result': 0, 'message': 'success'}
# -*- coding:utf-8 -*- from model.rule import * from model.common import Session session = Session() for item in [{ 'name': u'loss_rate', 'comment': u'网络丢包率'.encode('utf-8'), 'unit': '%' }, { 'name': u'delay', 'comment': u'网络延迟'.encode('utf-8'), 'unit': 'ms' }]: session.add(MonitorItem(**item)) # session.add(MonitorRegion(name='beijing-1-qu', _serverips=[MonitorServerIp(ip='10.40.44.2')])) # session.add(MonitorRegion(name='beijing-2-qu', _serverips=[ # MonitorServerIp(ip='120.1.12.3'), # # MonitorServerIp(ip='127.0.0.3'), # ])) session.commit() # item_ids = session.query(MonitorItem).all() # triggers_list = [ # { # "period": "1min", # "time": "5c", # "repeat": "1h", # "compare": ">", # "threshold": 90,
def __init__(self, application, request, **kwargs): super(MonitorRegionHandler, self).__init__(application, request, **kwargs) self.db_session = Session()
class MonitorRegionHandler(RequestHandler): def __init__(self, application, request, **kwargs): super(MonitorRegionHandler, self).__init__(application, request, **kwargs) self.db_session = Session() def __del__(self): self.db_session.close() @except_handler() def get(self, req_data): data_list = self.db_session.query(MonitorRegion).all() serialized = [ AlchemyEncoder().parse_sqlalchemy_object(item) for item in data_list ] return {'instance': serialized, 'result': 0, 'message': 'success'} @except_handler() def post(self, req_data): form = MonitorRegionPostForm(data=ImmutableMultiDict(req_data)) if not form.validate(): raise Status400(form.errors) result = MonitorRegion._validate(self.db_session, **req_data) instance = result['instance'] if instance is None: raise Status403(result['message']) instance._save(self.db_session) return { 'instance': AlchemyEncoder().parse_sqlalchemy_object(instance), 'result': 0, 'message': 'success' } @except_handler() def put(self, req_data): if not req_data.has_key('id'): raise Status400('id is required') form = MonitorRegionPutForm(data=ImmutableMultiDict(req_data)) if not form.validate(): raise Status400(form.errors) result = MonitorRegion._validate(self.db_session, req_data['id'], **req_data) instance = result['instance'] if instance is None: raise Status403(result['message']) instance._save(self.db_session) return { 'id': '{}'.format(instance.id), 'result': 0, 'message': 'success' } @except_handler() def delete(self, req_data): if not req_data.has_key('id'): raise Status400('id is required') result = MonitorRegion._validate(self.db_session, req_data['id']) instance = result['instance'] if instance is None: raise Status403(result['message']) instance._delete(self.db_session) return { 'id': '{}'.format(instance.id), 'result': 0, 'message': 'success' }
class MonitorRuleHandler(RequestHandler): def __init__(self, application, request, **kwargs): super(MonitorRuleHandler, self).__init__(application, request, **kwargs) self.db_session = Session() def __del__(self): self.db_session.close() # @except_handler() # def get(self, req_data): # data_list = self.db_session.query(MonitorRule).all() # serialized = [] # for rule in data_list: # triggers = [AlchemyEncoder().parse_sqlalchemy_object(item) for item in rule._triggers] # regions = [item.id for item in rule._regions] # actions = [AlchemyEncoder().parse_sqlalchemy_object(item) for item in rule._actions] # serialized.append( # {'id':rule.id, 'triggers':triggers, 'regions':regions, # 'actions':actions, 'name':rule.name} # ) # return {'instance': serialized, 'result': 0, 'message': 'success'} @except_handler() def get(self, req_data): data_list = self.db_session.query(MonitorRule).all() serialized = [] for rule in data_list: triggers = [] for item in rule._triggers: triggers.append({ 'id': item.id, 'period': item.period, 'time': item.time, 'repeat': item.repeat, 'compare': item.compare, 'threshold': item.threshold, 'update_time': item.update_time, 'item': { 'id': item._item.id, 'name': item._item.name, 'comment': item._item.comment, 'unit': item._item.unit } }) regions = [item.id for item in rule._regions] actions = [ AlchemyEncoder().parse_sqlalchemy_object(item) for item in rule._actions ] serialized.append({ 'id': rule.id, 'triggers': triggers, 'regions': regions, 'actions': actions, 'name': rule.name }) return {'instance': serialized, 'result': 0, 'message': 'success'} @except_handler() def post(self, req_data): form = MonitorRuleForm(data=MultiDict(req_data)) if not form.validate(): raise Status400(form.errors) result = MonitorRule._validate(self.db_session, **req_data) instance = result['instance'] if instance is None: raise Status403(result['message']) instance._save(self.db_session) return { 'instance': AlchemyEncoder().parse_sqlalchemy_object(instance), 'result': 0, 'message': 'success' } ''' 删除告警策略中的告警规则,每条告警规则只属于一个告警策略 ''' @except_handler() def delete(self, req_data): if not req_data.has_key('id'): raise Status400('id is required') result = MonitorTrigger._validate(self.db_session, req_data['id']) instance = result['instance'] if instance is None: raise Status403(result['message']) if len(instance._rule._triggers) == 1: # 删除整条告警规则 for i in instance._rule._actions: i._delete(self.db_session, commit=False) instance._rule._delete(self.db_session, commit=False) instance._delete(self.db_session) return { 'id': '{}'.format(instance.id), 'result': 0, 'message': 'success', } '''
# -*- coding:utf-8 -*- from model.rule import * from model.region import * from model.common import Session session = Session() for item in [{ 'name': u'loss_rate', 'comment': u'网络丢包率'.encode('utf-8'), 'unit': '%' }, { 'name': u'delay', 'comment': u'网络延迟'.encode('utf-8'), 'unit': 'ms' }]: session.add(MonitorItem(**item)) session.add( MonitorRegion(name='beijing-1-qu', _serverips=[MonitorServerIp(ip='10.40.44.2')])) session.add( MonitorRegion( name='beijing-2-qu', _serverips=[ MonitorServerIp(ip='120.1.12.3'), # MonitorServerIp(ip='127.0.0.3'), ])) session.commit() item_ids = session.query(MonitorItem).all() triggers_list = [{
def initialize(self): self.db_session = Session()
def initialize(self): self.db_session = Session() self.redis = RedisCache()
class MonitorAlertHandler(RequestHandler): def initialize(self): self.db_session = Session() self.redis = RedisCache() def on_finish(self): self.db_session.close() @except_handler() def get(self, req_data): form = MonitorAlertDashGetForm(ImmutableMultiDict(req_data)) if not form.validate(): raise Status400(form.errors) form_data = form.data.copy() region = form_data.pop('region') ser_instance = self.db_session.query(MonitorRegion).get(region) if not ser_instance: raise Status403('region id is invalid') server_list = [i.ip for i in ser_instance._serverips] form_data['broadband'] = form_data['broadband'].split(',') origin_data = es_client.get_region_record( settings.opt['ELASTICSEARCH']['INDEX'], servers=server_list, **form_data) for item in origin_data: if form_data['datatype'] == 'delay': item['level'] = MonitorAlertHandler.set_level( form_data['datatype'], delay=item['delay']) elif form_data['datatype'] == 'loss_rate': item['level'] = MonitorAlertHandler.set_level( form_data['datatype'], loss_rate=item['loss_rate']) else: item['level'] = MonitorAlertHandler.set_level( form_data['datatype'], item['delay'], item['loss_rate']) return {'return': 0, 'message': 'success', 'instance': origin_data} @staticmethod def set_level(datatype, delay=0, loss_rate=0): loss_level, delay_level = 0, 0 if datatype == 'loss_rate' or datatype == 'all': if loss_rate >= 0 and loss_rate <= 0.5: loss_level = 0 elif loss_rate >= 0.6 and loss_rate <= 1.0: loss_level = 1 elif loss_rate >= 1.1 and loss_rate <= 2.0: loss_level = 2 elif loss_rate >= 2.1 and loss_rate <= 5.0: loss_level = 3 else: loss_level = 4 if datatype == 'delay' or datatype == 'all': if delay >= 0 and delay <= 50: delay_level = 0 elif delay > 50 and delay <= 100: delay_level = 1 elif delay > 100 and delay <= 150: delay_level = 2 elif delay > 150 and delay <= 200: delay_level = 3 else: delay_level = 4 if datatype == 'loss_rate': return loss_level elif datatype == 'delay': return delay_level else: return delay_level if delay_level > loss_level else loss_level ''' serialize original data input: [ "_id": xxxxxx, "_source": { "datetime": '2017-08-01 12:12:00' "province": 'beijing' "county": '*' "server": '192.168.152.22' "data": '{ 'province': 'beijing', 'country': 'china', 'county': '*', 'server': '10.40.44.2', 'delay': 88, 'loss_rate': 92, 'time': 1502071380, 'broadband': 'xxx', 'counter': 29, 'app_id': '0987654321', 'max_delay': 145, 'min_delay': 64, 'type': 'network', 'devices': 1, }' } ... ] trigger和severip联合识别,一个报警对应于一个trigger,一个serverip组成的共同记录 ''' def alert_handler(self, req_data): alert_result = [] # province = req_data['province'] # county = req_data['county'] serverip = self.db_session.query(MonitorServerIp).filter( MonitorServerIp.ip == req_data['server']).first() if not serverip: return alert_result region = self.db_session.query(MonitorRegion).filter( MonitorRegion._serverips.contains(serverip)).first() if not region: return alert_result rules = self.db_session.query(MonitorRule).filter( MonitorRule._regions.contains(region)).all() for r in rules: for r_trigger in r._triggers: ''' 基于trigger, 从ES中取数据基于trigger.period ''' time_int = TimeInt[r_trigger.time] stoptime = datetime.datetime.strptime(req_data['time'], '%Y-%m-%d %H:%M:%S') t_delta = PeriodInt[r_trigger.period] * time_int starttime = stoptime + datetime.timedelta(minutes=-t_delta) history_data = es_client.get_record( settings.opt['ELASTICSEARCH']['INDEX'], starttime, stoptime, req_data['province'], req_data['county'], req_data['broadband'], req_data['server'], req_data['app_id'], PeriodInt[r_trigger.period], r_trigger._item.name) logger.debug('{0}'.format(r_trigger._item.name)) logger.debug('{0}'.format(history_data)) logger.debug('{0}'.format(r_trigger.time)) alert_ret = MonitorAlertHandler.alert_check( history_data, time_int, r_trigger.threshold, r_trigger.compare) if alert_ret is True: alert_result.append({ 'alert': True, 'trigger': r_trigger, 'region': r, 'data': history_data[1:] }) else: alert_result.append({ 'alert': False, 'trigger': r_trigger, 'region': r, 'data': history_data[1:] }) return alert_result @staticmethod def alert_check(data_list, last_time, threshold, compare): if last_time > len(data_list): return False for value in data_list[0:last_time]: if compare == '>': ret = True if value > threshold else False elif compare == '<': ret = True if value < threshold else False else: ret = True if value == threshold else False if not ret: break return ret ''' trig_id: MonitorTrigger id output: 告警已经确认: True 告警未确认: False ''' def check_confirm(self, trig_id, ip, province, county, broadband, appid): flag = 'cfm:{0};{1};{2};{3};{4};{5}'.format(trig_id, ip, province, county, broadband, appid) time_pre = self.redis.get_data(flag) if not time_pre: return False return True ''' input: MonitorTrigger instance output: 在重复时间段内: 'in_repeat' 不在重复时间段内: 'out_repeat' 之前无告警: 'no_alert' ''' def check_alert(self, trig, ip, province, county, broadband, appid, time_now): if trig.repeat == 'no_repeat': return 'out_repeat' flag = 'trig:{0};{1};{2};{3};{4};{5}'.format(trig.id, ip, province, county, broadband, appid) time_pre = self.redis.get_data(flag) if not time_pre: self.redis.set_data(flag, time_now) return 'no_alert' get_time = datetime.datetime.strptime time_delta = get_time(time_now, '%Y-%m-%d %H:%M:%S') - get_time( time_pre, '%Y-%m-%d %H:%M:%S') if time_delta.seconds > RepeatInt[trig.repeat] * 60: self.redis.set_data(flag, time_now) return 'out_repeat' return 'in_repeat' ''' input: MonitorTrigger instance output: 之前处于报警状态: True 之前不处于报警状态: False ''' def cancel_alert(self, trig_id, ip, province, county, broadband, appid): flag = 'trig:{0};{1};{2};{3};{4};{5}'.format(trig_id, ip, province, county, broadband, appid) time_pre = self.redis.del_data(flag) if not time_pre: return False flag_cfm = 'cfm:{0};{1};{2};{3};{4};{5}'.format( trig_id, ip, province, county, broadband, appid) # 删除报警确认信息 self.redis.del_data(flag_cfm) self.redis.del_data(flag) return True @staticmethod def request_data_format(req_data): key_list = [ 'province', 'country', 'county', 'server', 'delay', 'loss_rate', 'time', 'broadband', 'counter', 'app_id', 'max_delay', 'min_delay', 'type', 'devices' ] ret_data = {k: v for k, v in req_data.iteritems() if k in key_list} if ret_data['broadband'] not in CarrierList: ret_data['broadband'] = u'其他'.encode('utf-8') return ret_data @except_handler() def post(self, request_data): req_data = MonitorAlertHandler.request_data_format(request_data) es_client.create_record(settings.opt['ELASTICSEARCH']['INDEX'], req_data) trigger_alert = self.alert_handler(req_data) for _trig in trigger_alert: logger.debug('{0}'.format(_trig)) for trig_l in trigger_alert: if trig_l['alert']: ''' 检查是否需要告警,报警邮件发送的两个条件: 用户未确认 在重复周期内 一个告警日志记录由: trigger.id、服务器IP、省份、县市、运营商、app_id,共同识别 ''' ret_alert = self.check_alert( trig_l['trigger'], req_data['server'], req_data['province'], req_data['county'], req_data['broadband'], req_data['app_id'], req_data['time']) if ret_alert == 'no_alert': ''' 之前无告警,需要保存告警信息到告警日志 ''' logger.debug('create_alert_log:{0}, {1}, {2}'.format( trig_l['trigger'].id, req_data['server'], req_data['time'])) content_detail = u'[{}]的线路出现告警,问题为:[{}异常]<br/>' \ u'告警策略详情:<br/>' \ u'指标:{},触发条件:{}{}{},统计周期:{},持续时间:{}<br/>' \ u'告警数值:<br/>' \ u'{}({})'.format( # req_data['time'], # trig_l['region'].name, # req_data['server'], # u'{0}, {1}'.format(req_data['province'], req_data['county']), req_data['broadband'], trig_l['trigger']._item.comment, # {6} # trig_l['trigger']._rule.name, trig_l['trigger']._item.comment, trig_l['trigger'].compare, trig_l['trigger'].threshold, trig_l['trigger']._item.unit, # {10} trig_l['trigger'].period, trig_l['trigger'].time, # {12} ['{:.2f}'.format(i) for i in trig_l['data']], trig_l['trigger']._item.unit, ) instance_ma = MonitorAlertLog( start_time=req_data['time'], trigger='{0}'.format(trig_l['trigger'].id), rule_name=trig_l['trigger']._rule.name, region=trig_l['region'].name, province=req_data['province'].encode('utf8'), county=req_data['county'].encode('utf8'), broadband=req_data['broadband'].encode('utf8'), appid=req_data['app_id'], serverip=req_data['server'], state='open', content=trig_l['trigger']._item.comment.encode('utf8'), content_detail=content_detail.encode('utf8'), ) instance_ma._save(self.db_session) elif ret_alert == 'in_repeat': continue else: pass ret_cfm = self.check_confirm(trig_l['trigger'].id, req_data['server'], req_data['province'], req_data['county'], req_data['broadband'], req_data['app_id']) if ret_cfm: continue message = u'[{0}],您的[{1}]集群下IP为[{2}]的服务器连接[{3}]地区[{4}]的线路出现告警,问题为:\r\n[{5}异常]\r\n' \ u'告警策略详情:\r\n' \ u'名称:{6}, 指标:{7},触发条件:{8}{9}{10},统计周期:{11},持续时间:{12}\r\n' \ u'当前数值:\r\n' \ u'{13}({10})\r\n' \ u'请及时处理,更多详细信息请至控制台查看。'.format( req_data['time'], trig_l['region'].name, req_data['server'], u'{0}, {1}'.format(req_data['province'], req_data['county']), req_data['broadband'], u'{0}'.format(trig_l['trigger']._item.comment), # {6} trig_l['trigger']._rule.name, trig_l['trigger']._item.comment, trig_l['trigger'].compare, trig_l['trigger'].threshold, trig_l['trigger']._item.unit, # {10} trig_l['trigger'].period, trig_l['trigger'].time, # {12} ['{:.2f}'.format(i) for i in trig_l['data']], ) # logger.debug('alert_mail:{0}, {1}, {2}'.format( # trig_l['trigger'].id, req_data['server'], req_data['time'])) for action in trig_l['trigger']._rule._actions: send_mail(action.mail, action.contacter, message) else: ''' 检查是否需要取消告警 ''' ret_alert = self.cancel_alert(trig_l['trigger'].id, req_data['server'], req_data['province'], req_data['county'], req_data['broadband'], req_data['app_id']) if ret_alert: ''' 更新告警日志,保存本次告警信息 发送取消告警消息 ''' al_records = self.db_session.query(MonitorAlertLog).filter( MonitorAlertLog.rule_name == trig_l['trigger']._rule.name, MonitorAlertLog.trigger == '{0}'.format( trig_l['trigger'].id), MonitorAlertLog.serverip == req_data['server'], MonitorAlertLog.province == req_data['province'].encode('utf8'), MonitorAlertLog.county == req_data['county'].encode( 'utf8'), MonitorAlertLog.broadband == req_data['broadband'].encode('utf8'), MonitorAlertLog.appid == req_data['app_id'], or_(MonitorAlertLog.state == 'open', MonitorAlertLog.state == 'confirm'), ).all() for al_record in al_records: al_record.state = 'close' al_record.stop_time = req_data['time'] al_record.content_detail = u'{}<br/>' \ u'线路恢复正常,告警取消。<br/>' \ u'当前数值:<br/>' \ u'{}({})'.format( al_record.content_detail, ['{:.2f}'.format(i) for i in trig_l['data']], trig_l['trigger']._item.unit, ).encode('utf8') al_record._save(self.db_session, update=True, commit=False) self.db_session.commit() message = u'[{0}],您的[{1}]集群下IP为[{2}]的服务器连接[{3}]地区[{4}]的线路恢复正常,原告警问题描述为:\r\n[{5}异常]\r\n' \ u'触发告警策略详情:\r\n' \ u'名称:{6}, 指标:{7},触发条件:{8}{9}{10},统计周期:{11},持续时间:{12}\r\n' \ u'当前数值:\r\n' \ u'{13}({10})\r\n' \ u'更多详细信息请至控制台查看。'.format( req_data['time'], trig_l['region'].name, req_data['server'], u'{0}, {1}'.format(req_data['province'], req_data['county']), req_data['broadband'], u'{0}'.format(trig_l['trigger']._item.comment), # {6} trig_l['trigger']._rule.name, trig_l['trigger']._item.comment, trig_l['trigger'].compare, trig_l['trigger'].threshold, trig_l['trigger']._item.unit, # {10} trig_l['trigger'].period, trig_l['trigger'].time, # {12} ['{:.2f}'.format(i) for i in trig_l['data']], ) # logger.debug('cansel_alert_mail:{0}, {1}, {2}'.format( # trig_l['trigger'].id, req_data['server'], req_data['time'])) for action in trig_l['trigger']._rule._actions: send_mail(action.mail, action.contacter, message) return {'return': 0}