def follow(sakuya_db): user = auth.get_user() chart_list = [] invalids = [] for row in sakuya_db.\ query(Follows).\ filter_by(follower=user['userid']).\ order_by(Follows.id.desc()): chart = sakuya_db.query(Charts).get(row.cid) if chart is None: invalids.append(row) continue if haopan.is_haopan(chart.cate_id): title = haopan.format_title(chart.name) else: title = chart.name chart_list.append({ 'id': chart.id, 'name': title, 'recv_warning': row.recv_rules == 'all' }) if invalids: for row in invalids: sakuya_db.delete(row) sakuya_db.commit() return {'chart_list': chart_list}
def follow(sakuya_db): user = auth.get_user() chart_list = [] invalids = [] for row in sakuya_db.\ query(Follows).\ filter_by(follower=user['userid']).\ order_by(Follows.id.desc()): chart = sakuya_db.query(Charts).get(row.cid) if chart is None: invalids.append(row) continue if haopan.is_haopan(chart.cate_id): title = haopan.format_title(chart.name) else: title = chart.name chart_list.append({ 'id': chart.id, 'name': title, 'recv_warning': row.recv_rules == 'all' }) if invalids: for row in invalids: sakuya_db.delete(row) sakuya_db.commit() return { 'chart_list': chart_list }
def ack(sakuya_db, id): def error(msg): return {'status': 'error', 'msg': msg} def log(chart, category): with open(get_config('webapp')['chart_ack_log'], 'a') as f: l = u'[%s] user: %s, chart: %d-%s, category: %d-%s\n' %\ (datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), auth.get_user()['username'], chart.id, chart.name, category.id, category.name) f.write(l.encode('utf-8')) chart = sakuya_db.query(Charts).get(id) if chart is None: return error('图表不存在') if haopan.is_haopan(chart.cate_id): return error('栏目不正确') if 'cl2' in request.params: # create category and move into it cl2 = sakuya_db.query(Categories).get(request.params['cl2']) if cl2 is None: return error('参数不正确') cl3_name = request.params.get('cl3') if not cl3_name: return error('栏目名称不能为空') cl3_exists = sakuya_db.query(Categories).\ filter_by(parent_cate_id=cl2.id, name=cl3_name).\ count() > 0 if cl3_exists: return error('栏目已存在') cl3 = Categories() cl3.name = cl3_name cl3.parent_cate_id = cl2.id cl3.is_parent = False sakuya_db.add(cl3) sakuya_db.commit() chart.cate_id = cl3.id chart.owner = auth.get_user()['username'] sakuya_db.commit() else: # move to category cl3 = sakuya_db.query(Categories).get(request.params.get('cl3')) if cl3 is None: return error('栏目不存在') chart.cate_id = cl3.id chart.owner = auth.get_user()['username'] sakuya_db.commit() log(chart, cl3) return {'status': 'ok'}
def get_charts(sakuya_db, cate1=None, cate2=None): lv2_list = OrderedDict() if cate1 is not None: for row in sakuya_db.\ query(Categories).\ filter_by(parent_cate_id=cate1).\ order_by(Categories.id): lv2_list[row.id] = { 'id': row.id, 'name': row.name } if cate2 is None: cate2 = lv2_list.keys() elif cate2 in lv2_list: cate2 = [cate2] else: cate2 = [] is_admin = auth.is_role('admin') if cate2: if is_admin: rows = sakuya_db.query(Charts).\ filter(Charts.cate_id.in_(cate2)).\ order_by(Charts.criticals.desc(), Charts.id) else: rows = sakuya_db.query(Charts).\ filter(Charts.cate_id.in_(cate2), Charts.status==1).\ order_by(Charts.criticals.desc(), Charts.id) else: if is_admin: rows = sakuya_db.query(Charts).\ filter_by(alert_enable=1).\ order_by(Charts.criticals.desc(), Charts.id) else: rows = sakuya_db.query(Charts).\ filter_by(alert_enable=1, status=1).\ order_by(Charts.criticals.desc(), Charts.id) result = [] for row in rows: if haopan.is_haopan(row.cate_id): title = haopan.format_title(row.name) else: title = row.name result.append({ 'id': row.id, 'name': title, 'critical': row.criticals > 0 }) return json.dumps(result)
def get_charts(sakuya_db, cate1=None, cate2=None): lv2_list = OrderedDict() if cate1 is not None: for row in sakuya_db.\ query(Categories).\ filter_by(parent_cate_id=cate1).\ order_by(Categories.id): lv2_list[row.id] = {'id': row.id, 'name': row.name} if cate2 is None: cate2 = lv2_list.keys() elif cate2 in lv2_list: cate2 = [cate2] else: cate2 = [] is_admin = auth.is_role('admin') if cate2: if is_admin: rows = sakuya_db.query(Charts).\ filter(Charts.cate_id.in_(cate2)).\ order_by(Charts.criticals.desc(), Charts.id) else: rows = sakuya_db.query(Charts).\ filter(Charts.cate_id.in_(cate2), Charts.status==1).\ order_by(Charts.criticals.desc(), Charts.id) else: if is_admin: rows = sakuya_db.query(Charts).\ filter_by(alert_enable=1).\ order_by(Charts.criticals.desc(), Charts.id) else: rows = sakuya_db.query(Charts).\ filter_by(alert_enable=1, status=1).\ order_by(Charts.criticals.desc(), Charts.id) result = [] for row in rows: if haopan.is_haopan(row.cate_id): title = haopan.format_title(row.name) else: title = row.name result.append({ 'id': row.id, 'name': title, 'critical': row.criticals > 0 }) return json.dumps(result)
def edit(sakuya_db, id): row = sakuya_db.query(Charts).get(id) if row is None: abort(404) is_haopan = haopan.is_haopan(row.cate_id) is_copy = request.params.get('copy') == '1' if auth.is_role('sa'): warn_only = False elif auth.is_role( 'haopan' ) and is_haopan: # this can be replaced with userid-cid authorization warn_only = True elif auth.get_user()['username'] == row.owner: warn_only = True else: abort(403) if is_copy and (is_haopan or warn_only): abort(400) rt = _rt(sakuya_db) rt['id'] = id rt['editing'] = not is_copy rt['warn_only'] = warn_only rt['is_haopan'] = is_haopan forms = { 'name': haopan.format_title(row.name) if is_haopan else row.name, 'cid': row.cate_id, 'owner': row.owner, 'alert_enable': row.alert_enable } ext = row.get_ext() forms['rule'] = dump_rule(ext.get('rule')) forms['warn'] = dump_warn(warn.get_warn(sakuya_db, id)) forms['owner_name'] = '系统' for item in rt['users']: if item['username'] == forms['owner']: forms['owner_name'] = item['truename'] rt['forms'] = forms return rt
def edit(sakuya_db, id): row = sakuya_db.query(Charts).get(id) if row is None: abort(404) is_haopan = haopan.is_haopan(row.cate_id) is_copy = request.params.get('copy') == '1' if auth.is_role('sa'): warn_only = False elif auth.is_role('haopan') and is_haopan: # this can be replaced with userid-cid authorization warn_only = True elif auth.get_user()['username'] == row.owner: warn_only = True else: abort(403) if is_copy and (is_haopan or warn_only): abort(400) rt = _rt(sakuya_db) rt['id'] = id rt['editing'] = not is_copy rt['warn_only'] = warn_only rt['is_haopan'] = is_haopan forms = { 'name': haopan.format_title(row.name) if is_haopan else row.name, 'cid': row.cate_id, 'owner': row.owner, 'alert_enable': row.alert_enable } ext = row.get_ext() forms['rule'] = dump_rule(ext.get('rule')) forms['warn'] = dump_warn(warn.get_warn(sakuya_db, id)) forms['owner_name'] = '系统' for item in rt['users']: if item['username'] == forms['owner']: forms['owner_name'] = item['truename'] rt['forms'] = forms return rt
def delete(sakuya_db, id): row = sakuya_db.query(Charts).get(id) if row is None: abort(404) if haopan.is_haopan(row.cate_id): abort(404) cid = row.cate_id sakuya_db.delete(row) sakuya_db.query(Follows).filter_by(cid=id).delete() sakuya_db.commit() process_warn_rules(sakuya_db, id, []) storm.delete_rule(id) return redirect('/admin/category/%d' % cid)
def compare_and_notify_one(chart): logging.debug('Checking chart %d.' % chart.id) # get rules rules = get_rules(chart.id) if not rules: logging.debug('No rule to check.') return # parse haopan title if haopan.is_haopan(chart.cate_id): chart.name = haopan.format_title(chart.name.encode('utf-8')).decode('utf-8') chart_level, chart_msg = (Events.CONST_TYPE_OK, None) rule_warn = OrderedDict() for rule in rules: # get current data end = check_time start = end - datetime.timedelta(minutes=rule['latency']) data = get_data(chart.id, start, end) if data: if rule['warn_type'] == 'HWM': level, msg = get_alert_level_hwm(chart.name, rule['hwm_warning'], rule['hwm_critical'], data) elif rule['warn_type'] == 'LWM': level, msg = get_alert_level_lwm(chart.name, rule['lwm_warning'], rule['lwm_critical'], data) elif rule['warn_type'] == 'RANGE': prev_end = end - datetime.timedelta(7) prev_start = start - datetime.timedelta(7) prev_data = get_data(chart.id, prev_start, prev_end) if not prev_data: prev_end = end - datetime.timedelta(1) prev_start = start - datetime.timedelta(1) prev_data = get_data(chart.id, prev_start, prev_end) level, msg = get_alert_level_range(chart.name, rule['hwm_warning'], rule['hwm_critical'], rule['lwm_warning'], rule['lwm_critical'], data, prev_data) else: if 'rule' in chart.get_ext(): no_data = True else: level = Events.CONST_TYPE_CRITICAL msg = u'%s 没有数据' % chart.name if level > chart_level: chart_level, chart_msg = level, msg rule_warn[rule['id']] = (level, msg, rule) # for chart if chart_level == Events.CONST_TYPE_OK: logging.debug('Chart %s' % LEVEL_TEXT[chart_level]) else: logging.debug('Chart %s' % ' - '.join((LEVEL_TEXT[chart_level], chart_msg))) event_exists = session.\ query(Events).\ filter_by(cid=chart.id).\ filter(Events.type >= chart_level).\ filter(Events.time > check_time - datetime.timedelta(seconds=NOTIFY_SCREEN_LATENCY)).\ count() > 0 if event_exists: logging.debug('Chart event exists (within %d minutes).' % (NOTIFY_SCREEN_LATENCY / 60)) else: add_events(chart.id, chart_level, chart_msg) update_chart_status(chart.id, chart_level) # per user for follow in session.\ query(Follows).\ filter_by(cid=chart.id, recv_warning=True): if follow.follower in skipped_user: continue try: user = session.query(Users).get(follow.follower) if user is None: raise ValueError('User not found') if not user.email: raise ValueError('User email is empty') email_list = util.unique(user.email.splitlines()) if not email_list: raise ValueError('User email is empty') except Exception, e: logging.warn('%s: %d' % (e, follow.follower)) skipped_user.add(follow.follower) continue recv_rules = follow.get_recv_rules() if recv_rules == 'all': recv_rules = rule_warn.keys() level, msg, rule = Events.CONST_TYPE_OK, None, None for rid in recv_rules: if rid not in rule_warn: continue if rule_warn[rid][0] > level: level, msg, rule = rule_warn[rid] if level == Events.CONST_TYPE_CRITICAL: logging.debug('User %s %s' % (user.truename, ' - '.join((LEVEL_TEXT[level], msg)))) if util.timestamp(check_time) - rule['last_critical'] <= rule['interval'] * 60: logging.debug('User event exists (within %d minutes).' % rule['interval']) else: update_rule_status(rule['id']) if user.id not in user_events: user_events[user.id] = { 'truename': user.truename, 'email_list': email_list, 'events': [] } user_events[user.id]['events'].append((chart, msg))
def update(sakuya_db, id): row = sakuya_db.query(Charts).get(id) if row is None: abort(404) is_haopan = haopan.is_haopan(row.cate_id) if auth.is_role('sa'): warn_only = False elif auth.is_role('haopan') and is_haopan: warn_only = True elif auth.get_user()['username'] == row.owner: warn_only = True else: abort(403) rule = None warn = None def error(msg): rt = _rt(sakuya_db) rt['id'] = id rt['editing'] = True rt['warn_only'] = warn_only rt['is_haopan'] = is_haopan rt['forms'] = request.forms rt['forms']['rule'] = dump_rule(rule) rt['forms']['warn'] = dump_warn(warn) rt['error_msg'] = msg return rt try: alert_enable = request.forms['alert_enable'] == '1' warn = make_warn() if alert_enable and not warn: raise ValueError if not warn_only and not is_haopan: use_storm = request.forms['use_storm'] == '1' if use_storm: rule = make_rule() if not rule['datasource'] or not rule['filters']: raise ValueError name = request.forms['name'] if not name: raise ValueError cid = int(request.forms['cid']) if not sakuya_db.\ query(Categories).\ filter_by(id=cid, is_parent=False).\ count(): raise ValueError owner = request.forms['owner'] if not sakuya_db.\ query(Users).\ filter_by(username=owner).\ count(): raise ValueError except Exception: return error('参数错误。') try: row.alert_enable = int(alert_enable) row.warnings = 0 row.criticals = 0 process_warn_rules(sakuya_db, id, warn) if not warn_only and not is_haopan: row.name = name row.cate_id = cid row.root_category = get_parent_category(sakuya_db, cid) row.owner = owner ext = row.get_ext() if use_storm: ext['rule'] = rule storm.set_rule(id, json.dumps(rule)) else: ext.pop('rule', None) storm.delete_rule(id) row.ext_info = json.dumps(ext) sakuya_db.commit() except Exception: traceback.print_exc() sakuya_db.rollback() return error('图表更新失败。') if warn_only or is_haopan: redirect('/chart/%d' % id) redirect('/admin/category/%d' % cid)
def chart(sakuya_db, id, id2=None): chart_info = sakuya_db.query(Charts).get(id) if chart_info is None: abort(404) chart_info2 =[] if id2: chart_info2 = sakuya_db.query(Charts).get(id2) title = chart_info.name is_haopan = haopan.is_haopan(chart_info.cate_id) rt = { 'id': str(id), 'is_haopan': is_haopan, 'createtime': chart_info.createtime.strftime('%Y-%m-%d %H:%M:%S') } rt['is_admin'] = auth.is_role('admin') rt['is_admin'] = True rt['status'] = chart_info.status ''' more chart line ''' if is_haopan is False: if chart_info2: is_haopan = True title = chart_info2.name rt['is_haopan'] = True rt['t1_title'] = u'当日'+chart_info.name rt['t2_title'] = u'当日'+chart_info2.name rt['t3_title'] = u'上周'+chart_info.name rt['t4_title'] = u'上周'+chart_info2.name else: title=haopan.format_title(chart_info.name) rt['t1_title'] = u'当日有效点击' rt['t2_title'] = u'当日无效点击' rt['t3_title'] = u'上周有效点击' rt['t4_title'] = u'上周无效点击' id2 = haopan.get_id2(chart_info.id) if is_haopan: rt['title'] = title#haopan.format_title(chart_info.name) rt['id2'] = id2#haopan.get_id2(chart_info.id) rt['owner'] = '系统' else: rt['title'] = title # owner owner = sakuya_db.query(Users).filter_by(username=chart_info.owner).first() if owner is not None: rt['owner'] = owner.truename else: rt['owner'] = '未知' # datasource try: ext_info = json.loads(chart_info.ext_info) if 'rule' in ext_info: ruc = ext_info['rule'] storm_info = { 'datasource': ruc['datasource'], 'rule_type': storm.rule_types[ruc['rule_type']][0] } if storm.rule_types[ruc['rule_type']][1]: storm_info['field'] = ruc['field'] storm_info['filters'] = [] for item in ruc['filters']: storm_info['filters'].append({ 'field': item[0], 'operator': storm.operators[storm.datasources[ruc['datasource']][item[0]][0]][0][item[1]], 'negative': item[2], 'content': item[3] }) rt['storm_info'] = storm_info elif chart_info.api_ip and chart_info.api_ts: rt['api_ip'] = util.long2ip(chart_info.api_ip) rt['api_ts'] = datetime.datetime.fromtimestamp(chart_info.api_ts).strftime('%Y-%m-%d %H:%M:%S') except Exception, e: pass
def chart(sakuya_db, id, id2=None): chart_info = sakuya_db.query(Charts).get(id) if chart_info is None: abort(404) chart_info2 = [] if id2: chart_info2 = sakuya_db.query(Charts).get(id2) title = chart_info.name is_haopan = haopan.is_haopan(chart_info.cate_id) rt = { 'id': str(id), 'is_haopan': is_haopan, 'createtime': chart_info.createtime.strftime('%Y-%m-%d %H:%M:%S') } rt['is_admin'] = auth.is_role('admin') rt['is_admin'] = True rt['status'] = chart_info.status ''' more chart line ''' if is_haopan is False: if chart_info2: is_haopan = True title = chart_info2.name rt['is_haopan'] = True rt['t1_title'] = u'当日' + chart_info.name rt['t2_title'] = u'当日' + chart_info2.name rt['t3_title'] = u'上周' + chart_info.name rt['t4_title'] = u'上周' + chart_info2.name else: title = haopan.format_title(chart_info.name) rt['t1_title'] = u'当日有效点击' rt['t2_title'] = u'当日无效点击' rt['t3_title'] = u'上周有效点击' rt['t4_title'] = u'上周无效点击' id2 = haopan.get_id2(chart_info.id) if is_haopan: rt['title'] = title #haopan.format_title(chart_info.name) rt['id2'] = id2 #haopan.get_id2(chart_info.id) rt['owner'] = '系统' else: rt['title'] = title # owner owner = sakuya_db.query(Users).filter_by( username=chart_info.owner).first() if owner is not None: rt['owner'] = owner.truename else: rt['owner'] = '未知' # datasource try: ext_info = json.loads(chart_info.ext_info) if 'rule' in ext_info: ruc = ext_info['rule'] storm_info = { 'datasource': ruc['datasource'], 'rule_type': storm.rule_types[ruc['rule_type']][0] } if storm.rule_types[ruc['rule_type']][1]: storm_info['field'] = ruc['field'] storm_info['filters'] = [] for item in ruc['filters']: storm_info['filters'].append({ 'field': item[0], 'operator': storm.operators[storm.datasources[ruc['datasource']][ item[0]][0]][0][item[1]], 'negative': item[2], 'content': item[3] }) rt['storm_info'] = storm_info elif chart_info.api_ip and chart_info.api_ts: rt['api_ip'] = util.long2ip(chart_info.api_ip) rt['api_ts'] = datetime.datetime.fromtimestamp( chart_info.api_ts).strftime('%Y-%m-%d %H:%M:%S') except Exception, e: pass
def compare_and_notify_one(chart): logging.debug("Checking chart %d." % chart.id) # get rules rules = get_rules(chart.id) if not rules: logging.debug("No rule to check.") return # parse haopan title if haopan.is_haopan(chart.cate_id): chart.name = haopan.format_title(chart.name.encode("utf-8")).decode("utf-8") chart_level, chart_msg = (Events.CONST_TYPE_OK, None) rule_warn = OrderedDict() for rule in rules: # get current data end = check_time start = end - datetime.timedelta(minutes=rule["latency"]) data = get_data(chart.id, start, end) if data: if rule["warn_type"] == "HWM": level, msg = get_alert_level_hwm(chart.name, rule["hwm_warning"], rule["hwm_critical"], data) elif rule["warn_type"] == "LWM": level, msg = get_alert_level_lwm(chart.name, rule["lwm_warning"], rule["lwm_critical"], data) elif rule["warn_type"] == "RANGE": prev_end = end - datetime.timedelta(7) prev_start = start - datetime.timedelta(7) prev_data = get_data(chart.id, prev_start, prev_end) if not prev_data: prev_end = end - datetime.timedelta(1) prev_start = start - datetime.timedelta(1) prev_data = get_data(chart.id, prev_start, prev_end) level, msg = get_alert_level_range( chart.name, rule["hwm_warning"], rule["hwm_critical"], rule["lwm_warning"], rule["lwm_critical"], data, prev_data, ) else: if "rule" in chart.get_ext(): no_data = True else: level = Events.CONST_TYPE_CRITICAL msg = u"%s 没有数据" % chart.name if level > chart_level: chart_level, chart_msg = level, msg rule_warn[rule["id"]] = (level, msg, rule) # for chart if chart_level == Events.CONST_TYPE_OK: logging.debug("Chart %s" % LEVEL_TEXT[chart_level]) else: logging.debug("Chart %s" % " - ".join((LEVEL_TEXT[chart_level], chart_msg))) event_exists = ( session.query(Events) .filter_by(cid=chart.id) .filter(Events.type >= chart_level) .filter(Events.time > check_time - datetime.timedelta(seconds=NOTIFY_SCREEN_LATENCY)) .count() > 0 ) if event_exists: logging.debug("Chart event exists (within %d minutes)." % (NOTIFY_SCREEN_LATENCY / 60)) else: add_events(chart.id, chart_level, chart_msg) update_chart_status(chart.id, chart_level) # per user for follow in session.query(Follows).filter_by(cid=chart.id, recv_warning=True): if follow.follower in skipped_user: continue try: user = session.query(Users).get(follow.follower) if user is None: raise ValueError("User not found") if not user.email: raise ValueError("User email is empty") email_list = util.unique(user.email.splitlines()) if not email_list: raise ValueError("User email is empty") except Exception, e: logging.warn("%s: %d" % (e, follow.follower)) skipped_user.add(follow.follower) continue recv_rules = follow.get_recv_rules() if recv_rules == "all": recv_rules = rule_warn.keys() level, msg, rule = Events.CONST_TYPE_OK, None, None for rid in recv_rules: if rid not in rule_warn: continue if rule_warn[rid][0] > level: level, msg, rule = rule_warn[rid] if level == Events.CONST_TYPE_CRITICAL: logging.debug("User %s %s" % (user.truename, " - ".join((LEVEL_TEXT[level], msg)))) if util.timestamp(check_time) - rule["last_critical"] <= rule["interval"] * 60: logging.debug("User event exists (within %d minutes)." % rule["interval"]) else: update_rule_status(rule["id"]) if user.id not in user_events: user_events[user.id] = {"truename": user.truename, "email_list": email_list, "events": []} user_events[user.id]["events"].append((chart, msg))