def main(): chart_count = load_file('/home/jizhang/knowing.log/chart.log') dt_count = dict((id, count[0]) for id, count in chart_count.iteritems()) vp_count = dict((id, count[1]) for id, count in chart_count.iteritems()) event_count = load_file('/home/jizhang/knowing.log/event.log') follow_count = load_file('/home/jizhang/knowing.log/follow.log') warning_count = load_file('/home/jizhang/knowing.log/warning.log') print 'ID\t栏目\t标题\t创建人\t单页访问次数\t数据调用次数\t报警开启\t报警次数\t订阅人数\t接警人数' for chart in session.query(Charts): category = get_category(chart.cate_id) title = chart.name.encode('utf-8') if not category or category.startswith('Solr') or category.startswith('Suite'): continue if category.startswith('Haopan'): category = '监控 / 二手房 / 好盘' title = haopan.format_title(title) if title.startswith('hp'): continue if chart.alert_enable: alert_enable = 1 else: alert_enable = 0 info = [chart.id, category, title, get_truename(chart.owner), vp_count.get(chart.id, 0), dt_count.get(chart.id, 0), alert_enable, event_count.get(chart.id, 0), follow_count.get(chart.id, 0), warning_count.get(chart.id, 0)] print '\t'.join((str(s) for s in info))
def follow(sakuya_db): user = auth.get_user() chart_list = [] invalids = [] for row in sakuya_db.\ query(Follows).\ filter_by(follower=user['userid']).\ order_by(Follows.id.desc()): chart = sakuya_db.query(Charts).get(row.cid) if chart is None: invalids.append(row) continue if haopan.is_haopan(chart.cate_id): title = haopan.format_title(chart.name) else: title = chart.name chart_list.append({ 'id': chart.id, 'name': title, 'recv_warning': row.recv_rules == 'all' }) if invalids: for row in invalids: sakuya_db.delete(row) sakuya_db.commit() return {'chart_list': chart_list}
def haopan_category(): for chart in session.query(Charts).filter_by(cate_id=100): title = haopan.format_title(chart.name) if title.startswith('hp-'): continue line = u'监控\t二手房\t好盘\t%s\t%s' % (title.decode('utf-8'), get_receivers(chart.id)) print line.encode('utf-8')
def follow(sakuya_db): user = auth.get_user() chart_list = [] invalids = [] for row in sakuya_db.\ query(Follows).\ filter_by(follower=user['userid']).\ order_by(Follows.id.desc()): chart = sakuya_db.query(Charts).get(row.cid) if chart is None: invalids.append(row) continue if haopan.is_haopan(chart.cate_id): title = haopan.format_title(chart.name) else: title = chart.name chart_list.append({ 'id': chart.id, 'name': title, 'recv_warning': row.recv_rules == 'all' }) if invalids: for row in invalids: sakuya_db.delete(row) sakuya_db.commit() return { 'chart_list': chart_list }
def get_charts(sakuya_db, cate1=None, cate2=None): lv2_list = OrderedDict() if cate1 is not None: for row in sakuya_db.\ query(Categories).\ filter_by(parent_cate_id=cate1).\ order_by(Categories.id): lv2_list[row.id] = { 'id': row.id, 'name': row.name } if cate2 is None: cate2 = lv2_list.keys() elif cate2 in lv2_list: cate2 = [cate2] else: cate2 = [] is_admin = auth.is_role('admin') if cate2: if is_admin: rows = sakuya_db.query(Charts).\ filter(Charts.cate_id.in_(cate2)).\ order_by(Charts.criticals.desc(), Charts.id) else: rows = sakuya_db.query(Charts).\ filter(Charts.cate_id.in_(cate2), Charts.status==1).\ order_by(Charts.criticals.desc(), Charts.id) else: if is_admin: rows = sakuya_db.query(Charts).\ filter_by(alert_enable=1).\ order_by(Charts.criticals.desc(), Charts.id) else: rows = sakuya_db.query(Charts).\ filter_by(alert_enable=1, status=1).\ order_by(Charts.criticals.desc(), Charts.id) result = [] for row in rows: if haopan.is_haopan(row.cate_id): title = haopan.format_title(row.name) else: title = row.name result.append({ 'id': row.id, 'name': title, 'critical': row.criticals > 0 }) return json.dumps(result)
def get_charts(sakuya_db, cate1=None, cate2=None): lv2_list = OrderedDict() if cate1 is not None: for row in sakuya_db.\ query(Categories).\ filter_by(parent_cate_id=cate1).\ order_by(Categories.id): lv2_list[row.id] = {'id': row.id, 'name': row.name} if cate2 is None: cate2 = lv2_list.keys() elif cate2 in lv2_list: cate2 = [cate2] else: cate2 = [] is_admin = auth.is_role('admin') if cate2: if is_admin: rows = sakuya_db.query(Charts).\ filter(Charts.cate_id.in_(cate2)).\ order_by(Charts.criticals.desc(), Charts.id) else: rows = sakuya_db.query(Charts).\ filter(Charts.cate_id.in_(cate2), Charts.status==1).\ order_by(Charts.criticals.desc(), Charts.id) else: if is_admin: rows = sakuya_db.query(Charts).\ filter_by(alert_enable=1).\ order_by(Charts.criticals.desc(), Charts.id) else: rows = sakuya_db.query(Charts).\ filter_by(alert_enable=1, status=1).\ order_by(Charts.criticals.desc(), Charts.id) result = [] for row in rows: if haopan.is_haopan(row.cate_id): title = haopan.format_title(row.name) else: title = row.name result.append({ 'id': row.id, 'name': title, 'critical': row.criticals > 0 }) return json.dumps(result)
def edit(sakuya_db, id): row = sakuya_db.query(Charts).get(id) if row is None: abort(404) is_haopan = haopan.is_haopan(row.cate_id) is_copy = request.params.get('copy') == '1' if auth.is_role('sa'): warn_only = False elif auth.is_role( 'haopan' ) and is_haopan: # this can be replaced with userid-cid authorization warn_only = True elif auth.get_user()['username'] == row.owner: warn_only = True else: abort(403) if is_copy and (is_haopan or warn_only): abort(400) rt = _rt(sakuya_db) rt['id'] = id rt['editing'] = not is_copy rt['warn_only'] = warn_only rt['is_haopan'] = is_haopan forms = { 'name': haopan.format_title(row.name) if is_haopan else row.name, 'cid': row.cate_id, 'owner': row.owner, 'alert_enable': row.alert_enable } ext = row.get_ext() forms['rule'] = dump_rule(ext.get('rule')) forms['warn'] = dump_warn(warn.get_warn(sakuya_db, id)) forms['owner_name'] = '系统' for item in rt['users']: if item['username'] == forms['owner']: forms['owner_name'] = item['truename'] rt['forms'] = forms return rt
def edit(sakuya_db, id): row = sakuya_db.query(Charts).get(id) if row is None: abort(404) is_haopan = haopan.is_haopan(row.cate_id) is_copy = request.params.get('copy') == '1' if auth.is_role('sa'): warn_only = False elif auth.is_role('haopan') and is_haopan: # this can be replaced with userid-cid authorization warn_only = True elif auth.get_user()['username'] == row.owner: warn_only = True else: abort(403) if is_copy and (is_haopan or warn_only): abort(400) rt = _rt(sakuya_db) rt['id'] = id rt['editing'] = not is_copy rt['warn_only'] = warn_only rt['is_haopan'] = is_haopan forms = { 'name': haopan.format_title(row.name) if is_haopan else row.name, 'cid': row.cate_id, 'owner': row.owner, 'alert_enable': row.alert_enable } ext = row.get_ext() forms['rule'] = dump_rule(ext.get('rule')) forms['warn'] = dump_warn(warn.get_warn(sakuya_db, id)) forms['owner_name'] = '系统' for item in rt['users']: if item['username'] == forms['owner']: forms['owner_name'] = item['truename'] rt['forms'] = forms return rt
def compare_and_notify_one(chart): logging.debug('Checking chart %d.' % chart.id) # get rules rules = get_rules(chart.id) if not rules: logging.debug('No rule to check.') return # parse haopan title if haopan.is_haopan(chart.cate_id): chart.name = haopan.format_title(chart.name.encode('utf-8')).decode('utf-8') chart_level, chart_msg = (Events.CONST_TYPE_OK, None) rule_warn = OrderedDict() for rule in rules: # get current data end = check_time start = end - datetime.timedelta(minutes=rule['latency']) data = get_data(chart.id, start, end) if data: if rule['warn_type'] == 'HWM': level, msg = get_alert_level_hwm(chart.name, rule['hwm_warning'], rule['hwm_critical'], data) elif rule['warn_type'] == 'LWM': level, msg = get_alert_level_lwm(chart.name, rule['lwm_warning'], rule['lwm_critical'], data) elif rule['warn_type'] == 'RANGE': prev_end = end - datetime.timedelta(7) prev_start = start - datetime.timedelta(7) prev_data = get_data(chart.id, prev_start, prev_end) if not prev_data: prev_end = end - datetime.timedelta(1) prev_start = start - datetime.timedelta(1) prev_data = get_data(chart.id, prev_start, prev_end) level, msg = get_alert_level_range(chart.name, rule['hwm_warning'], rule['hwm_critical'], rule['lwm_warning'], rule['lwm_critical'], data, prev_data) else: if 'rule' in chart.get_ext(): no_data = True else: level = Events.CONST_TYPE_CRITICAL msg = u'%s 没有数据' % chart.name if level > chart_level: chart_level, chart_msg = level, msg rule_warn[rule['id']] = (level, msg, rule) # for chart if chart_level == Events.CONST_TYPE_OK: logging.debug('Chart %s' % LEVEL_TEXT[chart_level]) else: logging.debug('Chart %s' % ' - '.join((LEVEL_TEXT[chart_level], chart_msg))) event_exists = session.\ query(Events).\ filter_by(cid=chart.id).\ filter(Events.type >= chart_level).\ filter(Events.time > check_time - datetime.timedelta(seconds=NOTIFY_SCREEN_LATENCY)).\ count() > 0 if event_exists: logging.debug('Chart event exists (within %d minutes).' % (NOTIFY_SCREEN_LATENCY / 60)) else: add_events(chart.id, chart_level, chart_msg) update_chart_status(chart.id, chart_level) # per user for follow in session.\ query(Follows).\ filter_by(cid=chart.id, recv_warning=True): if follow.follower in skipped_user: continue try: user = session.query(Users).get(follow.follower) if user is None: raise ValueError('User not found') if not user.email: raise ValueError('User email is empty') email_list = util.unique(user.email.splitlines()) if not email_list: raise ValueError('User email is empty') except Exception, e: logging.warn('%s: %d' % (e, follow.follower)) skipped_user.add(follow.follower) continue recv_rules = follow.get_recv_rules() if recv_rules == 'all': recv_rules = rule_warn.keys() level, msg, rule = Events.CONST_TYPE_OK, None, None for rid in recv_rules: if rid not in rule_warn: continue if rule_warn[rid][0] > level: level, msg, rule = rule_warn[rid] if level == Events.CONST_TYPE_CRITICAL: logging.debug('User %s %s' % (user.truename, ' - '.join((LEVEL_TEXT[level], msg)))) if util.timestamp(check_time) - rule['last_critical'] <= rule['interval'] * 60: logging.debug('User event exists (within %d minutes).' % rule['interval']) else: update_rule_status(rule['id']) if user.id not in user_events: user_events[user.id] = { 'truename': user.truename, 'email_list': email_list, 'events': [] } user_events[user.id]['events'].append((chart, msg))
def chart(sakuya_db, id, id2=None): chart_info = sakuya_db.query(Charts).get(id) if chart_info is None: abort(404) chart_info2 =[] if id2: chart_info2 = sakuya_db.query(Charts).get(id2) title = chart_info.name is_haopan = haopan.is_haopan(chart_info.cate_id) rt = { 'id': str(id), 'is_haopan': is_haopan, 'createtime': chart_info.createtime.strftime('%Y-%m-%d %H:%M:%S') } rt['is_admin'] = auth.is_role('admin') rt['is_admin'] = True rt['status'] = chart_info.status ''' more chart line ''' if is_haopan is False: if chart_info2: is_haopan = True title = chart_info2.name rt['is_haopan'] = True rt['t1_title'] = u'当日'+chart_info.name rt['t2_title'] = u'当日'+chart_info2.name rt['t3_title'] = u'上周'+chart_info.name rt['t4_title'] = u'上周'+chart_info2.name else: title=haopan.format_title(chart_info.name) rt['t1_title'] = u'当日有效点击' rt['t2_title'] = u'当日无效点击' rt['t3_title'] = u'上周有效点击' rt['t4_title'] = u'上周无效点击' id2 = haopan.get_id2(chart_info.id) if is_haopan: rt['title'] = title#haopan.format_title(chart_info.name) rt['id2'] = id2#haopan.get_id2(chart_info.id) rt['owner'] = '系统' else: rt['title'] = title # owner owner = sakuya_db.query(Users).filter_by(username=chart_info.owner).first() if owner is not None: rt['owner'] = owner.truename else: rt['owner'] = '未知' # datasource try: ext_info = json.loads(chart_info.ext_info) if 'rule' in ext_info: ruc = ext_info['rule'] storm_info = { 'datasource': ruc['datasource'], 'rule_type': storm.rule_types[ruc['rule_type']][0] } if storm.rule_types[ruc['rule_type']][1]: storm_info['field'] = ruc['field'] storm_info['filters'] = [] for item in ruc['filters']: storm_info['filters'].append({ 'field': item[0], 'operator': storm.operators[storm.datasources[ruc['datasource']][item[0]][0]][0][item[1]], 'negative': item[2], 'content': item[3] }) rt['storm_info'] = storm_info elif chart_info.api_ip and chart_info.api_ts: rt['api_ip'] = util.long2ip(chart_info.api_ip) rt['api_ts'] = datetime.datetime.fromtimestamp(chart_info.api_ts).strftime('%Y-%m-%d %H:%M:%S') except Exception, e: pass
def index(sakuya_db): rt = {} rt['stats'] = { 'charts': sakuya_db.query(Charts).count(), 'users': sakuya_db.query(Users).count(), 'follows': sakuya_db.query(Follows).count() } rt['events'] = [] for row in sakuya_db.\ query(Events).\ order_by(Events.id.desc()).\ limit(7): if row.type == Events.CONST_TYPE_CRITICAL: type_text = 'Critical' else: type_text = 'Warning' rt['events'].append({ 'cid': row.cid, 'info': row.info, 'time': row.time.strftime('%H:%M'), 'type': type_text }) rt['top_monitor'] = [] for row in sakuya_db.\ query(Charts).\ filter_by(root_category=2).\ order_by(Charts.followers.desc(), Charts.id).\ limit(7): rt['top_monitor'].append({ 'id': row.id, 'name': haopan.format_title(row.name), 'num': row.followers }) rt['top_speed'] = [] for row in sakuya_db.\ query(Charts).\ filter_by(root_category=3).\ order_by(Charts.followers.desc(), Charts.id).\ limit(7): rt['top_speed'].append({ 'id': row.id, 'name': row.name, 'num': row.followers }) user = auth.get_user() if user is None: rt['login_info'] = auth.get_login_info() else: rt['my_follows'] = [] for row in sakuya_db.\ query(Follows).\ filter_by(follower=user['userid']).\ limit(8): chart = sakuya_db.query(Charts).get(row.cid) if chart is None: continue rt['my_follows'].append({ 'id': chart.id, 'name': haopan.format_title(chart.name) }) rt['my_charts'] = [] for row in sakuya_db.\ query(Charts).\ filter_by(owner=user['username']).\ limit(8): rt['my_charts'].append({ 'id': row.id, 'name': row.name }) return rt
def index(sakuya_db): rt = {} rt['stats'] = { 'charts': sakuya_db.query(Charts).count(), 'users': sakuya_db.query(Users).count(), 'follows': sakuya_db.query(Follows).count() } rt['events'] = [] for row in sakuya_db.\ query(Events).\ order_by(Events.id.desc()).\ limit(7): if row.type == Events.CONST_TYPE_CRITICAL: type_text = 'Critical' else: type_text = 'Warning' rt['events'].append({ 'cid': row.cid, 'info': row.info, 'time': row.time.strftime('%H:%M'), 'type': type_text }) rt['top_monitor'] = [] for row in sakuya_db.\ query(Charts).\ filter_by(root_category=2).\ order_by(Charts.followers.desc(), Charts.id).\ limit(7): rt['top_monitor'].append({ 'id': row.id, 'name': haopan.format_title(row.name), 'num': row.followers }) rt['top_speed'] = [] for row in sakuya_db.\ query(Charts).\ filter_by(root_category=3).\ order_by(Charts.followers.desc(), Charts.id).\ limit(7): rt['top_speed'].append({ 'id': row.id, 'name': row.name, 'num': row.followers }) user = auth.get_user() if user is None: rt['login_info'] = auth.get_login_info() else: rt['my_follows'] = [] for row in sakuya_db.\ query(Follows).\ filter_by(follower=user['userid']).\ limit(8): chart = sakuya_db.query(Charts).get(row.cid) if chart is None: continue rt['my_follows'].append({ 'id': chart.id, 'name': haopan.format_title(chart.name) }) rt['my_charts'] = [] for row in sakuya_db.\ query(Charts).\ filter_by(owner=user['username']).\ limit(8): rt['my_charts'].append({'id': row.id, 'name': row.name}) return rt
def chart(sakuya_db, id, id2=None): chart_info = sakuya_db.query(Charts).get(id) if chart_info is None: abort(404) chart_info2 = [] if id2: chart_info2 = sakuya_db.query(Charts).get(id2) title = chart_info.name is_haopan = haopan.is_haopan(chart_info.cate_id) rt = { 'id': str(id), 'is_haopan': is_haopan, 'createtime': chart_info.createtime.strftime('%Y-%m-%d %H:%M:%S') } rt['is_admin'] = auth.is_role('admin') rt['is_admin'] = True rt['status'] = chart_info.status ''' more chart line ''' if is_haopan is False: if chart_info2: is_haopan = True title = chart_info2.name rt['is_haopan'] = True rt['t1_title'] = u'当日' + chart_info.name rt['t2_title'] = u'当日' + chart_info2.name rt['t3_title'] = u'上周' + chart_info.name rt['t4_title'] = u'上周' + chart_info2.name else: title = haopan.format_title(chart_info.name) rt['t1_title'] = u'当日有效点击' rt['t2_title'] = u'当日无效点击' rt['t3_title'] = u'上周有效点击' rt['t4_title'] = u'上周无效点击' id2 = haopan.get_id2(chart_info.id) if is_haopan: rt['title'] = title #haopan.format_title(chart_info.name) rt['id2'] = id2 #haopan.get_id2(chart_info.id) rt['owner'] = '系统' else: rt['title'] = title # owner owner = sakuya_db.query(Users).filter_by( username=chart_info.owner).first() if owner is not None: rt['owner'] = owner.truename else: rt['owner'] = '未知' # datasource try: ext_info = json.loads(chart_info.ext_info) if 'rule' in ext_info: ruc = ext_info['rule'] storm_info = { 'datasource': ruc['datasource'], 'rule_type': storm.rule_types[ruc['rule_type']][0] } if storm.rule_types[ruc['rule_type']][1]: storm_info['field'] = ruc['field'] storm_info['filters'] = [] for item in ruc['filters']: storm_info['filters'].append({ 'field': item[0], 'operator': storm.operators[storm.datasources[ruc['datasource']][ item[0]][0]][0][item[1]], 'negative': item[2], 'content': item[3] }) rt['storm_info'] = storm_info elif chart_info.api_ip and chart_info.api_ts: rt['api_ip'] = util.long2ip(chart_info.api_ip) rt['api_ts'] = datetime.datetime.fromtimestamp( chart_info.api_ts).strftime('%Y-%m-%d %H:%M:%S') except Exception, e: pass
def compare_and_notify_one(chart): logging.debug("Checking chart %d." % chart.id) # get rules rules = get_rules(chart.id) if not rules: logging.debug("No rule to check.") return # parse haopan title if haopan.is_haopan(chart.cate_id): chart.name = haopan.format_title(chart.name.encode("utf-8")).decode("utf-8") chart_level, chart_msg = (Events.CONST_TYPE_OK, None) rule_warn = OrderedDict() for rule in rules: # get current data end = check_time start = end - datetime.timedelta(minutes=rule["latency"]) data = get_data(chart.id, start, end) if data: if rule["warn_type"] == "HWM": level, msg = get_alert_level_hwm(chart.name, rule["hwm_warning"], rule["hwm_critical"], data) elif rule["warn_type"] == "LWM": level, msg = get_alert_level_lwm(chart.name, rule["lwm_warning"], rule["lwm_critical"], data) elif rule["warn_type"] == "RANGE": prev_end = end - datetime.timedelta(7) prev_start = start - datetime.timedelta(7) prev_data = get_data(chart.id, prev_start, prev_end) if not prev_data: prev_end = end - datetime.timedelta(1) prev_start = start - datetime.timedelta(1) prev_data = get_data(chart.id, prev_start, prev_end) level, msg = get_alert_level_range( chart.name, rule["hwm_warning"], rule["hwm_critical"], rule["lwm_warning"], rule["lwm_critical"], data, prev_data, ) else: if "rule" in chart.get_ext(): no_data = True else: level = Events.CONST_TYPE_CRITICAL msg = u"%s 没有数据" % chart.name if level > chart_level: chart_level, chart_msg = level, msg rule_warn[rule["id"]] = (level, msg, rule) # for chart if chart_level == Events.CONST_TYPE_OK: logging.debug("Chart %s" % LEVEL_TEXT[chart_level]) else: logging.debug("Chart %s" % " - ".join((LEVEL_TEXT[chart_level], chart_msg))) event_exists = ( session.query(Events) .filter_by(cid=chart.id) .filter(Events.type >= chart_level) .filter(Events.time > check_time - datetime.timedelta(seconds=NOTIFY_SCREEN_LATENCY)) .count() > 0 ) if event_exists: logging.debug("Chart event exists (within %d minutes)." % (NOTIFY_SCREEN_LATENCY / 60)) else: add_events(chart.id, chart_level, chart_msg) update_chart_status(chart.id, chart_level) # per user for follow in session.query(Follows).filter_by(cid=chart.id, recv_warning=True): if follow.follower in skipped_user: continue try: user = session.query(Users).get(follow.follower) if user is None: raise ValueError("User not found") if not user.email: raise ValueError("User email is empty") email_list = util.unique(user.email.splitlines()) if not email_list: raise ValueError("User email is empty") except Exception, e: logging.warn("%s: %d" % (e, follow.follower)) skipped_user.add(follow.follower) continue recv_rules = follow.get_recv_rules() if recv_rules == "all": recv_rules = rule_warn.keys() level, msg, rule = Events.CONST_TYPE_OK, None, None for rid in recv_rules: if rid not in rule_warn: continue if rule_warn[rid][0] > level: level, msg, rule = rule_warn[rid] if level == Events.CONST_TYPE_CRITICAL: logging.debug("User %s %s" % (user.truename, " - ".join((LEVEL_TEXT[level], msg)))) if util.timestamp(check_time) - rule["last_critical"] <= rule["interval"] * 60: logging.debug("User event exists (within %d minutes)." % rule["interval"]) else: update_rule_status(rule["id"]) if user.id not in user_events: user_events[user.id] = {"truename": user.truename, "email_list": email_list, "events": []} user_events[user.id]["events"].append((chart, msg))