Example #1
0
def main():

    chart_count = load_file('/home/jizhang/knowing.log/chart.log')
    dt_count = dict((id, count[0]) for id, count in chart_count.iteritems())
    vp_count = dict((id, count[1]) for id, count in chart_count.iteritems())

    event_count = load_file('/home/jizhang/knowing.log/event.log')
    follow_count = load_file('/home/jizhang/knowing.log/follow.log')
    warning_count = load_file('/home/jizhang/knowing.log/warning.log')

    print 'ID\t栏目\t标题\t创建人\t单页访问次数\t数据调用次数\t报警开启\t报警次数\t订阅人数\t接警人数'
    for chart in session.query(Charts):
        category = get_category(chart.cate_id)
        title = chart.name.encode('utf-8')
        if not category or category.startswith('Solr') or category.startswith('Suite'):
            continue
        if category.startswith('Haopan'):
            category = '监控 / 二手房 / 好盘'
            title = haopan.format_title(title)
            if title.startswith('hp'):
                continue

        if chart.alert_enable:
            alert_enable = 1
        else:
            alert_enable = 0

        info = [chart.id, category, title, get_truename(chart.owner),
                vp_count.get(chart.id, 0), dt_count.get(chart.id, 0),
                alert_enable, event_count.get(chart.id, 0),
                follow_count.get(chart.id, 0), warning_count.get(chart.id, 0)]
        print '\t'.join((str(s) for s in info))
Example #2
0
def follow(sakuya_db):

    user = auth.get_user()

    chart_list = []
    invalids = []
    for row in sakuya_db.\
               query(Follows).\
               filter_by(follower=user['userid']).\
               order_by(Follows.id.desc()):

        chart = sakuya_db.query(Charts).get(row.cid)
        if chart is None:
            invalids.append(row)
            continue

        if haopan.is_haopan(chart.cate_id):
            title = haopan.format_title(chart.name)
        else:
            title = chart.name

        chart_list.append({
            'id': chart.id,
            'name': title,
            'recv_warning': row.recv_rules == 'all'
        })

    if invalids:
        for row in invalids:
            sakuya_db.delete(row)
        sakuya_db.commit()

    return {'chart_list': chart_list}
Example #3
0
def haopan_category():
    for chart in session.query(Charts).filter_by(cate_id=100):
        title = haopan.format_title(chart.name)
        if title.startswith('hp-'):
            continue
        line = u'监控\t二手房\t好盘\t%s\t%s' % (title.decode('utf-8'), get_receivers(chart.id))
        print line.encode('utf-8')
Example #4
0
def follow(sakuya_db):

    user = auth.get_user()

    chart_list = []
    invalids = []
    for row in sakuya_db.\
               query(Follows).\
               filter_by(follower=user['userid']).\
               order_by(Follows.id.desc()):

        chart = sakuya_db.query(Charts).get(row.cid)
        if chart is None:
            invalids.append(row)
            continue

        if haopan.is_haopan(chart.cate_id):
            title = haopan.format_title(chart.name)
        else:
            title = chart.name

        chart_list.append({
            'id': chart.id,
            'name': title,
            'recv_warning': row.recv_rules == 'all'
        })

    if invalids:
        for row in invalids:
            sakuya_db.delete(row)
        sakuya_db.commit()

    return {
        'chart_list': chart_list
    }
Example #5
0
def get_charts(sakuya_db, cate1=None, cate2=None):
    lv2_list = OrderedDict()
    if cate1 is not None:
        for row in sakuya_db.\
                   query(Categories).\
                   filter_by(parent_cate_id=cate1).\
                   order_by(Categories.id):

            lv2_list[row.id] = {
                'id': row.id,
                'name': row.name
            }

    if cate2 is None:
        cate2 = lv2_list.keys()
    elif cate2 in lv2_list:
        cate2 = [cate2]
    else:
        cate2 = []

    is_admin = auth.is_role('admin')

    if cate2:
        if is_admin:
            rows = sakuya_db.query(Charts).\
                   filter(Charts.cate_id.in_(cate2)).\
                   order_by(Charts.criticals.desc(), Charts.id)
        else:
            rows = sakuya_db.query(Charts).\
                   filter(Charts.cate_id.in_(cate2), Charts.status==1).\
                   order_by(Charts.criticals.desc(), Charts.id)
    else:
        if is_admin:
            rows = sakuya_db.query(Charts).\
                   filter_by(alert_enable=1).\
                   order_by(Charts.criticals.desc(), Charts.id)
        else:
            rows = sakuya_db.query(Charts).\
                   filter_by(alert_enable=1, status=1).\
                   order_by(Charts.criticals.desc(), Charts.id)

    result = []
    for row in rows:
        if haopan.is_haopan(row.cate_id):
            title = haopan.format_title(row.name)
        else:
            title = row.name

        result.append({
            'id': row.id,
            'name': title,
            'critical': row.criticals > 0
        })
    return json.dumps(result)
Example #6
0
def get_charts(sakuya_db, cate1=None, cate2=None):
    lv2_list = OrderedDict()
    if cate1 is not None:
        for row in sakuya_db.\
                   query(Categories).\
                   filter_by(parent_cate_id=cate1).\
                   order_by(Categories.id):

            lv2_list[row.id] = {'id': row.id, 'name': row.name}

    if cate2 is None:
        cate2 = lv2_list.keys()
    elif cate2 in lv2_list:
        cate2 = [cate2]
    else:
        cate2 = []

    is_admin = auth.is_role('admin')

    if cate2:
        if is_admin:
            rows = sakuya_db.query(Charts).\
                   filter(Charts.cate_id.in_(cate2)).\
                   order_by(Charts.criticals.desc(), Charts.id)
        else:
            rows = sakuya_db.query(Charts).\
                   filter(Charts.cate_id.in_(cate2), Charts.status==1).\
                   order_by(Charts.criticals.desc(), Charts.id)
    else:
        if is_admin:
            rows = sakuya_db.query(Charts).\
                   filter_by(alert_enable=1).\
                   order_by(Charts.criticals.desc(), Charts.id)
        else:
            rows = sakuya_db.query(Charts).\
                   filter_by(alert_enable=1, status=1).\
                   order_by(Charts.criticals.desc(), Charts.id)

    result = []
    for row in rows:
        if haopan.is_haopan(row.cate_id):
            title = haopan.format_title(row.name)
        else:
            title = row.name

        result.append({
            'id': row.id,
            'name': title,
            'critical': row.criticals > 0
        })
    return json.dumps(result)
Example #7
0
def edit(sakuya_db, id):

    row = sakuya_db.query(Charts).get(id)
    if row is None:
        abort(404)

    is_haopan = haopan.is_haopan(row.cate_id)
    is_copy = request.params.get('copy') == '1'

    if auth.is_role('sa'):
        warn_only = False
    elif auth.is_role(
            'haopan'
    ) and is_haopan:  # this can be replaced with userid-cid authorization
        warn_only = True
    elif auth.get_user()['username'] == row.owner:
        warn_only = True
    else:
        abort(403)

    if is_copy and (is_haopan or warn_only):
        abort(400)

    rt = _rt(sakuya_db)
    rt['id'] = id
    rt['editing'] = not is_copy
    rt['warn_only'] = warn_only
    rt['is_haopan'] = is_haopan

    forms = {
        'name': haopan.format_title(row.name) if is_haopan else row.name,
        'cid': row.cate_id,
        'owner': row.owner,
        'alert_enable': row.alert_enable
    }

    ext = row.get_ext()
    forms['rule'] = dump_rule(ext.get('rule'))
    forms['warn'] = dump_warn(warn.get_warn(sakuya_db, id))

    forms['owner_name'] = '系统'
    for item in rt['users']:
        if item['username'] == forms['owner']:
            forms['owner_name'] = item['truename']

    rt['forms'] = forms

    return rt
Example #8
0
def edit(sakuya_db, id):

    row = sakuya_db.query(Charts).get(id)
    if row is None:
        abort(404)

    is_haopan = haopan.is_haopan(row.cate_id)
    is_copy = request.params.get('copy') == '1'

    if auth.is_role('sa'):
        warn_only = False
    elif auth.is_role('haopan') and is_haopan: # this can be replaced with userid-cid authorization
        warn_only = True
    elif auth.get_user()['username'] == row.owner:
        warn_only = True
    else:
        abort(403)

    if is_copy and (is_haopan or warn_only):
        abort(400)

    rt = _rt(sakuya_db)
    rt['id'] = id
    rt['editing'] = not is_copy
    rt['warn_only'] = warn_only
    rt['is_haopan'] = is_haopan

    forms = {
        'name': haopan.format_title(row.name) if is_haopan else row.name,
        'cid': row.cate_id,
        'owner': row.owner,
        'alert_enable': row.alert_enable
    }

    ext = row.get_ext()
    forms['rule'] = dump_rule(ext.get('rule'))
    forms['warn'] = dump_warn(warn.get_warn(sakuya_db, id))

    forms['owner_name'] = '系统'
    for item in rt['users']:
        if item['username'] == forms['owner']:
            forms['owner_name'] = item['truename']

    rt['forms'] = forms

    return rt
Example #9
0
def compare_and_notify_one(chart):

    logging.debug('Checking chart %d.' % chart.id)

    # get rules
    rules = get_rules(chart.id)
    if not rules:
        logging.debug('No rule to check.')
        return

    # parse haopan title
    if haopan.is_haopan(chart.cate_id):
        chart.name = haopan.format_title(chart.name.encode('utf-8')).decode('utf-8')

    chart_level, chart_msg = (Events.CONST_TYPE_OK, None)
    rule_warn = OrderedDict()
    for rule in rules:

        # get current data
        end = check_time
        start = end - datetime.timedelta(minutes=rule['latency'])
        data = get_data(chart.id, start, end)

        if data:
            if rule['warn_type'] == 'HWM':
                level, msg = get_alert_level_hwm(chart.name, rule['hwm_warning'], rule['hwm_critical'], data)

            elif rule['warn_type'] == 'LWM':
                level, msg = get_alert_level_lwm(chart.name, rule['lwm_warning'], rule['lwm_critical'], data)

            elif rule['warn_type'] == 'RANGE':
                prev_end = end - datetime.timedelta(7)
                prev_start = start - datetime.timedelta(7)
                prev_data = get_data(chart.id, prev_start, prev_end)
                if not prev_data:
                    prev_end = end - datetime.timedelta(1)
                    prev_start = start - datetime.timedelta(1)
                    prev_data = get_data(chart.id, prev_start, prev_end)
                level, msg = get_alert_level_range(chart.name,
                                                   rule['hwm_warning'], rule['hwm_critical'],
                                                   rule['lwm_warning'], rule['lwm_critical'],
                                                   data, prev_data)

        else:
            if 'rule' in chart.get_ext():
                no_data = True
            else:
                level = Events.CONST_TYPE_CRITICAL
                msg = u'%s 没有数据' % chart.name

        if level > chart_level:
            chart_level, chart_msg = level, msg

        rule_warn[rule['id']] = (level, msg, rule)

    # for chart
    if chart_level == Events.CONST_TYPE_OK:
        logging.debug('Chart %s' % LEVEL_TEXT[chart_level])
    else:
        logging.debug('Chart %s' % ' - '.join((LEVEL_TEXT[chart_level], chart_msg)))
        event_exists = session.\
                       query(Events).\
                       filter_by(cid=chart.id).\
                       filter(Events.type >= chart_level).\
                       filter(Events.time > check_time - datetime.timedelta(seconds=NOTIFY_SCREEN_LATENCY)).\
                       count() > 0
        if event_exists:
            logging.debug('Chart event exists (within %d minutes).' % (NOTIFY_SCREEN_LATENCY / 60))
        else:
            add_events(chart.id, chart_level, chart_msg)
    update_chart_status(chart.id, chart_level)

    # per user
    for follow in session.\
                  query(Follows).\
                  filter_by(cid=chart.id, recv_warning=True):

        if follow.follower in skipped_user:
            continue

        try:
            user = session.query(Users).get(follow.follower)
            if user is None:
                raise ValueError('User not found')

            if not user.email:
                raise ValueError('User email is empty')

            email_list = util.unique(user.email.splitlines())
            if not email_list:
                raise ValueError('User email is empty')

        except Exception, e:
            logging.warn('%s: %d' % (e, follow.follower))
            skipped_user.add(follow.follower)
            continue

        recv_rules = follow.get_recv_rules()
        if recv_rules == 'all':
            recv_rules = rule_warn.keys()

        level, msg, rule = Events.CONST_TYPE_OK, None, None
        for rid in recv_rules:
            if rid not in rule_warn:
                continue
            if rule_warn[rid][0] > level:
                level, msg, rule = rule_warn[rid]

        if level == Events.CONST_TYPE_CRITICAL:
            logging.debug('User %s %s' % (user.truename, ' - '.join((LEVEL_TEXT[level], msg))))
            if util.timestamp(check_time) - rule['last_critical'] <= rule['interval'] * 60:
                logging.debug('User event exists (within %d minutes).' % rule['interval'])
            else:
                update_rule_status(rule['id'])
                if user.id not in user_events:
                    user_events[user.id] = {
                        'truename': user.truename,
                        'email_list': email_list,
                        'events': []
                    }
                user_events[user.id]['events'].append((chart, msg))
Example #10
0
def chart(sakuya_db, id, id2=None):

    chart_info = sakuya_db.query(Charts).get(id)
    if chart_info is None:
        abort(404)

    chart_info2 =[]
    if id2:
        chart_info2 = sakuya_db.query(Charts).get(id2)

    title = chart_info.name 
    is_haopan = haopan.is_haopan(chart_info.cate_id)
    
    rt = {
        'id': str(id),
        'is_haopan': is_haopan,
        'createtime': chart_info.createtime.strftime('%Y-%m-%d %H:%M:%S')
    }

    rt['is_admin'] = auth.is_role('admin')
    rt['is_admin'] = True
    rt['status'] = chart_info.status
    '''
    more chart line
    '''
    if is_haopan is False:
        if chart_info2:
            is_haopan = True

            title = chart_info2.name
            rt['is_haopan'] = True
            rt['t1_title'] = u'当日'+chart_info.name
            rt['t2_title'] = u'当日'+chart_info2.name
            rt['t3_title'] = u'上周'+chart_info.name
            rt['t4_title'] = u'上周'+chart_info2.name

    else:
        title=haopan.format_title(chart_info.name)
        rt['t1_title'] = u'当日有效点击'
        rt['t2_title'] = u'当日无效点击'
        rt['t3_title'] = u'上周有效点击'
        rt['t4_title'] = u'上周无效点击'
        id2 = haopan.get_id2(chart_info.id)

    if is_haopan:
        rt['title'] = title#haopan.format_title(chart_info.name)
        rt['id2'] = id2#haopan.get_id2(chart_info.id)
        rt['owner'] = '系统'

    else:
        rt['title'] = title

        # owner
        owner = sakuya_db.query(Users).filter_by(username=chart_info.owner).first()
        if owner is not None:
            rt['owner'] = owner.truename
        else:
            rt['owner'] = '未知'

        # datasource
        try:
            ext_info = json.loads(chart_info.ext_info)
            if 'rule' in ext_info:
                ruc = ext_info['rule']
                storm_info = {
                    'datasource': ruc['datasource'],
                    'rule_type': storm.rule_types[ruc['rule_type']][0]
                }
                if storm.rule_types[ruc['rule_type']][1]:
                    storm_info['field'] = ruc['field']

                storm_info['filters'] = []
                for item in ruc['filters']:

                    storm_info['filters'].append({
                        'field': item[0],
                        'operator': storm.operators[storm.datasources[ruc['datasource']][item[0]][0]][0][item[1]],
                        'negative': item[2],
                        'content': item[3]
                    })

                rt['storm_info'] = storm_info

            elif chart_info.api_ip and chart_info.api_ts:
                rt['api_ip'] = util.long2ip(chart_info.api_ip)
                rt['api_ts'] = datetime.datetime.fromtimestamp(chart_info.api_ts).strftime('%Y-%m-%d %H:%M:%S')

        except Exception, e:
            pass
Example #11
0
def index(sakuya_db):

    rt = {}
    rt['stats'] = {
        'charts': sakuya_db.query(Charts).count(),
        'users': sakuya_db.query(Users).count(),
        'follows': sakuya_db.query(Follows).count()
    }

    rt['events'] = []
    for row in sakuya_db.\
               query(Events).\
               order_by(Events.id.desc()).\
               limit(7):

        if row.type == Events.CONST_TYPE_CRITICAL:
            type_text = 'Critical'
        else:
            type_text = 'Warning'

        rt['events'].append({
            'cid': row.cid,
            'info': row.info,
            'time': row.time.strftime('%H:%M'),
            'type': type_text
        })

    rt['top_monitor'] = []
    for row in sakuya_db.\
               query(Charts).\
               filter_by(root_category=2).\
               order_by(Charts.followers.desc(), Charts.id).\
               limit(7):

        rt['top_monitor'].append({
            'id': row.id,
            'name': haopan.format_title(row.name),
            'num': row.followers
        })

    rt['top_speed'] = []
    for row in sakuya_db.\
               query(Charts).\
               filter_by(root_category=3).\
               order_by(Charts.followers.desc(), Charts.id).\
               limit(7):

        rt['top_speed'].append({
            'id': row.id,
            'name': row.name,
            'num': row.followers
        })

    user = auth.get_user()
    if user is None:
        rt['login_info'] = auth.get_login_info()

    else:
        rt['my_follows'] = []
        for row in sakuya_db.\
                   query(Follows).\
                   filter_by(follower=user['userid']).\
                   limit(8):

            chart = sakuya_db.query(Charts).get(row.cid)
            if chart is None:
                continue
            rt['my_follows'].append({
                'id': chart.id,
                'name': haopan.format_title(chart.name)
            })

        rt['my_charts'] = []
        for row in sakuya_db.\
                   query(Charts).\
                   filter_by(owner=user['username']).\
                   limit(8):

            rt['my_charts'].append({
                'id': row.id,
                'name': row.name
            })

    return rt
Example #12
0
def index(sakuya_db):

    rt = {}
    rt['stats'] = {
        'charts': sakuya_db.query(Charts).count(),
        'users': sakuya_db.query(Users).count(),
        'follows': sakuya_db.query(Follows).count()
    }

    rt['events'] = []
    for row in sakuya_db.\
               query(Events).\
               order_by(Events.id.desc()).\
               limit(7):

        if row.type == Events.CONST_TYPE_CRITICAL:
            type_text = 'Critical'
        else:
            type_text = 'Warning'

        rt['events'].append({
            'cid': row.cid,
            'info': row.info,
            'time': row.time.strftime('%H:%M'),
            'type': type_text
        })

    rt['top_monitor'] = []
    for row in sakuya_db.\
               query(Charts).\
               filter_by(root_category=2).\
               order_by(Charts.followers.desc(), Charts.id).\
               limit(7):

        rt['top_monitor'].append({
            'id': row.id,
            'name': haopan.format_title(row.name),
            'num': row.followers
        })

    rt['top_speed'] = []
    for row in sakuya_db.\
               query(Charts).\
               filter_by(root_category=3).\
               order_by(Charts.followers.desc(), Charts.id).\
               limit(7):

        rt['top_speed'].append({
            'id': row.id,
            'name': row.name,
            'num': row.followers
        })

    user = auth.get_user()
    if user is None:
        rt['login_info'] = auth.get_login_info()

    else:
        rt['my_follows'] = []
        for row in sakuya_db.\
                   query(Follows).\
                   filter_by(follower=user['userid']).\
                   limit(8):

            chart = sakuya_db.query(Charts).get(row.cid)
            if chart is None:
                continue
            rt['my_follows'].append({
                'id': chart.id,
                'name': haopan.format_title(chart.name)
            })

        rt['my_charts'] = []
        for row in sakuya_db.\
                   query(Charts).\
                   filter_by(owner=user['username']).\
                   limit(8):

            rt['my_charts'].append({'id': row.id, 'name': row.name})

    return rt
Example #13
0
def chart(sakuya_db, id, id2=None):

    chart_info = sakuya_db.query(Charts).get(id)
    if chart_info is None:
        abort(404)

    chart_info2 = []
    if id2:
        chart_info2 = sakuya_db.query(Charts).get(id2)

    title = chart_info.name
    is_haopan = haopan.is_haopan(chart_info.cate_id)

    rt = {
        'id': str(id),
        'is_haopan': is_haopan,
        'createtime': chart_info.createtime.strftime('%Y-%m-%d %H:%M:%S')
    }

    rt['is_admin'] = auth.is_role('admin')
    rt['is_admin'] = True
    rt['status'] = chart_info.status
    '''
    more chart line
    '''
    if is_haopan is False:
        if chart_info2:
            is_haopan = True

            title = chart_info2.name
            rt['is_haopan'] = True
            rt['t1_title'] = u'当日' + chart_info.name
            rt['t2_title'] = u'当日' + chart_info2.name
            rt['t3_title'] = u'上周' + chart_info.name
            rt['t4_title'] = u'上周' + chart_info2.name

    else:
        title = haopan.format_title(chart_info.name)
        rt['t1_title'] = u'当日有效点击'
        rt['t2_title'] = u'当日无效点击'
        rt['t3_title'] = u'上周有效点击'
        rt['t4_title'] = u'上周无效点击'
        id2 = haopan.get_id2(chart_info.id)

    if is_haopan:
        rt['title'] = title  #haopan.format_title(chart_info.name)
        rt['id2'] = id2  #haopan.get_id2(chart_info.id)
        rt['owner'] = '系统'

    else:
        rt['title'] = title

        # owner
        owner = sakuya_db.query(Users).filter_by(
            username=chart_info.owner).first()
        if owner is not None:
            rt['owner'] = owner.truename
        else:
            rt['owner'] = '未知'

        # datasource
        try:
            ext_info = json.loads(chart_info.ext_info)
            if 'rule' in ext_info:
                ruc = ext_info['rule']
                storm_info = {
                    'datasource': ruc['datasource'],
                    'rule_type': storm.rule_types[ruc['rule_type']][0]
                }
                if storm.rule_types[ruc['rule_type']][1]:
                    storm_info['field'] = ruc['field']

                storm_info['filters'] = []
                for item in ruc['filters']:

                    storm_info['filters'].append({
                        'field':
                        item[0],
                        'operator':
                        storm.operators[storm.datasources[ruc['datasource']][
                            item[0]][0]][0][item[1]],
                        'negative':
                        item[2],
                        'content':
                        item[3]
                    })

                rt['storm_info'] = storm_info

            elif chart_info.api_ip and chart_info.api_ts:
                rt['api_ip'] = util.long2ip(chart_info.api_ip)
                rt['api_ts'] = datetime.datetime.fromtimestamp(
                    chart_info.api_ts).strftime('%Y-%m-%d %H:%M:%S')

        except Exception, e:
            pass
Example #14
0
def compare_and_notify_one(chart):

    logging.debug("Checking chart %d." % chart.id)

    # get rules
    rules = get_rules(chart.id)
    if not rules:
        logging.debug("No rule to check.")
        return

    # parse haopan title
    if haopan.is_haopan(chart.cate_id):
        chart.name = haopan.format_title(chart.name.encode("utf-8")).decode("utf-8")

    chart_level, chart_msg = (Events.CONST_TYPE_OK, None)
    rule_warn = OrderedDict()
    for rule in rules:

        # get current data
        end = check_time
        start = end - datetime.timedelta(minutes=rule["latency"])
        data = get_data(chart.id, start, end)

        if data:
            if rule["warn_type"] == "HWM":
                level, msg = get_alert_level_hwm(chart.name, rule["hwm_warning"], rule["hwm_critical"], data)

            elif rule["warn_type"] == "LWM":
                level, msg = get_alert_level_lwm(chart.name, rule["lwm_warning"], rule["lwm_critical"], data)

            elif rule["warn_type"] == "RANGE":
                prev_end = end - datetime.timedelta(7)
                prev_start = start - datetime.timedelta(7)
                prev_data = get_data(chart.id, prev_start, prev_end)
                if not prev_data:
                    prev_end = end - datetime.timedelta(1)
                    prev_start = start - datetime.timedelta(1)
                    prev_data = get_data(chart.id, prev_start, prev_end)
                level, msg = get_alert_level_range(
                    chart.name,
                    rule["hwm_warning"],
                    rule["hwm_critical"],
                    rule["lwm_warning"],
                    rule["lwm_critical"],
                    data,
                    prev_data,
                )

        else:
            if "rule" in chart.get_ext():
                no_data = True
            else:
                level = Events.CONST_TYPE_CRITICAL
                msg = u"%s 没有数据" % chart.name

        if level > chart_level:
            chart_level, chart_msg = level, msg

        rule_warn[rule["id"]] = (level, msg, rule)

    # for chart
    if chart_level == Events.CONST_TYPE_OK:
        logging.debug("Chart %s" % LEVEL_TEXT[chart_level])
    else:
        logging.debug("Chart %s" % " - ".join((LEVEL_TEXT[chart_level], chart_msg)))
        event_exists = (
            session.query(Events)
            .filter_by(cid=chart.id)
            .filter(Events.type >= chart_level)
            .filter(Events.time > check_time - datetime.timedelta(seconds=NOTIFY_SCREEN_LATENCY))
            .count()
            > 0
        )
        if event_exists:
            logging.debug("Chart event exists (within %d minutes)." % (NOTIFY_SCREEN_LATENCY / 60))
        else:
            add_events(chart.id, chart_level, chart_msg)
    update_chart_status(chart.id, chart_level)

    # per user
    for follow in session.query(Follows).filter_by(cid=chart.id, recv_warning=True):

        if follow.follower in skipped_user:
            continue

        try:
            user = session.query(Users).get(follow.follower)
            if user is None:
                raise ValueError("User not found")

            if not user.email:
                raise ValueError("User email is empty")

            email_list = util.unique(user.email.splitlines())
            if not email_list:
                raise ValueError("User email is empty")

        except Exception, e:
            logging.warn("%s: %d" % (e, follow.follower))
            skipped_user.add(follow.follower)
            continue

        recv_rules = follow.get_recv_rules()
        if recv_rules == "all":
            recv_rules = rule_warn.keys()

        level, msg, rule = Events.CONST_TYPE_OK, None, None
        for rid in recv_rules:
            if rid not in rule_warn:
                continue
            if rule_warn[rid][0] > level:
                level, msg, rule = rule_warn[rid]

        if level == Events.CONST_TYPE_CRITICAL:
            logging.debug("User %s %s" % (user.truename, " - ".join((LEVEL_TEXT[level], msg))))
            if util.timestamp(check_time) - rule["last_critical"] <= rule["interval"] * 60:
                logging.debug("User event exists (within %d minutes)." % rule["interval"])
            else:
                update_rule_status(rule["id"])
                if user.id not in user_events:
                    user_events[user.id] = {"truename": user.truename, "email_list": email_list, "events": []}
                user_events[user.id]["events"].append((chart, msg))