コード例 #1
0
ファイル: broodling.py プロジェクト: smiledaddy/Crystal
def run():
    while True:
        # check time to update xml record /per one minute
        if is_update_time():
            update_record_flag = True
        else:
            update_record_flag = False
        check_newdate_file()
#        logger.info('----------run---------')
        today = get_today()
        for (event_id, parser_obj) in parser_list.items():
            parser_obj.update_log_date(today)
        # general logic, we come here usually
        for parse_log in parse_log_list:
            ret = parse_log.check()
            (file_path, log_date, log_type, inode,
             offset) = parse_log.get_file_record()
            if ret == 0:
                if update_record_flag:
                    logger.info("time to update XML record, %s, inode:%s, offset:%s"
                                % (file_path, inode, offset))
                    # show current state of parse log
                    parse_log.show_state()
                    # update the record in xml by update interval
                    record_xml.set_log_record(file_path, inode, offset)
            else:
                logger.info("%s check error!" % log_type)

        time.sleep(0.3)
コード例 #2
0
ファイル: parser.py プロジェクト: smiledaddy/Crystal
 def __init__(self, event_list, regex_list):
     self._log_date = get_today()
     self._event_list = event_list
     self._regex_pattern_list = []
     self._date_map = {}
     self.init_regex_pattern(regex_list)
     self.send_log = []
     print "FLUME_IP %s, FLUME_PORT %s" %(FLUME_IP, FLUME_PORT)
     self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
コード例 #3
0
ファイル: broodling.py プロジェクト: smiledaddy/Crystal
def check_newdate_file():
    today = get_today()
    # check if new date comes for each log file
    for parse_log in parse_log_list:
        (file_path, log_date, log_type, inode,
         offset) = parse_log.get_file_record()
        if compare_date(today, log_date):
            old_date_str = get_date_str_ex(log_date)
            new_date_str = get_date_str_ex(today)
            new_file_path = file_path.replace(old_date_str, new_date_str)
            if is_file(new_file_path):
                # last time check the old log file then update mysql
                logger.info("last time to check the old file %s" % file_path)
                ret = parse_log.check()
                # update parse_log
                parse_log.set_file_record(new_file_path, today, 0, 0)
                # delete old xml node in xml file
                record_xml.delete_log_record(file_path)
コード例 #4
0
ファイル: broodling.py プロジェクト: smiledaddy/Crystal
def init():
    init_time()
    init_log(setting.LOG_PATH,
             maxB=setting.MAX_BYTES,
             bc=setting.BACK_COUNT,
             level=setting.LOG_LEVEL)
    signal.signal(signal.SIGCHLD, signal.SIG_IGN)
    signal.signal(signal.SIGTERM, sig_term_handler)
    today = get_today()
    for event_id in EventID.EVENT_IDS:
        try:
            print "event_id:%d" % event_id
            parser_obj = ParserFactory.new(event_id)
            if parser_obj:
                parser_list[event_id] = parser_obj
                parser_obj.update_log_date(today)
            else:
                logger.error("wrong event id:%s" % event_id)
        except Exception, e:
            logger.error("wrong in new ParserFactory:%s" % e)