def run(): while True: # check time to update xml record /per one minute if is_update_time(): update_record_flag = True else: update_record_flag = False check_newdate_file() # logger.info('----------run---------') today = get_today() for (event_id, parser_obj) in parser_list.items(): parser_obj.update_log_date(today) # general logic, we come here usually for parse_log in parse_log_list: ret = parse_log.check() (file_path, log_date, log_type, inode, offset) = parse_log.get_file_record() if ret == 0: if update_record_flag: logger.info("time to update XML record, %s, inode:%s, offset:%s" % (file_path, inode, offset)) # show current state of parse log parse_log.show_state() # update the record in xml by update interval record_xml.set_log_record(file_path, inode, offset) else: logger.info("%s check error!" % log_type) time.sleep(0.3)
def __init__(self, event_list, regex_list): self._log_date = get_today() self._event_list = event_list self._regex_pattern_list = [] self._date_map = {} self.init_regex_pattern(regex_list) self.send_log = [] print "FLUME_IP %s, FLUME_PORT %s" %(FLUME_IP, FLUME_PORT) self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
def check_newdate_file(): today = get_today() # check if new date comes for each log file for parse_log in parse_log_list: (file_path, log_date, log_type, inode, offset) = parse_log.get_file_record() if compare_date(today, log_date): old_date_str = get_date_str_ex(log_date) new_date_str = get_date_str_ex(today) new_file_path = file_path.replace(old_date_str, new_date_str) if is_file(new_file_path): # last time check the old log file then update mysql logger.info("last time to check the old file %s" % file_path) ret = parse_log.check() # update parse_log parse_log.set_file_record(new_file_path, today, 0, 0) # delete old xml node in xml file record_xml.delete_log_record(file_path)
def init(): init_time() init_log(setting.LOG_PATH, maxB=setting.MAX_BYTES, bc=setting.BACK_COUNT, level=setting.LOG_LEVEL) signal.signal(signal.SIGCHLD, signal.SIG_IGN) signal.signal(signal.SIGTERM, sig_term_handler) today = get_today() for event_id in EventID.EVENT_IDS: try: print "event_id:%d" % event_id parser_obj = ParserFactory.new(event_id) if parser_obj: parser_list[event_id] = parser_obj parser_obj.update_log_date(today) else: logger.error("wrong event id:%s" % event_id) except Exception, e: logger.error("wrong in new ParserFactory:%s" % e)