def run(): while True: # check time to update xml record /per one minute if is_update_time(): update_record_flag = True else: update_record_flag = False check_newdate_file() # logger.info('----------run---------') today = get_today() for (event_id, parser_obj) in parser_list.items(): parser_obj.update_log_date(today) # general logic, we come here usually for parse_log in parse_log_list: ret = parse_log.check() (file_path, log_date, log_type, inode, offset) = parse_log.get_file_record() if ret == 0: if update_record_flag: logger.info("time to update XML record, %s, inode:%s, offset:%s" % (file_path, inode, offset)) # show current state of parse log parse_log.show_state() # update the record in xml by update interval record_xml.set_log_record(file_path, inode, offset) else: logger.info("%s check error!" % log_type) time.sleep(0.3)
def new(cls, event_id): if event_id == EventID.News_weibo: print "create News weibo parser object ..." return News_weibo() elif event_id == EventID.News_show: print "create News show parser object ..." logger.info("get a show event") return News_show() else: return None
def check(self): if not self._file_path or not self._log_name: logger.error("File path %s of %s was not initialed correctlly!" % (self._file_path, self._log_name)) return -1 logger.debug("start parse %s in %s" % (self._file_path, self._log_name)) if not os.path.exists(self._file_path): logger.error("no exists file %s" % self._file_path) return -2 try: file_stat = os.stat(self._file_path) if not stat.S_ISREG(file_stat.st_mode): logger.error("%s is not regular file" % self._file_path) return -2 logger.debug("last inode:%d, last pos:%d" % (self._inode, self._last_pos)) if self._inode <= 0: self._inode = file_stat.st_ino self._last_pos = 0 if self._fp: self._fp.close() self._fp = None elif self._inode != file_stat.st_ino: logger.info("File(%s)'s inode has been changed from %d to %d!" % (self._file_path, self._inode, file_stat.st_ino)) self._inode = file_stat.st_ino # here we can consider whether system archiving happened or someone remove the log, then do something appropriately # and now we just consider it system archiving self._last_pos = 0 if self._fp: self._fp.close() self._fp = None if self._last_pos > file_stat.st_size: logger.info("File(%s)'s size has been changed from %d to %d!" % (self._file_path, self._last_pos, file_stat.st_size)) # here, may be system archiving happened or someone cut the # log, so the same as upstair. self._last_pos = 0 elif self._last_pos < file_stat.st_size: # normal condition we come to here logger.debug("File(%s) size increase from %d to %d" % (self._file_path, self._last_pos, file_stat.st_size)) self.__read_content() return 0 except Exception, e: logger.error("Exception accured during Check File %s! (%s)" % (self._file_path, e)) return -3
def sig_term_handler(sig, func=None): logger.info("catch sig term...") for (event_id, parser_obj) in parser_list.items(): print "desctruct event %s" % event_id parser_obj.destruct() for parse_log in parse_log_list: (file_path, log_date, log_type, inode, offset) = parse_log.get_file_record() logger.info("lastly update XML record, %s, inode:%s, offset:%s" % (file_path, inode, offset)) # update the record in xml by update interval record_xml.set_log_record(file_path, inode, offset) exit()
def check_newdate_file(): today = get_today() # check if new date comes for each log file for parse_log in parse_log_list: (file_path, log_date, log_type, inode, offset) = parse_log.get_file_record() if compare_date(today, log_date): old_date_str = get_date_str_ex(log_date) new_date_str = get_date_str_ex(today) new_file_path = file_path.replace(old_date_str, new_date_str) if is_file(new_file_path): # last time check the old log file then update mysql logger.info("last time to check the old file %s" % file_path) ret = parse_log.check() # update parse_log parse_log.set_file_record(new_file_path, today, 0, 0) # delete old xml node in xml file record_xml.delete_log_record(file_path)
def update_logs(date): global record_xml record_xml = RecordXML(setting.LOG_RECORD_FILE) date_str = get_date_str_ex(date) for (log_type, log_config) in setting.RULE_MAP.items(): log_file = log_config["path"] log_file = log_file.replace('$DATE', date_str) (log_inode, log_off) = record_xml.get_log_record(log_file) logger.info("log file %s, log_inode %d", log_file, log_inode) tmp_parser_list = [] for rule in log_config['rules']: event_id = rule['rule_id'] if event_id in parser_list: tmp_parser_list.append(parser_list.get(event_id)) new_parse_log = ParseLog(log_type, tmp_parser_list) new_parse_log.set_file_record(log_file, date, log_inode, log_off) print '%s %s %s-%s-%s' % (log_file, log_type, date.year, date.month, date.day) parse_log_list.append(new_parse_log)
offset) = parse_log.get_file_record() if ret == 0: if update_record_flag: logger.info("time to update XML record, %s, inode:%s, offset:%s" % (file_path, inode, offset)) # show current state of parse log parse_log.show_state() # update the record in xml by update interval record_xml.set_log_record(file_path, inode, offset) else: logger.info("%s check error!" % log_type) time.sleep(0.3) if __name__ == "__main__": try: init() run() except Exception, e: for (event_id, parser_obj) in parser_list.items(): print "desctruct event %s" % event_id parser_obj.destruct() for parse_log in parse_log_list: (file_path, log_date, log_type, inode, offset) = parse_log.get_file_record() logger.info("lastly update XML record, %s, inode:%s, offset:%s" % (file_path, inode, offset)) # update the record in xml by update interval record_xml.set_log_record(file_path, inode, offset) logger.error("Exception happen,%s" % e)