def run():
while True:
# check time to update xml record /per one minute
if is_update_time():
update_record_flag = True
else:
update_record_flag = False
check_newdate_file()
# logger.info('----------run---------')
today = get_today()
for (event_id, parser_obj) in parser_list.items():
parser_obj.update_log_date(today)
# general logic, we come here usually
for parse_log in parse_log_list:
ret = parse_log.check()
(file_path, log_date, log_type, inode,
offset) = parse_log.get_file_record()
if ret == 0:
if update_record_flag:
logger.info("time to update XML record, %s, inode:%s, offset:%s"
% (file_path, inode, offset))
# show current state of parse log
parse_log.show_state()
# update the record in xml by update interval
record_xml.set_log_record(file_path, inode, offset)
else:
logger.info("%s check error!" % log_type)
time.sleep(0.3)
def new(cls, event_id):
if event_id == EventID.News_weibo:
print "create News weibo parser object ..."
return News_weibo()
elif event_id == EventID.News_show:
print "create News show parser object ..."
logger.info("get a show event")
return News_show()
else:
return None
def check(self):
if not self._file_path or not self._log_name:
logger.error("File path %s of %s was not initialed correctlly!"
% (self._file_path, self._log_name))
return -1
logger.debug("start parse %s in %s" %
(self._file_path, self._log_name))
if not os.path.exists(self._file_path):
logger.error("no exists file %s" % self._file_path)
return -2
try:
file_stat = os.stat(self._file_path)
if not stat.S_ISREG(file_stat.st_mode):
logger.error("%s is not regular file" % self._file_path)
return -2
logger.debug("last inode:%d, last pos:%d" %
(self._inode, self._last_pos))
if self._inode <= 0:
self._inode = file_stat.st_ino
self._last_pos = 0
if self._fp:
self._fp.close()
self._fp = None
elif self._inode != file_stat.st_ino:
logger.info("File(%s)'s inode has been changed from %d to %d!"
% (self._file_path, self._inode, file_stat.st_ino))
self._inode = file_stat.st_ino
# here we can consider whether system archiving happened or someone remove the log, then do something appropriately
# and now we just consider it system archiving
self._last_pos = 0
if self._fp:
self._fp.close()
self._fp = None
if self._last_pos > file_stat.st_size:
logger.info("File(%s)'s size has been changed from %d to %d!"
% (self._file_path, self._last_pos, file_stat.st_size))
# here, may be system archiving happened or someone cut the
# log, so the same as upstair.
self._last_pos = 0
elif self._last_pos < file_stat.st_size:
# normal condition we come to here
logger.debug("File(%s) size increase from %d to %d"
% (self._file_path, self._last_pos, file_stat.st_size))
self.__read_content()
return 0
except Exception, e:
logger.error("Exception accured during Check File %s! (%s)" %
(self._file_path, e))
return -3
def sig_term_handler(sig, func=None):
logger.info("catch sig term...")
for (event_id, parser_obj) in parser_list.items():
print "desctruct event %s" % event_id
parser_obj.destruct()
for parse_log in parse_log_list:
(file_path, log_date, log_type, inode,
offset) = parse_log.get_file_record()
logger.info("lastly update XML record, %s, inode:%s, offset:%s"
% (file_path, inode, offset))
# update the record in xml by update interval
record_xml.set_log_record(file_path, inode, offset)
exit()
def check_newdate_file():
today = get_today()
# check if new date comes for each log file
for parse_log in parse_log_list:
(file_path, log_date, log_type, inode,
offset) = parse_log.get_file_record()
if compare_date(today, log_date):
old_date_str = get_date_str_ex(log_date)
new_date_str = get_date_str_ex(today)
new_file_path = file_path.replace(old_date_str, new_date_str)
if is_file(new_file_path):
# last time check the old log file then update mysql
logger.info("last time to check the old file %s" % file_path)
ret = parse_log.check()
# update parse_log
parse_log.set_file_record(new_file_path, today, 0, 0)
# delete old xml node in xml file
record_xml.delete_log_record(file_path)
def update_logs(date):
global record_xml
record_xml = RecordXML(setting.LOG_RECORD_FILE)
date_str = get_date_str_ex(date)
for (log_type, log_config) in setting.RULE_MAP.items():
log_file = log_config["path"]
log_file = log_file.replace('$DATE', date_str)
(log_inode, log_off) = record_xml.get_log_record(log_file)
logger.info("log file %s, log_inode %d", log_file, log_inode)
tmp_parser_list = []
for rule in log_config['rules']:
event_id = rule['rule_id']
if event_id in parser_list:
tmp_parser_list.append(parser_list.get(event_id))
new_parse_log = ParseLog(log_type, tmp_parser_list)
new_parse_log.set_file_record(log_file, date, log_inode, log_off)
print '%s %s %s-%s-%s' % (log_file, log_type, date.year, date.month, date.day)
parse_log_list.append(new_parse_log)
offset) = parse_log.get_file_record()
if ret == 0:
if update_record_flag:
logger.info("time to update XML record, %s, inode:%s, offset:%s"
% (file_path, inode, offset))
# show current state of parse log
parse_log.show_state()
# update the record in xml by update interval
record_xml.set_log_record(file_path, inode, offset)
else:
logger.info("%s check error!" % log_type)
time.sleep(0.3)
if __name__ == "__main__":
try:
init()
run()
except Exception, e:
for (event_id, parser_obj) in parser_list.items():
print "desctruct event %s" % event_id
parser_obj.destruct()
for parse_log in parse_log_list:
(file_path, log_date, log_type, inode,
offset) = parse_log.get_file_record()
logger.info("lastly update XML record, %s, inode:%s, offset:%s"
% (file_path, inode, offset))
# update the record in xml by update interval
record_xml.set_log_record(file_path, inode, offset)
logger.error("Exception happen,%s" % e)
