def run():
while True:
# check time to update xml record /per one minute
if is_update_time():
update_record_flag = True
else:
update_record_flag = False
check_newdate_file()
# logger.info('----------run---------')
today = get_today()
for (event_id, parser_obj) in parser_list.items():
parser_obj.update_log_date(today)
# general logic, we come here usually
for parse_log in parse_log_list:
ret = parse_log.check()
(file_path, log_date, log_type, inode,
offset) = parse_log.get_file_record()
if ret == 0:
if update_record_flag:
logger.info("time to update XML record, %s, inode:%s, offset:%s"
% (file_path, inode, offset))
# show current state of parse log
parse_log.show_state()
# update the record in xml by update interval
record_xml.set_log_record(file_path, inode, offset)
else:
logger.info("%s check error!" % log_type)
time.sleep(0.3)
def __init__(self, event_list, regex_list):
self._log_date = get_today()
self._event_list = event_list
self._regex_pattern_list = []
self._date_map = {}
self.init_regex_pattern(regex_list)
self.send_log = []
print "FLUME_IP %s, FLUME_PORT %s" %(FLUME_IP, FLUME_PORT)
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
def check_newdate_file():
today = get_today()
# check if new date comes for each log file
for parse_log in parse_log_list:
(file_path, log_date, log_type, inode,
offset) = parse_log.get_file_record()
if compare_date(today, log_date):
old_date_str = get_date_str_ex(log_date)
new_date_str = get_date_str_ex(today)
new_file_path = file_path.replace(old_date_str, new_date_str)
if is_file(new_file_path):
# last time check the old log file then update mysql
logger.info("last time to check the old file %s" % file_path)
ret = parse_log.check()
# update parse_log
parse_log.set_file_record(new_file_path, today, 0, 0)
# delete old xml node in xml file
record_xml.delete_log_record(file_path)
def init():
init_time()
init_log(setting.LOG_PATH,
maxB=setting.MAX_BYTES,
bc=setting.BACK_COUNT,
level=setting.LOG_LEVEL)
signal.signal(signal.SIGCHLD, signal.SIG_IGN)
signal.signal(signal.SIGTERM, sig_term_handler)
today = get_today()
for event_id in EventID.EVENT_IDS:
try:
print "event_id:%d" % event_id
parser_obj = ParserFactory.new(event_id)
if parser_obj:
parser_list[event_id] = parser_obj
parser_obj.update_log_date(today)
else:
logger.error("wrong event id:%s" % event_id)
except Exception, e:
logger.error("wrong in new ParserFactory:%s" % e)