def process_files(conf, targets, rflag, fflag):
if len(targets) == 0:
if conf.getboolean("general", "src_recur") or rflag:
l_fp = fslib.recur_dir(conf.getlist("general", "src_path"))
else:
l_fp = fslib.rep_dir(conf.getlist("general", "src_path"))
else:
if rflag:
l_fp = fslib.recur_dir(targets)
else:
l_fp = fslib.rep_dir(targets)
lp = logparser.LogParser(conf)
ld = LogData(conf, fflag)
ld.set_ltm()
start_dt = datetime.datetime.now()
_logger.info("log_db task start")
for fp in l_fp:
with open(fp, 'r') as f:
_logger.info("log_db processing {0}".format(fp))
for line in f:
dt, host, l_w, l_s = lp.process_line(line)
if l_w is None: continue
ltline = ld.ltm.process_line(l_w, l_s)
if ltline is None:
_logger.warning("Log template not found " + \
"for message [{0}]".format(line))
else:
ld.add_line(ltline.ltid, dt, host, l_w)
ld.commit_db()
end_dt = datetime.datetime.now()
_logger.info("log_db task done ({0})".format(end_dt - start_dt))
def test_parse(conf):
LP = LogParser(conf)
ret = []
if conf.getboolean("general", "src_recur"):
l_fp = fslib.recur_dir(conf.getlist("general", "src_path"))
else:
l_fp = fslib.rep_dir(conf.getlist("general", "src_path"))
for fp in l_fp:
with open(fp, 'r') as f:
for line in f:
ret.append(LP.process_line(line.rstrip("\n")))
return ret
def test_parse(conf):
LP = LogParser(conf)
ret = []
if conf.getboolean("general", "src_recur"):
l_fp = fslib.recur_dir(conf.getlist("general", "src_path"))
else:
l_fp = fslib.rep_dir(conf.getlist("general", "src_path"))
for fp in l_fp:
with open(fp, 'r') as f:
for line in f:
ret.append(LP.process_line(line.rstrip("\n")))
return ret
