def process_line(msg, ld, lp, ha, isnew_check = False, latest = None):
"""Add a log message to DB.
Args:
msg (str): A log message to process.
Line feed code will be ignored.
ld (LogData): An log database interface opened in edit mode.
Needs to initialize template classifier with ld.init_ltmanager.
lp (logparser.LogParser): An open message parser.
latest (Optional[datetime.datetime]): If not None,
Ignore messages that have later timestamp than 'latest'.
Returns:
LogMessage: An annotated log message instance.
Same as lines given with LogData.iterlines.
"""
line = None
dt, org_host, l_w, l_s = lp.process_line(msg)
if latest is not None and dt < latest: return None
if l_w is None: return None
l_w = [strutil.add_esc(w) for w in l_w]
host = ha.resolve_host(org_host)
if host is None: host = org_host
_logger.debug("Processing [{0}]".format(" ".join(l_w)))
ltline = ld.ltm.process_line(l_w, l_s)
if ltline is None:
ld.ltm.failure_output(msg)
else:
_logger.debug("Template [{0}]".format(ltline))
line = ld.add_line(ltline.ltid, dt, host, l_w)
return line
def generate_lt_from_file(conf, fn):
_logger.info("job for ({0}) start".format(fn))
import logparser
lp = logparser.LogParser(conf)
table = lt_common.TemplateTable()
sym = conf.get("log_template", "variable_symbol")
d_symlist = {}
ltgen = LTGenCRF(table, sym, conf)
with open(fn, "r") as f:
for line in f:
dt, org_host, l_w, l_s = lp.process_line(line)
if l_w is None: continue
l_w = [strutil.add_esc(w) for w in l_w]
tid, dummy = ltgen.process_line(l_w, l_s)
d_symlist[tid] = l_s
ret = []
for tid in table.tids():
tpl = table.get_template(tid)
l_s = d_symlist[tid]
ret.append((tpl, l_s))
_logger.info("job for ({0}) done".format(fn))
return ret
def generate_lt_from_file(conf, fn):
_logger.info("job for ({0}) start".format(fn))
import logparser
lp = logparser.LogParser(conf)
table = lt_common.TemplateTable()
sym = conf.get("log_template", "variable_symbol")
d_symlist = {}
ltgen = LTGenCRF(table, sym, conf)
with open(fn, "r") as f:
for line in f:
dt, org_host, l_w, l_s = lp.process_line(line)
if l_w is None: continue
l_w = [strutil.add_esc(w) for w in l_w]
tid, dummy = ltgen.process_line(l_w, l_s)
d_symlist[tid] = l_s
ret = []
for tid in table.tids():
tpl = table.get_template(tid)
l_s = d_symlist[tid]
ret.append((tpl, l_s))
_logger.info("job for ({0}) done".format(fn))
return ret
def process_init_data(conf, targets, isnew_check = False):
"""Add log messages to DB from files. This function do NOT process
messages incrementally. Use this to avoid bad-start problem of
log template generation with clustering or training methods.
Note:
This function needs large memory space.
Args:
conf (config.ExtendedConfigParser): A common configuration object.
targets (List[str]): A sequence of filepaths to process.
isnew_check (Optional[bool]): If True, add message to DB
only if its timestamp is newest of existing messages in DB.
Raises:
IOError: If a file in targets not found.
"""
ld = LogData(conf, edit = True, reset_db = True)
ld.init_ltmanager()
lp = logparser.LogParser(conf)
ha = host_alias.HostAlias(conf)
latest = ld.dt_term()[1] if isnew_check else None
drop_undefhost = conf.getboolean("database", "undefined_host")
l_line = []
l_data = []
for line in _iter_line_from_files(targets):
dt, org_host, l_w, l_s = lp.process_line(line)
if latest is not None and dt < latest: continue
if l_w is None: continue
l_w = [strutil.add_esc(w) for w in l_w]
host = ha.resolve_host(org_host)
if host is None:
if drop_undefhost:
ld.ltm.failure_output(msg)
return None
else:
host = org_host
l_line.append((l_w, l_s))
l_data.append((dt, host))
for ltline, line, data in zip(ld.ltm.process_init_data(l_line),
l_line, l_data):
l_w, l_s = line
dt, host = data
ld.add_line(ltline.ltid, dt, host, l_w)
ld.commit_db()
def process_line(msg, ld, lp, ha, isnew_check = False, latest = None,
drop_undefhost = False):
"""Add a log message to DB.
Args:
msg (str): A log message to process.
Line feed code will be ignored.
ld (LogData): An log database interface opened in edit mode.
Needs to initialize template classifier with ld.init_ltmanager.
lp (logparser.LogParser): An open message parser.
latest (Optional[datetime.datetime]): If not None,
Ignore messages that have later timestamp than 'latest'.
Returns:
LogMessage: An annotated log message instance.
Same as lines given with LogData.iterlines.
"""
line = None
dt, org_host, l_w, l_s = lp.process_line(msg)
if latest is not None and dt < latest: return None
if l_w is None: return None
l_w = [strutil.add_esc(w) for w in l_w]
host = ha.resolve_host(org_host)
#if host is None: host = org_host
if host is None:
#if conf.getboolean("database", "undefined_host"):
if drop_undefhost:
ld.ltm.failure_output(msg)
return None
else:
host = org_host
_logger.debug("Processing [{0}]".format(" ".join(l_w)))
ltline = ld.ltm.process_line(l_w, l_s)
if ltline is None:
ld.ltm.failure_output(msg)
return None
else:
_logger.debug("Template [{0}]".format(ltline))
line = ld.add_line(ltline.ltid, dt, host, l_w)
return line
def process_init_data(conf, targets, isnew_check = False):
"""Add log messages to DB from files. This function do NOT process
messages incrementally. Use this to avoid bad-start problem of
log template generation with clustering or training methods.
Note:
This function needs large memory space.
Args:
conf (config.ExtendedConfigParser): A common configuration object.
targets (List[str]): A sequence of filepaths to process.
isnew_check (Optional[bool]): If True, add message to DB
only if its timestamp is newest of existing messages in DB.
Raises:
IOError: If a file in targets not found.
"""
ld = LogData(conf, edit = True, reset_db = True)
ld.init_ltmanager()
lp = logparser.LogParser(conf)
ha = host_alias.HostAlias(conf)
latest = ld.dt_term()[1] if isnew_check else None
l_line = []
l_data = []
for line in _iter_line_from_files(targets):
dt, org_host, l_w, l_s = lp.process_line(line)
if latest is not None and dt < latest: continue
if l_w is None: continue
l_w = [strutil.add_esc(w) for w in l_w]
host = ha.resolve_host(org_host)
if host is None: host = org_host
l_line.append((l_w, l_s))
l_data.append((dt, host))
for ltline, line, data in zip(ld.ltm.process_init_data(l_line),
l_line, l_data):
l_w, l_s = line
dt, host = data
ld.add_line(ltline.ltid, dt, host, l_w)
ld.commit_db()
def search_exception(conf, targets):
import logparser
import strutil
lp = logparser.LogParser(conf)
def_path = conf.get("log_template_import", "def_path")
sym = conf.get("log_template", "variable_symbol")
mode = conf.get("log_template_import", "mode")
table = lt_common.TemplateTable()
temp_lp = logparser.LogParser(conf, sep_variable = True)
ltgen = LTGenImport(table, sym, def_path, mode, temp_lp)
for fn in targets:
_logger.info("lt_import job for ({0}) start".format(fn))
with open(fn, "r") as f:
for line in f:
dt, org_host, l_w, l_s = lp.process_line(line)
if l_w is None: continue
l_w = [strutil.add_esc(w) for w in l_w]
tid, dummy = ltgen.process_line(l_w, l_s)
if tid is None:
print line.rstrip("\n")
_logger.info("lt_import job for ({0}) done".format(fn))
def _key_template(self, template):
l_word = [strutil.add_esc(w) for w in template]
return "@".join(l_word)
def _key_template(self, template):
l_word = [strutil.add_esc(w) for w in template]
return "@".join(l_word)