def _open_def(self, filename, mode, shuffle):
if filename.strip() == "":
errmes = "lt_import initialized with empty template set"
_logger.info(errmes)
return
elif not os.path.exists(filename):
errmes = ("log_template_import.def_path {0} is invalid".format(
filename))
raise IOError(errmes)
cnt = 0
with open(filename, 'r') as f:
for line in f:
if mode == "plain":
mes = line.rstrip("\n")
elif mode == "manual":
pline = self._lp.process_message(strutil.add_esc(line))
mes = pline["words"]
else:
raise ValueError("invalid import_mode {0}".format(
mode))
if len(mes) == 0:
continue
ltw, lts = self._lp.process_statement(mes)
self.add_definition(ltw)
cnt += 1
_logger.info("LTGenImport: {0} template imported".format(cnt))
if shuffle:
self._ltmap.shuffle()
def _process_offline_single(self, iterable_lines):
d_pline = {}
for mid, line in enumerate(iterable_lines):
pline = parse_line(strutil.add_esc(line), self._lp)
pline = normalize_pline(pline, self._ha, self._drop_undefhost)
d_pline[mid] = pline
offline_input = {mid: pline for mid, pline in d_pline.items()
if pline is not None}
d_tid = self._ltgen.process_offline(offline_input)
return d_pline, d_tid
def iter_plines(conf, targets, pass_none=True):
lp = load_log2seq(conf)
ha = host_alias.init_hostalias(conf)
drop_undefhost = conf.getboolean("manager", "undefined_host")
for line in iter_lines(targets):
pline = parse_line(strutil.add_esc(line), lp)
pline = normalize_pline(pline, ha, drop_undefhost)
if pline is None and pass_none:
pass
else:
yield pline
def _pool_task(batch):
lp = _MULTIPROCESS_LOCAL_OBJECTS["lp"]
ha = _MULTIPROCESS_LOCAL_OBJECTS["ha"]
ltgen = _MULTIPROCESS_LOCAL_OBJECTS["ltgen"]
drop_undefhost = _MULTIPROCESS_LOCAL_OBJECTS["drop_undefhost"]
ret = []
for message_id, line in batch:
pline = parse_line(strutil.add_esc(line), lp)
pline = normalize_pline(pline, ha, drop_undefhost)
if pline is None:
ret.append([message_id, None, None])
else:
tpl = ltgen.generate_tpl(pline)
ret.append([message_id, pline, tpl])
return ret
def get_parsed_line(self, line):
pline = parse_line(strutil.add_esc(line), self._lp)
pline = normalize_pline(pline, self._ha, self._drop_undefhost)
return pline