def list_trouble_stat(ns):
conf = open_logdag_config(ns)
amulog_conf = config.open_config(conf["database_amulog"]["source_conf"])
from . import trouble
tm = trouble.init_trouble_manager(conf)
from amulog import log_db
ld = log_db.LogData(amulog_conf)
gid_name = conf.get("database_amulog", "event_gid")
from scipy.stats import entropy
table = [[
"trouble_id", "group", "messages", "gids", "hosts", "events", "groups",
"entropy_events", "entropy_groups"
]]
for tr in tm:
line = []
d_ev, d_gid, d_host = trouble.event_stat(tr, ld, gid_name)
d_group = trouble.event_label(d_gid, ld, gid_name)
ent_ev = entropy(list(d_ev.values()), base=2)
ent_group = entropy(
[sum([d_gid[gid] for gid in l_gid]) for l_gid in d_group.values()],
base=2)
line.append(tr.tid)
line.append(tr.data["group"])
line.append(sum(d_gid.values())) # messages
line.append(len(d_gid.keys())) # gids
line.append(len(d_host.keys())) # hosts
line.append(len(d_ev.keys())) # events
line.append(len(d_group.keys())) # groups
line.append(ent_ev) # entropy of events
line.append(ent_group) # entropy of groups
table.append(line)
print(common.cli_table(table))
def show_trouble_info(ns):
conf = open_logdag_config(ns)
tid = ns.tid
from . import trouble
dirname = conf.get("eval", "path")
tm = trouble.TroubleManager(dirname)
from amulog import log_db
ld = log_db.LogData(conf)
from amulog import lt_label
ll = lt_label.init_ltlabel(conf)
gid_name = conf.get("dag", "event_gid")
tr = tm[tid]
d_ev, d_gid, d_host = trouble.event_stat(tr, ld, gid_name)
d_group = trouble.event_label(d_gid, ld, ll)
print(tr)
print("{0} related events".format(len(d_ev)))
print("{0} related hosts: {1}".format(len(d_host), sorted(d_host.keys())))
print("{0} related templates: {1}".format(len(d_gid),
sorted(d_gid.keys())))
for group, l_gid in d_group.items():
num = sum([d_gid[gid] for gid in l_gid])
print(" group {0}: {1} messages, {2} templates {3}".format(
group, num, len(l_gid), l_gid))
def search_trouble(ns):
conf = open_logdag_config(ns)
amulog_conf = config.open_config(conf["database_amulog"]["source_conf"])
d = parse_condition(ns.conditions)
from . import trouble
tm = trouble.init_trouble_manager(conf)
from amulog import log_db
ld = log_db.LogData(amulog_conf)
gid_name = conf.get("database_amulog", "event_gid")
# match group
if "group" in d:
l_tr = [tr for tr in tm if tr.data["group"] == d["group"]]
else:
l_tr = [tr for tr in tm]
# match event
if "gid" in d or "host" in d:
search_gid = d.get("gid", None)
search_host = d.get("host", None)
ret = []
for tr in l_tr:
for lid in tr.data["message"]:
lm = ld.get_line(lid)
gid = lm.lt.get(gid_name)
host = lm.host
if (search_gid is None or search_gid == gid) and \
(search_host is None or search_host == host):
ret.append(tr)
break
l_tr = ret
for tr in l_tr:
print(tr)
def test_db_sqlite3(self):
path_testlog = "/tmp/amulog_testlog"
path_db = "/tmp/amulog_db"
conf = config.open_config()
path_testlog = conf['general']['src_path']
path_db = conf['database']['sqlite3_filename']
tlg = testlog.TestLogGenerator(testlog.DEFAULT_CONFIG, seed=3)
tlg.dump_log(path_testlog)
l_path = config.getlist(conf, "general", "src_path")
if conf.getboolean("general", "src_recur"):
targets = common.recur_dir(l_path)
else:
targets = common.rep_dir(l_path)
log_db.process_files(conf, targets, True)
ld = log_db.LogData(conf)
num = ld.count_lines()
self.assertEqual(num, 6539, "not all logs added to database")
ltg_num = len([gid for gid in ld.iter_ltgid()])
self.assertTrue(ltg_num > 3 and ltg_num < 10,
("log template generation fails? "
"(groups: {0})".format(ltg_num)))
del ld
common.rm(path_testlog)
common.rm(path_db)
def log2ts_pal(conf, dt_range, pal=1):
from amulog import common
timer = common.Timer(
"make-tsdb subtask ({0[0]} - {0[1]})".format(dt_range), output=_logger)
timer.start()
gid_name = conf.get("dag", "event_gid")
usefilter = conf.getboolean("database_ts", "usefilter")
from amulog import log_db
ld = log_db.LogData(conf)
if gid_name == "ltid":
iterobj = ld.whole_host_lt(dt_range[0], dt_range[1], "all")
elif gid_name == "ltgid":
iterobj = ld.whole_host_ltg(dt_range[0], dt_range[1], "all")
else:
raise NotImplementedError
import multiprocessing
td = TimeSeriesDB(conf, edit=True)
l_args = [(conf, dt_range, gid, host) for host, gid in iterobj]
with multiprocessing.Pool(processes=pal) as pool:
for ret in pool.imap_unordered(log2ts_elem, l_args):
gid, host, stat, new_l_dt, val = ret
if new_l_dt is not None and len(new_l_dt) > 0:
for dt in new_l_dt:
td.add_line(dt, gid, host)
td.add_filterlog(dt_range, gid, host, stat, val)
pool.close()
pool.join()
td.commit()
timer.stop()
return
def log2ts_elem(args):
conf, dt_range, gid, host = args
name = "{0}_{1}_{2}".format(dtutil.shortstr(dt_range[0]), gid, host)
_logger.info("make-tsdb job start ({0})".format(name))
from amulog import log_db
ld = log_db.LogData(conf)
gid_name = conf.get("dag", "event_gid")
d = {
gid_name: gid,
"host": host,
"top_dt": dt_range[0],
"end_dt": dt_range[1]
}
iterobj = ld.iter_lines(**d)
l_dt = sorted([line.dt for line in iterobj])
del iterobj
_logger.debug("gid {0}, host {1}: {2} counts".format(gid, host, len(l_dt)))
assert len(l_dt) > 0
evdef = (gid, host)
stat, new_l_dt, val = apply_filter(conf, ld, l_dt, dt_range, evdef)
fl = FilterLog(dt_range, gid, host, stat, val)
_logger.debug(str(fl))
_logger.info("make-tsdb job done ({0})".format(name))
return (gid, host, stat, new_l_dt, val)
def process_files_offline(conf, targets, reset_db, parallel=False):
"""Add log messages to DB from files. This function do NOT process
messages incrementally. Use this to avoid bad-start problem of
log template generation with clustering or training methods.
Note:
This function needs large memory space.
Args:
conf (config.ExtendedConfigParser): A common configuration object.
targets (List[str]): A sequence of filepaths to process.
reset_db (bool): True if DB needs to reset before adding.
parallel (bool, optional): Use multiprocessing.
Raises:
IOError: If a file in targets not found.
"""
msg = "amulog offline processing"
if parallel:
msg += " in parallel"
_logger.info(msg)
ld = log_db.LogData(conf, edit=True, reset_db=reset_db)
ltm = LTManager(conf, ld.db, ld.lttable, reset_db=reset_db,
parallel=parallel)
l_line = [line for line in iter_lines(targets)]
ltm.process_offline(l_line)
def process_files_online(conf, targets, reset_db):
"""Add log messages to DB from files.
Args:
conf (config.ExtendedConfigParser): A common configuration object.
targets (List[str]): A sequence of filepaths to process.
reset_db (bool): True if DB needs to reset before adding.
Raises:
IOError: If a file in targets not found.
"""
def _sigterm_handler():
raise KeyboardInterrupt
import signal
signal.signal(signal.SIGTERM, _sigterm_handler)
msg = "amulog online processing"
_logger.info(msg)
ld = log_db.LogData(conf, edit=True, reset_db=reset_db)
ltm = LTManager(conf, ld.db, ld.lttable, reset_db=reset_db)
try:
for line in iter_lines(targets):
ltm.process_line(line)
except KeyboardInterrupt:
pass
finally:
ltm.commit_db()
ltm.dump()
def search_trouble(ns):
conf = open_logdag_config(ns)
d = parse_condition(ns.conditions)
from . import trouble
dirname = conf.get("eval", "path")
tm = trouble.TroubleManager(dirname)
from amulog import log_db
ld = log_db.LogData(conf)
gid_name = conf.get("dag", "event_gid")
from logdag import dtutil
# match group
if "group" in d:
l_tr = [tr for tr in tm if tr.data["group"] == d["group"]]
else:
l_tr = [tr for tr in tm]
# match event
if "gid" in d or "host" in d:
search_gid = d.get("gid", None)
search_host = d.get("host", None)
ret = []
for tr in l_tr:
for lid in tr.data["message"]:
lm = ld.get_line(lid)
gid = lm.lt.get(gid_name)
host = lm.host
if (search_gid is None or search_gid == gid) and \
(search_host is None or search_host == host):
ret.append(tr)
break
l_tr = ret
for tr in l_tr:
print(tr)
def make_crf_model(ns):
conf_load = config.open_config(ns.config_load)
conf_dump = config.open_config(ns.config_dump)
lv = logging.DEBUG if ns.debug else logging.INFO
config.set_common_logging(conf_dump, logger=_logger, lv=lv)
from . import train
output = ns.output
if ns.train_file is None:
from amulog import log_db
d = parse_condition(ns.conditions)
output_sampled = ns.output_sampled
get_output_sampled = output_sampled is not None
ld = log_db.LogData(conf_load)
iterobj = [lm for lm in ld.iter_lines(**d)]
if get_output_sampled:
fn, l_train = train.make_crf_model(conf_dump, iterobj, output,
return_sampled_messages=True)
import pickle
with open(output_sampled, 'wb') as f:
pickle.dump(l_train, f)
print("> {0}".format(output_sampled))
else:
fn = train.make_crf_model(conf_dump, iterobj, output)
else:
fn = train.make_crf_model_from_trainfile(conf_dump, ns.train_file, output)
print("> {0}".format(fn))
def test_makedb_offline(self):
from amulog import __main__ as amulog_main
targets = amulog_main.get_targets_conf(self._conf)
manager.process_files_offline(self._conf, targets, reset_db=True)
ld = log_db.LogData(self._conf)
num = ld.count_lines()
self.assertEqual(num, 6539, "not all logs added to database")
ltg_num = len([gid for gid in ld.iter_ltgid()])
self.assertTrue(3 < ltg_num < 20, ("log template generation fails? "
"(groups: {0})".format(ltg_num)))
def make_crf_train(ns):
conf_load = config.open_config(ns.config_load)
conf_dump = config.open_config(ns.config_dump)
lv = logging.DEBUG if ns.debug else logging.INFO
config.set_common_logging(conf_dump, logger=_logger, lv=lv)
from . import train
from amulog import log_db
d = parse_condition(ns.conditions)
ld = log_db.LogData(conf_load)
iterobj = ld.iter_lines(**d)
print(train.crf_trainfile(conf_dump, iterobj))
def __init__(self, dt_range, conf_path, gid_name, use_mapping):
self.conf = config.open_config(conf_path)
self._ld = log_db.LogData(self.conf)
self._gid_name = gid_name
self.dt_range = dt_range
self._mapper = None
if use_mapping:
# use if tsdb is anonymized but amulog db is original
from amulog import anonymize
self._mapper = anonymize.AnonymizeMapper(self.conf)
self._mapper.load()
def show_trouble(ns):
conf = open_logdag_config(ns)
tid = ns.tid
from . import trouble
dirname = conf.get("eval", "path")
tm = trouble.TroubleManager(dirname)
from amulog import log_db
ld = log_db.LogData(conf)
tr = tm[tid]
print(tr)
print("\n".join(tr.get_message(ld, show_lid=ns.lid_header)))
def show_trouble(ns):
conf = open_logdag_config(ns)
amulog_conf = config.open_config(conf["database_amulog"]["source_conf"])
tid = ns.tid
from . import trouble
tm = trouble.init_trouble_manager(conf)
from amulog import log_db
ld = log_db.LogData(amulog_conf)
tr = tm[tid]
print(tr)
print("\n".join(tr.get_message(ld, show_lid=ns.lid_header)))
def count_node_label(conf):
from amulog import log_db
ld = log_db.LogData(conf)
from amulog import lt_label
ll = lt_label.init_ltlabel(conf)
d_group = defaultdict(int)
from logdag import showdag
for r in showdag.iter_results(conf):
r.load_ltlabel(conf, ld=ld, ll=ll)
for node in r.graph.nodes():
evdef = r.node_evdef(node)
node_group = r._label_group_ltg(evdef.gid)
d_group[node_group] += 1
return d_group
def separate_args(conf, tr):
"""Some troubles can appear among multiple days.
This function separates DAG arguments and corresponding logs.
"""
from logdag import arguments
am = arguments.ArgumentManager(conf)
am.load()
from amulog import log_db
ld = log_db.LogData(arguments.open_amulog_config(conf))
d_args = defaultdict(list)
for lid in tr.data["message"]:
lm = ld.get_line(lid)
for args in am.args_from_time(lm.dt):
name = arguments.args2name(args)
d_args[name].append(lm)
return [(arguments.name2args(name, conf), l_lm)
for name, l_lm in d_args.items()]
def count_edge_label(conf):
from amulog import log_db
ld = log_db.LogData(conf)
from amulog import lt_label
ll = lt_label.init_ltlabel(conf)
d_group = defaultdict(int)
from logdag import showdag
for r in showdag.iter_results(conf):
r.load_ltlabel(conf, ld=ld, ll=ll)
g = r.graph.to_undirected()
for edge in g.edges():
src_evdef, dst_evdef = r.edge_evdef(edge)
src_group = r._label_group_ltg(src_evdef.gid)
d_group[src_group] += 1
dst_group = r._label_group_ltg(dst_evdef.gid)
d_group[dst_group] += 1
return d_group
def log2ts(conf, dt_range):
_logger.info("make-tsdb job start ({0[0]} - {0[1]})".format(dt_range))
gid_name = conf.get("dag", "event_gid")
usefilter = conf.getboolean("database_ts", "usefilter")
top_dt, end_dt = dt_range
from amulog import log_db
ld = log_db.LogData(conf)
if gid_name == "ltid":
iterobj = ld.whole_host_lt(top_dt, end_dt, "all")
elif gid_name == "ltgid":
iterobj = ld.whole_host_ltg(top_dt, end_dt, "all")
else:
raise NotImplementedError
for host, gid in iterobj:
# load time-series from db
d = {gid_name: gid,
"host": host,
"top_dt": top_dt,
"end_dt": end_dt}
iterobj = ld.iter_lines(**d)
l_dt = sorted([line.dt for line in iterobj])
_logger.debug("gid {0}, host {1}: {2} counts".format(gid, host,
len(l_dt)))
assert len(l_dt) > 0
# apply preprocessing(filter)
evdef = (gid, host)
stat, new_l_dt, val = apply_filter(conf, ld, l_dt, dt_range, evdef)
# update database
td = TimeSeriesDB(conf, edit = True)
if new_l_dt is not None and len(new_l_dt) > 0:
for dt in new_l_dt:
td.add_line(dt, gid, host)
td.add_filterlog(dt_range, gid, host, stat, val)
td.commit()
fl = FilterLog(dt_range, gid, host, stat, val)
_logger.debug(str(fl))
_logger.info("make-tsdb job done".format(dt_range))
def show_graph_diff_lts(ns):
conf_fn1, conf_fn2 = ns.confs
conf1 = arguments.open_logdag_config(conf_fn1)
conf2 = arguments.open_logdag_config(conf_fn2)
lv = logging.DEBUG if ns.debug else logging.INFO
am_logger = logging.getLogger("amulog")
config.set_common_logging(conf1, logger=[_logger, am_logger], lv=lv)
from amulog import log_db
ld = log_db.LogData(conf1)
from . import comparison
d_ltid = comparison.edge_diff_gid(conf1, conf2)
for ltid, l_name in sorted(d_ltid.items(),
key=lambda x: len(x[1]),
reverse=True):
print("{0}: {1} ({2})".format(len(l_name), ltid, ld.lt(ltid)))
if len(l_name) < 100:
print(l_name)
def ts_filtered(conf, **kwargs):
assert "dts" in kwargs
assert "dte" in kwargs
assert "gid" in kwargs
assert "host" in kwargs
from amulog import log_db
ld = log_db.LogData(conf)
gid_name = conf.get("dag", "event_gid")
d = {"top_dt": kwargs["dts"],
"end_dt": kwargs["dte"],
gid_name: kwargs["gid"],
"host": kwargs["host"]}
l_dt = [line.dt for line in ld.iter_lines(**d)]
td = TimeSeriesDB(conf)
l_ts = [dt for dt in td.iter_ts(**kwargs)]
l_filtered = [dt for dt in l_dt if not dt in l_ts]
return l_filtered, l_ts
def list_trouble_label(ns):
conf = open_logdag_config(ns)
amulog_conf = config.open_config(conf["database_amulog"]["source_conf"])
from . import trouble
tm = trouble.init_trouble_manager(conf)
from amulog import log_db
ld = log_db.LogData(amulog_conf)
gid_name = conf.get("database_amulog", "event_gid")
for tr in tm:
d_ev, d_gid, d_host = trouble.event_stat(tr, ld, gid_name)
d_group = trouble.event_label(d_gid, ld, gid_name)
buf = "{0} ({1}): ".format(tr.tid, tr.data["group"])
for group, l_gid in sorted(d_group.items(),
key=lambda x: len(x[1]),
reverse=True):
num = sum([d_gid[gid] for gid in l_gid])
buf += "{0}({1},{2}) ".format(group, len(l_gid), num)
print(buf)
def show_graph_diff_lts(ns):
l_conffp = ns.confs
assert len(l_conffp) == 2
openconf = lambda c: config.open_config(
c, ex_defaults=[arguments.DEFAULT_CONFIG])
conf1, conf2 = [openconf(c) for c in l_conffp]
lv = logging.DEBUG if ns.debug else logging.INFO
am_logger = logging.getLogger("amulog")
config.set_common_logging(conf1, logger=[_logger, am_logger], lv=lv)
from amulog import log_db
ld = log_db.LogData(conf1)
from . import comp_conf
d_ltid = comp_conf.edge_diff_gid(conf1, conf2)
for ltid, l_name in sorted(d_ltid.items(),
key=lambda x: len(x[1]),
reverse=True):
print("{0}: {1} ({2})".format(len(l_name), ltid, ld.lt(ltid)))
if len(l_name) < 100:
print(l_name)
def list_trouble_label(ns):
conf = open_logdag_config(ns)
from . import trouble
dirname = conf.get("eval", "path")
tm = trouble.TroubleManager(dirname)
from amulog import log_db
ld = log_db.LogData(conf)
from amulog import lt_label
ll = lt_label.init_ltlabel(conf)
gid_name = conf.get("dag", "event_gid")
for tr in tm:
d_ev, d_gid, d_host = trouble.event_stat(tr, ld, gid_name)
d_group = trouble.event_label(d_gid, ld, ll)
buf = "{0} ({1}): ".format(tr.tid, tr.data["group"])
for group, l_gid in sorted(d_group.items(),
key=lambda x: len(x[1]),
reverse=True):
num = sum([d_gid[gid] for gid in l_gid])
buf += "{0}({1},{2}) ".format(group, len(l_gid), num)
print(buf)
def all_args(conf):
amulog_conf = config.open_config(conf["database_amulog"]["source_conf"])
from amulog import log_db
ld = log_db.LogData(amulog_conf)
w_top_dt, w_end_dt = config.getterm(conf, "dag", "whole_term")
term = config.getdur(conf, "dag", "unit_term")
diff = config.getdur(conf, "dag", "unit_diff")
l_args = []
top_dt = w_top_dt
while top_dt < w_end_dt:
end_dt = top_dt + term
l_area = config.getlist(conf, "dag", "area")
if "each" in l_area:
l_area.pop(l_area.index("each"))
l_area += [
"host_" + host for host in ld.whole_host(top_dt, end_dt)
]
for area in l_area:
l_args.append((conf, (top_dt, end_dt), area))
top_dt = top_dt + diff
return l_args
def test_makedb_parallel(self):
import copy
conf = copy.copy(self._conf)
conf["manager"]["n_process"] = "2"
conf["log_template"]["lt_methods"] = "re"
conf["log_template_re"]["variable_rule"] = \
common.filepath_local(__file__, "test_re.conf")
from amulog import __main__ as amulog_main
targets = amulog_main.get_targets_conf(conf)
manager.process_files_offline(conf,
targets,
reset_db=True,
parallel=True)
ld = log_db.LogData(self._conf)
#for ltobj in ld.iter_lt():
# print(ltobj)
num = ld.count_lines()
self.assertEqual(num, 6539, "not all logs added to database")
ltg_num = len([gid for gid in ld.iter_ltgid()])
self.assertTrue(3 < ltg_num < 20, ("log template generation fails? "
"(groups: {0})".format(ltg_num)))
def show_trouble_info(ns):
conf = open_logdag_config(ns)
amulog_conf = config.open_config(conf["database_amulog"]["source_conf"])
tid = ns.tid
from . import trouble
tm = trouble.init_trouble_manager(conf)
from amulog import log_db
ld = log_db.LogData(amulog_conf)
gid_name = conf.get("database_amulog", "event_gid")
tr = tm[tid]
d_ev, d_gid, d_host = trouble.event_stat(tr, ld, gid_name)
d_group = trouble.event_label(d_gid, ld, gid_name)
print(tr)
print("{0} related events".format(len(d_ev)))
print("{0} related hosts: {1}".format(len(d_host), sorted(d_host.keys())))
print("{0} related templates: {1}".format(len(d_gid),
sorted(d_gid.keys())))
for group, l_gid in d_group.items():
num = sum([d_gid[gid] for gid in l_gid])
print(" group {0}: {1} messages, {2} templates {3}".format(
group, num, len(l_gid), l_gid))
def remake_ltgroup(conf):
ld = log_db.LogData(conf, edit=True, reset_db=False)
ltm = LTManager(conf, ld.db, ld.lttable, reset_db=False)
ltm.remake_ltg()
ltm.commit_db()
ACCEPT_SYM = False
if len(sys.argv) < 4:
sys.exit("usage: {0} CONFIG RULE1 RULE2".format(sys.argv[0]))
conf = config.open_config(sys.argv[1])
if ACCEPT_SYM:
sym = conf.get("log_template", "variable_symbol")
else:
sym = None
d_rule1 = parse_condition(sys.argv[2].split(","))
d_rule2 = parse_condition(sys.argv[3].split(","))
s_ltid1 = set()
s_ltid2 = set()
ld = log_db.LogData(conf)
for lm in ld.iter_lines(**d_rule1):
s_ltid1.add(lm.lt.ltid)
for lm in ld.iter_lines(**d_rule2):
s_ltid2.add(lm.lt.ltid)
common = s_ltid1 & s_ltid2
print("{0} common log template found... ".format(len(common)))
d_ed1 = {}
d_ed2 = {}
for key in common:
d_ed1[key] = 0
d_ed2[key] = 0
s_ltid1.remove(key)
def __init__(self, dt_range, conf_fn, gid_name):
self.conf = config.open_config(conf_fn)
self._ld = log_db.LogData(self.conf)
self._gid_name = gid_name
self.dt_range = dt_range
self._ll = None
