def _count_all(self):
self._d_evdef_args = defaultdict(list)
self._d_evdef_count = defaultdict(dict)
self._d_total = defaultdict(int)
for args in self._am:
ldag = showdag.LogDAG(args)
ldag.load()
jobname = self._am.jobname(args)
for node in ldag.graph.nodes():
key = self.node_key(node, ldag)
count = ldag.node_count(node)
self._d_evdef_args[key].append(jobname)
self._d_evdef_count[jobname][key] = count
self._d_total[jobname] += count
def edge_set_common(conf1, conf2, dt_range):
am1 = arguments.ArgumentManager(conf1)
am1.load()
am2 = arguments.ArgumentManager(conf2)
am2.load()
temp_cevmap = log2event.EventDefinitionMap()
temp_cgraph = nx.Graph()
for args in am1.args_in_time(dt_range):
r1 = showdag.LogDAG(args)
r1.load()
temp_cevmap = _add_nodes(temp_cevmap, r1)
temp_cgraph = _add_edges(temp_cevmap, temp_cgraph, r1)
cevmap = log2event.EventDefinitionMap()
cgraph = nx.Graph()
for args in am2.args_in_time(dt_range):
r2 = showdag.LogDAG(args)
r2.load()
g = r2.graph.to_undirected()
for edge in g.edges():
src_evdef, dst_evdef = r2.edge_evdef(edge)
if temp_cevmap.has_evdef(src_evdef) \
and temp_cevmap.has_evdef(dst_evdef):
temp_src_eid = temp_cevmap.get_eid(src_evdef)
temp_dst_eid = temp_cevmap.get_eid(dst_evdef)
if temp_cgraph.has_edge(temp_src_eid, temp_dst_eid):
if cevmap.has_evdef(src_evdef):
new_src_eid = cevmap.get_eid(src_evdef)
else:
new_src_eid = cevmap.add_evdef(src_evdef)
if cevmap.has_evdef(dst_evdef):
new_dst_eid = cevmap.get_eid(dst_evdef)
else:
new_dst_eid = cevmap.add_evdef(dst_evdef)
cgraph.add_edge(new_src_eid, new_dst_eid)
return cevmap, cgraph
def show_match(ns):
conf = open_logdag_config(ns)
from . import trouble
tm = trouble.init_trouble_manager(conf)
from logdag import showdag
from . import match_edge
tr = tm[ns.tid]
d_args = match_edge.match_edges(conf, tr, rule=ns.rule, cond=ns.cond)
cnt = sum([len(l_edge) for l_edge in d_args.values()])
print("{0[date]} ({0[group]}): {1}".format(tr.data, cnt))
for name, l_edge in d_args.items():
r = showdag.LogDAG(arguments.name2args(name, conf))
r.load()
for edge in l_edge:
edgestr = r.edge_str(edge, graph=r.graph.to_undirected())
print(name, edgestr)
def match_edges(conf, tr, rule="all", cond=None):
def _pass_condition(edge_evdef, cond):
if cond is None:
return True
elif cond == "xhost":
src_evdef, dst_evdef = edge_evdef
return not src_evdef.host == dst_evdef.host
else:
raise NotImplementedError
def _lm2ev(lm, gid_name):
gid = lm.lt.get(gid_name)
d = {
"source": "log",
"gid": gid,
"host": lm.host,
"group": al.label(gid)
}
return evgen_log.LogEventDefinition(**d)
from amulog import config
from logdag.source import source_amulog
from logdag.source import evgen_log
dt_range = config.getterm(conf, "dag", "whole_term")
al = source_amulog.init_amulogloader(conf, dt_range)
gid_name = conf.get("database_amulog", "event_gid")
d = defaultdict(list)
for args, l_lm in separate_args(conf, tr):
r = showdag.LogDAG(args)
r.load()
g = r.graph.to_undirected()
for edge in g.edges():
edevdef = r.edge_evdef(edge)
if not _pass_condition(edevdef, cond):
continue
s_evdef = {str(_lm2ev(lm, gid_name)) for lm in l_lm}
if _match_edge(s_evdef, edevdef, rule):
d[r.name].append(edge)
return d
def _get_direction(am, ev1, ev2, dt_range):
for tmp_args in am.args_in_time(dt_range):
r = showdag.LogDAG(tmp_args)
r.load()
if not r._evmap_original().has_evdef(ev1):
continue
if not r._evmap_original().has_evdef(ev2):
continue
n1, n2 = [r.evdef2node(ev) for ev in (ev1, ev2)]
if (n1, n2) in r.graph.edges():
if (n2, n1) in r.graph.edges():
di = "-"
else:
di = "->"
elif (n2, n1) in r.graph.edges():
di = "<-"
return di
else:
raise ValueError("Edge {0} - {1} not found in {2}".format(
ev1, ev2, r.name))
def dag_anomaly_score(conf, feature="edge", score="tfidf"):
am = arguments.ArgumentManager(conf)
am.load()
counter = init_counter(conf, feature, am=am)
d_score = {}
for args in am:
jobname = am.jobname(args)
ldag = showdag.LogDAG(args)
ldag.load()
edges = showdag.remove_edge_duplication(ldag.graph.edges(), ldag)
score = sum(
edges_anomaly_score(edges,
ldag,
feature=feature,
score=score,
counter=counter,
am=am))
d_score[jobname] = score
return d_score
def edge_direction_diff(conf1, conf2, dt_range):
def _get_direction(am, ev1, ev2, dt_range):
for tmp_args in am.args_in_time(dt_range):
r = showdag.LogDAG(tmp_args)
r.load()
if not r._evmap().has_evdef(ev1):
continue
if not r._evmap().has_evdef(ev2):
continue
n1, n2 = [r.evdef2node(ev) for ev in (ev1, ev2)]
if (n1, n2) in r.graph.edges():
if (n2, n1) in r.graph.edges():
di = "-"
else:
di = "->"
elif (n2, n1) in r.graph.edges():
di = "<-"
return di
else:
raise ValueError("Edge {0} - {1} not found in {2}".format(
ev1, ev2, r.name))
ret = []
am1 = arguments.ArgumentManager(conf1)
am1.load()
am2 = arguments.ArgumentManager(conf2)
am2.load()
for args in am1.args_in_time(dt_range):
r2 = showdag.LogDAG(args)
cevmap, cgraph = edge_set_common(conf1, conf2, dt_range)
for edge in cgraph.edges():
ev1, ev2 = [cevmap.evdef(node) for node in edge]
di1 = _get_direction(am1, ev1, ev2, dt_range)
di2 = _get_direction(am2, ev1, ev2, dt_range)
if di1 == di2:
pass
else:
ret.append([ev1, ev2, di1, di2])
return ret
def match_edges(conf, tr, rule="all", cond=None):
def _pass_condition(edge_evdef, condition):
if condition is None:
return True
elif condition == "xhost":
src_evdef, dst_evdef = edge_evdef
return not src_evdef.host == dst_evdef.host
else:
raise NotImplementedError
from amulog import config
from logdag.source import src_amulog
from logdag.source import evgen_log
dt_range = config.getterm(conf, "dag", "whole_term")
al = src_amulog.init_amulogloader(conf, dt_range)
gid_name = conf.get("database_amulog", "event_gid")
d_results = defaultdict(list)
for args, l_lm in separate_args(conf, tr):
s_evdef = set()
for lm in l_lm:
gid = lm.lt.get(gid_name)
evdef = evgen_log.LogEventDefinition(source="log",
gid=gid,
host=lm.host,
group=al.group(gid))
s_evdef = s_evdef | set(evdef.member_identifiers())
r = showdag.LogDAG(args)
r.load()
g = r.graph.to_undirected()
for edge in g.edges():
edevdef = r.edge_evdef(edge)
if _pass_condition(edevdef, cond) and \
_match_edge(s_evdef, edevdef, rule):
d_results[r.name].append(edge)
return d_results
def show_edges_temporal_sort(ns):
conf = open_logdag_config(ns)
args = arguments.name2args(ns.argname, conf)
from . import edge_search
ldag = showdag.LogDAG(args)
ldag.load()
g = showdag.apply_filter(ldag, ns.filters, th=ns.threshold)
if ns.detail:
context = "detail"
elif ns.instruction:
context = "instruction"
else:
context = "edge"
from logdag import dtutil
if ns.timestr_end is None:
dt = dtutil.shortstr2dt(ns.timestr)
time_condition = {"time": dt}
else:
dts = dtutil.shortstr2dt(ns.timestr)
dte = dtutil.shortstr2dt(ns.timestr_end)
time_condition = {"time_range": (dts, dte)}
from ..__main__ import _parse_condition
search_condition = _parse_condition(ns.conditions)
print(
edge_search.edge_temporal_sort(ldag,
time_condition,
search_condition=search_condition,
reverse=ns.reverse,
view_context=context,
load_cache=(not ns.nocache),
graph=g))
def _dag_from_name(conf, name):
args = arguments.name2args(name, conf)
r = showdag.LogDAG(args)
r.load()
return r