def __init__(self, conf, dry=False):
super().__init__(conf, dry=dry)
src = conf["general"]["log_source"]
if src == "amulog":
from . import src_amulog
args = [
config.getterm(conf, "general", "evdb_whole_term"),
conf["database_amulog"]["source_conf"],
conf["database_amulog"]["event_gid"],
conf.getboolean("database_amulog",
"use_anonymize_mapping")
]
self.source = src_amulog.AmulogLoader(*args)
else:
raise NotImplementedError
self._filter_rules = config.getlist(conf, "filter", "rules")
for method in self._filter_rules:
assert method in filter_log.FUNCTIONS
self.evdb = self._init_evdb(conf, "log_dbname")
# dst = conf["general"]["evdb"]
# if dst == "influx":
# dbname = conf["database_influx"]["log_dbname"]
# from . import influx
# self.evdb = influx.init_influx(conf, dbname, df=False)
# # self.evdb_df = influx.init_influx(conf, dbname, df = True)
# else:
# raise NotImplementedError
self._lf = None
if len(self._filter_rules) > 0:
self._lf = filter_log.init_logfilter(conf, self.source)
self._feature_unit_diff = config.getdur(conf,
"general", "evdb_unit_diff")
self._given_amulog_database = conf["database_amulog"]["given_amulog_database"]
def __init__(self, conf, dry=False):
self.conf = conf
self.dry = dry
src = conf["general"]["log_source"]
if src == "amulog":
from . import source_amulog
args = [config.getterm(conf, "general", "evdb_whole_term"),
conf["database_amulog"]["source_conf"],
conf["database_amulog"]["event_gid"]]
self.source = source_amulog.AmulogLoader(*args)
else:
raise NotImplementedError
self._filter_rules = config.getlist(conf, "filter", "rules")
for method in self._filter_rules:
assert method in filter_log.FUNCTIONS
dst = conf["general"]["evdb"]
if dst == "influx":
dbname = conf["database_influx"]["log_dbname"]
from . import influx
self.evdb = influx.init_influx(conf, dbname, df=False)
# self.evdb_df = influx.init_influx(conf, dbname, df = True)
else:
raise NotImplementedError
self._lf = filter_log.init_logfilter(conf, self.source)
self._feature_unit_diff = config.getdur(conf,
"general", "evdb_unit_diff")
def all_terms(conf, term, diff, w_term=None):
w_top_dt, w_end_dt = config.getterm(conf, "dag", "whole_term")
l_args = []
top_dt = w_top_dt
while top_dt < w_end_dt:
end_dt = top_dt + term
l_args.append((conf, (top_dt, end_dt)))
top_dt = top_dt + diff
return l_args
def all_args(conf):
w_top_dt, w_end_dt = config.getterm(conf, "dag", "whole_term")
term = config.getdur(conf, "dag", "unit_term")
diff = config.getdur(conf, "dag", "unit_diff")
l_args = []
top_dt = w_top_dt
while top_dt < w_end_dt:
end_dt = top_dt + term
l_area = config.getlist(conf, "dag", "area")
for area in l_area:
l_args.append((conf, (top_dt, end_dt), area))
top_dt = top_dt + diff
return l_args
def make_evdb_log_all(ns):
conf = open_logdag_config(ns)
dump_org = ns.org
dry = ns.dry
timer = common.Timer("make-evdb-log task", output=_logger)
timer.start()
from . import evgen_log
w_term = config.getterm(conf, "general", "evdb_whole_term")
term = config.getdur(conf, "general", "evdb_unit_diff")
el = evgen_log.LogEventLoader(conf, dry=dry)
for dt_range in dtutil.iter_term(w_term, term):
el.read(dt_range, dump_org=dump_org)
timer.lap_diff("{0}".format(dt_range))
timer.stop()
def match_edges(conf, tr, rule="all", cond=None):
def _pass_condition(edge_evdef, cond):
if cond is None:
return True
elif cond == "xhost":
src_evdef, dst_evdef = edge_evdef
return not src_evdef.host == dst_evdef.host
else:
raise NotImplementedError
def _lm2ev(lm, gid_name):
gid = lm.lt.get(gid_name)
d = {
"source": "log",
"gid": gid,
"host": lm.host,
"group": al.label(gid)
}
return evgen_log.LogEventDefinition(**d)
from amulog import config
from logdag.source import source_amulog
from logdag.source import evgen_log
dt_range = config.getterm(conf, "dag", "whole_term")
al = source_amulog.init_amulogloader(conf, dt_range)
gid_name = conf.get("database_amulog", "event_gid")
d = defaultdict(list)
for args, l_lm in separate_args(conf, tr):
r = showdag.LogDAG(args)
r.load()
g = r.graph.to_undirected()
for edge in g.edges():
edevdef = r.edge_evdef(edge)
if not _pass_condition(edevdef, cond):
continue
s_evdef = {str(_lm2ev(lm, gid_name)) for lm in l_lm}
if _match_edge(s_evdef, edevdef, rule):
d[r.name].append(edge)
return d
def all_args(conf):
amulog_conf = config.open_config(conf["database_amulog"]["source_conf"])
from amulog import log_db
ld = log_db.LogData(amulog_conf)
w_top_dt, w_end_dt = config.getterm(conf, "dag", "whole_term")
term = config.getdur(conf, "dag", "unit_term")
diff = config.getdur(conf, "dag", "unit_diff")
l_args = []
top_dt = w_top_dt
while top_dt < w_end_dt:
end_dt = top_dt + term
l_area = config.getlist(conf, "dag", "area")
if "each" in l_area:
l_area.pop(l_area.index("each"))
l_area += [
"host_" + host for host in ld.whole_host(top_dt, end_dt)
]
for area in l_area:
l_args.append((conf, (top_dt, end_dt), area))
top_dt = top_dt + diff
return l_args
def __init__(self, conf_fn, seed=None):
if seed is None:
random.seed()
else:
random.seed(seed)
self.conf = configparser.ConfigParser()
self.conf.read(conf_fn)
self.term = config.getterm(self.conf, "main", "term")
self.top_dt, self.end_dt = self.term
self.d_host = {}
for group in config.gettuple(self.conf, "main", "host_groups"):
for host in config.gettuple(self.conf, "main", "group_" + group):
self.d_host.setdefault(group, []).append(host)
self.l_event = []
for event_name in config.gettuple(self.conf, "main", "events"):
self._generate_event(event_name)
self.l_log = []
for event in self.l_event:
self._generate_log(event)
def search_valid(conf, path, th=1.0):
import rrdtool
import numpy as np
from amulog import common
for fp in common.recur_dir(path):
ut_range = [
dt.timestamp()
for dt in config.getterm(conf, "general", "whole_term")
]
try:
robj = rrd.fetch(fp, ut_range)
except IOError as e:
sys.stderr(e)
except rrdtool.OperationalError as e:
pass
else:
nanratio = np.mean(
[int(np.isnan(v)) for v in robj.values.reshape(-1, )])
if nanratio < th:
yield fp
else:
pass
def match_edges(conf, tr, rule="all", cond=None):
def _pass_condition(edge_evdef, condition):
if condition is None:
return True
elif condition == "xhost":
src_evdef, dst_evdef = edge_evdef
return not src_evdef.host == dst_evdef.host
else:
raise NotImplementedError
from amulog import config
from logdag.source import src_amulog
from logdag.source import evgen_log
dt_range = config.getterm(conf, "dag", "whole_term")
al = src_amulog.init_amulogloader(conf, dt_range)
gid_name = conf.get("database_amulog", "event_gid")
d_results = defaultdict(list)
for args, l_lm in separate_args(conf, tr):
s_evdef = set()
for lm in l_lm:
gid = lm.lt.get(gid_name)
evdef = evgen_log.LogEventDefinition(source="log",
gid=gid,
host=lm.host,
group=al.group(gid))
s_evdef = s_evdef | set(evdef.member_identifiers())
r = showdag.LogDAG(args)
r.load()
g = r.graph.to_undirected()
for edge in g.edges():
edevdef = r.edge_evdef(edge)
if _pass_condition(edevdef, cond) and \
_match_edge(s_evdef, edevdef, rule):
d_results[r.name].append(edge)
return d_results
def _whole_term(conf):
return config.getterm(conf, "general", "evdb_whole_term")
def _iter_evdb_term(conf):
w_term = config.getterm(conf, "general", "evdb_whole_term")
term = config.getdur(conf, "general", "evdb_unit_diff")
return dtutil.iter_term(w_term, term)