def __init__(self, conf, dry=False): super().__init__(conf, dry=dry) src = conf["general"]["log_source"] if src == "amulog": from . import src_amulog args = [ config.getterm(conf, "general", "evdb_whole_term"), conf["database_amulog"]["source_conf"], conf["database_amulog"]["event_gid"], conf.getboolean("database_amulog", "use_anonymize_mapping") ] self.source = src_amulog.AmulogLoader(*args) else: raise NotImplementedError self._filter_rules = config.getlist(conf, "filter", "rules") for method in self._filter_rules: assert method in filter_log.FUNCTIONS self.evdb = self._init_evdb(conf, "log_dbname") # dst = conf["general"]["evdb"] # if dst == "influx": # dbname = conf["database_influx"]["log_dbname"] # from . import influx # self.evdb = influx.init_influx(conf, dbname, df=False) # # self.evdb_df = influx.init_influx(conf, dbname, df = True) # else: # raise NotImplementedError self._lf = None if len(self._filter_rules) > 0: self._lf = filter_log.init_logfilter(conf, self.source) self._feature_unit_diff = config.getdur(conf, "general", "evdb_unit_diff") self._given_amulog_database = conf["database_amulog"]["given_amulog_database"]
def __init__(self, conf, dry=False): self.conf = conf self.dry = dry src = conf["general"]["log_source"] if src == "amulog": from . import source_amulog args = [config.getterm(conf, "general", "evdb_whole_term"), conf["database_amulog"]["source_conf"], conf["database_amulog"]["event_gid"]] self.source = source_amulog.AmulogLoader(*args) else: raise NotImplementedError self._filter_rules = config.getlist(conf, "filter", "rules") for method in self._filter_rules: assert method in filter_log.FUNCTIONS dst = conf["general"]["evdb"] if dst == "influx": dbname = conf["database_influx"]["log_dbname"] from . import influx self.evdb = influx.init_influx(conf, dbname, df=False) # self.evdb_df = influx.init_influx(conf, dbname, df = True) else: raise NotImplementedError self._lf = filter_log.init_logfilter(conf, self.source) self._feature_unit_diff = config.getdur(conf, "general", "evdb_unit_diff")
def all_terms(conf, term, diff, w_term=None): w_top_dt, w_end_dt = config.getterm(conf, "dag", "whole_term") l_args = [] top_dt = w_top_dt while top_dt < w_end_dt: end_dt = top_dt + term l_args.append((conf, (top_dt, end_dt))) top_dt = top_dt + diff return l_args
def all_args(conf): w_top_dt, w_end_dt = config.getterm(conf, "dag", "whole_term") term = config.getdur(conf, "dag", "unit_term") diff = config.getdur(conf, "dag", "unit_diff") l_args = [] top_dt = w_top_dt while top_dt < w_end_dt: end_dt = top_dt + term l_area = config.getlist(conf, "dag", "area") for area in l_area: l_args.append((conf, (top_dt, end_dt), area)) top_dt = top_dt + diff return l_args
def make_evdb_log_all(ns): conf = open_logdag_config(ns) dump_org = ns.org dry = ns.dry timer = common.Timer("make-evdb-log task", output=_logger) timer.start() from . import evgen_log w_term = config.getterm(conf, "general", "evdb_whole_term") term = config.getdur(conf, "general", "evdb_unit_diff") el = evgen_log.LogEventLoader(conf, dry=dry) for dt_range in dtutil.iter_term(w_term, term): el.read(dt_range, dump_org=dump_org) timer.lap_diff("{0}".format(dt_range)) timer.stop()
def match_edges(conf, tr, rule="all", cond=None): def _pass_condition(edge_evdef, cond): if cond is None: return True elif cond == "xhost": src_evdef, dst_evdef = edge_evdef return not src_evdef.host == dst_evdef.host else: raise NotImplementedError def _lm2ev(lm, gid_name): gid = lm.lt.get(gid_name) d = { "source": "log", "gid": gid, "host": lm.host, "group": al.label(gid) } return evgen_log.LogEventDefinition(**d) from amulog import config from logdag.source import source_amulog from logdag.source import evgen_log dt_range = config.getterm(conf, "dag", "whole_term") al = source_amulog.init_amulogloader(conf, dt_range) gid_name = conf.get("database_amulog", "event_gid") d = defaultdict(list) for args, l_lm in separate_args(conf, tr): r = showdag.LogDAG(args) r.load() g = r.graph.to_undirected() for edge in g.edges(): edevdef = r.edge_evdef(edge) if not _pass_condition(edevdef, cond): continue s_evdef = {str(_lm2ev(lm, gid_name)) for lm in l_lm} if _match_edge(s_evdef, edevdef, rule): d[r.name].append(edge) return d
def all_args(conf): amulog_conf = config.open_config(conf["database_amulog"]["source_conf"]) from amulog import log_db ld = log_db.LogData(amulog_conf) w_top_dt, w_end_dt = config.getterm(conf, "dag", "whole_term") term = config.getdur(conf, "dag", "unit_term") diff = config.getdur(conf, "dag", "unit_diff") l_args = [] top_dt = w_top_dt while top_dt < w_end_dt: end_dt = top_dt + term l_area = config.getlist(conf, "dag", "area") if "each" in l_area: l_area.pop(l_area.index("each")) l_area += [ "host_" + host for host in ld.whole_host(top_dt, end_dt) ] for area in l_area: l_args.append((conf, (top_dt, end_dt), area)) top_dt = top_dt + diff return l_args
def __init__(self, conf_fn, seed=None): if seed is None: random.seed() else: random.seed(seed) self.conf = configparser.ConfigParser() self.conf.read(conf_fn) self.term = config.getterm(self.conf, "main", "term") self.top_dt, self.end_dt = self.term self.d_host = {} for group in config.gettuple(self.conf, "main", "host_groups"): for host in config.gettuple(self.conf, "main", "group_" + group): self.d_host.setdefault(group, []).append(host) self.l_event = [] for event_name in config.gettuple(self.conf, "main", "events"): self._generate_event(event_name) self.l_log = [] for event in self.l_event: self._generate_log(event)
def search_valid(conf, path, th=1.0): import rrdtool import numpy as np from amulog import common for fp in common.recur_dir(path): ut_range = [ dt.timestamp() for dt in config.getterm(conf, "general", "whole_term") ] try: robj = rrd.fetch(fp, ut_range) except IOError as e: sys.stderr(e) except rrdtool.OperationalError as e: pass else: nanratio = np.mean( [int(np.isnan(v)) for v in robj.values.reshape(-1, )]) if nanratio < th: yield fp else: pass
def match_edges(conf, tr, rule="all", cond=None): def _pass_condition(edge_evdef, condition): if condition is None: return True elif condition == "xhost": src_evdef, dst_evdef = edge_evdef return not src_evdef.host == dst_evdef.host else: raise NotImplementedError from amulog import config from logdag.source import src_amulog from logdag.source import evgen_log dt_range = config.getterm(conf, "dag", "whole_term") al = src_amulog.init_amulogloader(conf, dt_range) gid_name = conf.get("database_amulog", "event_gid") d_results = defaultdict(list) for args, l_lm in separate_args(conf, tr): s_evdef = set() for lm in l_lm: gid = lm.lt.get(gid_name) evdef = evgen_log.LogEventDefinition(source="log", gid=gid, host=lm.host, group=al.group(gid)) s_evdef = s_evdef | set(evdef.member_identifiers()) r = showdag.LogDAG(args) r.load() g = r.graph.to_undirected() for edge in g.edges(): edevdef = r.edge_evdef(edge) if _pass_condition(edevdef, cond) and \ _match_edge(s_evdef, edevdef, rule): d_results[r.name].append(edge) return d_results
def _whole_term(conf): return config.getterm(conf, "general", "evdb_whole_term")
def _iter_evdb_term(conf): w_term = config.getterm(conf, "general", "evdb_whole_term") term = config.getdur(conf, "general", "evdb_unit_diff") return dtutil.iter_term(w_term, term)