def show_diff_info(ns): conf_fn1, conf_fn2 = ns.confs conf1 = arguments.open_logdag_config(conf_fn1) conf2 = arguments.open_logdag_config(conf_fn2) lv = logging.DEBUG if ns.debug else logging.INFO am_logger = logging.getLogger("amulog") config.set_common_logging(conf1, logger=[_logger, am_logger], lv=lv) from . import comparison d = defaultdict(int) am = arguments.ArgumentManager(conf1) am.load() for dt_range in am.iter_dt_range(): cevmap_common, cgraph_common = comparison.edge_set_common( conf1, conf2, dt_range) d["common"] += cgraph_common.number_of_edges() cevmap_lor, cgraph_lor = comparison.edge_set_lor( conf1, conf2, dt_range) d["lor"] += cgraph_lor.number_of_edges() cevmap_diff1, cgraph_diff1 = comparison.edge_set_diff(conf1, conf2, dt_range, lor=(cevmap_lor, cgraph_lor)) d["diff1"] += cgraph_diff1.number_of_edges() cevmap_diff2, cgraph_diff2 = comparison.edge_set_diff(conf2, conf1, dt_range, lor=(cevmap_lor, cgraph_lor)) d["diff2"] += cgraph_diff2.number_of_edges() print("Logical OR edges: {0}".format(d["lor"])) print("Common edges: {0}".format(d["common"])) print("Edges only found in {0}: {1}".format(ns.confs[0], d["diff1"])) print("Edges only found in {0}: {1}".format(ns.confs[1], d["diff2"]))
def show_diff_direction(ns): l_conffp = ns.confs assert len(l_conffp) == 2 openconf = lambda c: config.open_config( c, ex_defaults=[arguments.DEFAULT_CONFIG]) conf1, conf2 = [openconf(c) for c in l_conffp] lv = logging.DEBUG if ns.debug else logging.INFO am_logger = logging.getLogger("amulog") config.set_common_logging(conf1, logger=[_logger, am_logger], lv=lv) def _print_diff(ret): for ev1, ev2, di1, di2 in ret: print("{0} {1} | {2} {3}".format(ev1, di1, di2, ev2)) cnt = 0 from . import comp_conf am = arguments.ArgumentManager(conf1) am.load() if ns.argname is None: for dt_range in sorted(am.iter_dt_range()): ret = comp_conf.edge_direction_diff(conf1, conf2, dt_range) cnt += len(ret) if len(ret) > 0: print(dt_range) _print_diff(ret) print("") else: args = am.jobname2args(ns.argname, conf) dt_range = args[2] ret = comp_conf.edge_direction_diff(conf1, conf2, dt_range) cnt += len(ret) _print_diff(ret) print(cnt)
def show_diff_edges(ns): conf_fn1, conf_fn2 = ns.confs conf1 = arguments.open_logdag_config(conf_fn1) conf2 = arguments.open_logdag_config(conf_fn2) lv = logging.DEBUG if ns.debug else logging.INFO am_logger = logging.getLogger("amulog") config.set_common_logging(conf1, logger=[_logger, am_logger], lv=lv) from . import comparison am = arguments.ArgumentManager(conf1) am.load() for dt_range in sorted(am.iter_dt_range()): cevmap, cgraph = comparison.edge_set_diff(conf1, conf2, dt_range) buf_edges = [] for edge in cgraph.edges(): buf = "" src_info = cevmap.evdef(edge[0]) buf += "[gid={0[0]}, host = {0[1]}]".format(src_info) if showdag.isdirected(edge, cgraph): buf += " -> " else: buf += " <-> " dst_info = cevmap.evdef(edge[1]) buf += "[gid={0[0]}, host = {0[1]}]".format(dst_info) buf_edges.append(buf) if len(buf_edges) > 0: print("date: {0}".format(dt_range[0])) print("\n".join(buf_edges))
def show_diff_edges(ns): l_conffp = ns.confs assert len(l_conffp) == 2 openconf = lambda c: config.open_config( c, ex_defaults=[arguments.DEFAULT_CONFIG]) conf1, conf2 = [openconf(c) for c in l_conffp] lv = logging.DEBUG if ns.debug else logging.INFO am_logger = logging.getLogger("amulog") config.set_common_logging(conf1, logger=[_logger, am_logger], lv=lv) from . import comp_conf am = arguments.ArgumentManager(conf1) am.load() for dt_range in sorted(am.iter_dt_range()): cevmap, cgraph = comp_conf.edge_set_diff(conf1, conf2, dt_range) buf_edges = [] for edge in cgraph.edges(): buf = "" src_info = cevmap.evdef(edge[0]) buf += "[gid={0[0]}, host = {0[1]}]".format(src_info) if showdag.isdirected(edge, cgraph): buf += " -> " else: buf += " <-> " dst_info = cevmap.evdef(edge[1]) buf += "[gid={0[0]}, host = {0[1]}]".format(dst_info) buf_edges.append(buf) if len(buf_edges) > 0: print("date: {0}".format(dt_range[0])) print("\n".join(buf_edges))
def make_crf_model(ns): conf_load = config.open_config(ns.config_load) conf_dump = config.open_config(ns.config_dump) lv = logging.DEBUG if ns.debug else logging.INFO config.set_common_logging(conf_dump, logger=_logger, lv=lv) from . import train output = ns.output if ns.train_file is None: from amulog import log_db d = parse_condition(ns.conditions) output_sampled = ns.output_sampled get_output_sampled = output_sampled is not None ld = log_db.LogData(conf_load) iterobj = [lm for lm in ld.iter_lines(**d)] if get_output_sampled: fn, l_train = train.make_crf_model(conf_dump, iterobj, output, return_sampled_messages=True) import pickle with open(output_sampled, 'wb') as f: pickle.dump(l_train, f) print("> {0}".format(output_sampled)) else: fn = train.make_crf_model(conf_dump, iterobj, output) else: fn = train.make_crf_model_from_trainfile(conf_dump, ns.train_file, output) print("> {0}".format(fn))
def show_diff_direction(ns): conf_fn1, conf_fn2 = ns.confs conf1 = arguments.open_logdag_config(conf_fn1) conf2 = arguments.open_logdag_config(conf_fn2) lv = logging.DEBUG if ns.debug else logging.INFO am_logger = logging.getLogger("amulog") config.set_common_logging(conf1, logger=[_logger, am_logger], lv=lv) def _print_diff(_ret): for ev1, ev2, di1, di2 in _ret: print("{0} {1} | {2} {3}".format(ev1, di1, di2, ev2)) cnt = 0 from . import comparison am = arguments.ArgumentManager(conf1) am.load() if ns.argname is None: for dt_range in sorted(am.iter_dt_range()): ret = comparison.edge_direction_diff(conf1, conf2, dt_range) cnt += len(ret) if len(ret) > 0: print(dt_range) _print_diff(ret) print("") else: args = am.jobname2args(ns.argname, conf1) dt_range = args[2] ret = comparison.edge_direction_diff(conf1, conf2, dt_range) cnt += len(ret) _print_diff(ret) print(cnt)
def show_graph_diff_search(ns): conf_fn1, conf_fn2 = ns.confs conf1 = arguments.open_logdag_config(conf_fn1) conf2 = arguments.open_logdag_config(conf_fn2) lv = logging.DEBUG if ns.debug else logging.INFO am_logger = logging.getLogger("amulog") config.set_common_logging(conf1, logger=[_logger, am_logger], lv=lv) gid = ns.gid from . import comparison comparison.edge_diff_gid_search(conf1, conf2, gid)
def make_crf_train(ns): conf_load = config.open_config(ns.config_load) conf_dump = config.open_config(ns.config_dump) lv = logging.DEBUG if ns.debug else logging.INFO config.set_common_logging(conf_dump, logger=_logger, lv=lv) from . import train from amulog import log_db d = parse_condition(ns.conditions) ld = log_db.LogData(conf_load) iterobj = ld.iter_lines(**d) print(train.crf_trainfile(conf_dump, iterobj))
def show_graph_diff_search(ns): l_conffp = ns.confs assert len(l_conffp) == 2 openconf = lambda c: config.open_config( c, ex_defaults=[arguments.DEFAULT_CONFIG]) conf1, conf2 = [openconf(c) for c in l_conffp] lv = logging.DEBUG if ns.debug else logging.INFO am_logger = logging.getLogger("amulog") config.set_common_logging(conf1, logger=[_logger, am_logger], lv=lv) gid = ns.gid from . import comp_conf comp_conf.edge_diff_gid_search(conf1, conf2, gid)
def open_logdag_config(conf_path=None, debug=False): if conf_path is None: conf = config.open_config(DEFAULT_CONFIG, env="LOGDAG_CONFIG", base_default=False) else: conf = config.open_config(conf_path, env="LOGDAG_CONFIG", base_default=False, ex_defaults=[DEFAULT_CONFIG]) lv = logging.DEBUG if debug else logging.INFO am_logger = logging.getLogger("amulog") config.set_common_logging(conf, logger=[_logger, am_logger], lv=lv) return conf
def show_match_diff(ns): l_conffp = ns.confs assert len(l_conffp) == 2 openconf = lambda c: config.open_config( c, ex_defaults=[arguments.DEFAULT_CONFIG]) conf1, conf2 = [openconf(c) for c in l_conffp] lv = logging.DEBUG if ns.debug else logging.INFO am_logger = logging.getLogger("amulog") config.set_common_logging(conf1, logger=[_logger, am_logger], lv=lv) from . import trouble dirname = conf1.get("eval", "path") tm = trouble.TroubleManager(dirname) def _dag_from_name(conf, name): args = arguments.name2args(name, conf) r = showdag.LogDAG(args) r.load() return r from logdag import showdag from . import match_edge for tr in tm: d_args1 = match_edge.match_edges(conf1, tr, rule=ns.rule, cond=ns.cond) cnt1 = sum([len(l_edge) for l_edge in d_args1.values()]) d_args2 = match_edge.match_edges(conf2, tr, rule=ns.rule, cond=ns.cond) cnt2 = sum([len(l_edge) for l_edge in d_args2.values()]) if cnt1 == cnt2: pass else: print("Trouble {0} {1} ({2})".format(tr.tid, tr.data["date"], tr.data["group"])) print("{0}: {1}".format(config.getname(conf1), cnt1)) for key, l_edge in d_args1.items(): r1 = _dag_from_name(conf1, key) for edge in l_edge: edgestr = r1.edge_str(edge, graph=r1.graph.to_undirected()) print(key, edgestr) print("{0}: {1}".format(config.getname(conf2), cnt2)) for key, l_edge in d_args2.items(): r2 = _dag_from_name(conf2, key) for edge in l_edge: edgestr = r2.edge_str(edge, graph=r2.graph.to_undirected()) print(key, edgestr) print("")
def show_graph_diff_lts(ns): conf_fn1, conf_fn2 = ns.confs conf1 = arguments.open_logdag_config(conf_fn1) conf2 = arguments.open_logdag_config(conf_fn2) lv = logging.DEBUG if ns.debug else logging.INFO am_logger = logging.getLogger("amulog") config.set_common_logging(conf1, logger=[_logger, am_logger], lv=lv) from amulog import log_db ld = log_db.LogData(conf1) from . import comparison d_ltid = comparison.edge_diff_gid(conf1, conf2) for ltid, l_name in sorted(d_ltid.items(), key=lambda x: len(x[1]), reverse=True): print("{0}: {1} ({2})".format(len(l_name), ltid, ld.lt(ltid))) if len(l_name) < 100: print(l_name)
def draw_graph_diff(ns): conf_fn1, conf_fn2 = ns.confs conf1 = arguments.open_logdag_config(conf_fn1) conf2 = arguments.open_logdag_config(conf_fn2) lv = logging.DEBUG if ns.debug else logging.INFO am_logger = logging.getLogger("amulog") config.set_common_logging(conf1, logger=[_logger, am_logger], lv=lv) dts = dtutil.shortstr2dt(ns.timestr) dte = dts + config.getdur(conf1, "dag", "unit_term") output = ns.filename from . import comparison cevmap, cgraph = comparison.edge_set_diff(conf1, conf2, (dts, dte)) from . import draw rgraph = draw.relabel_nodes(cgraph, cevmap) draw.graph_nx(output, rgraph) print(output)
def show_graph_diff_lts(ns): l_conffp = ns.confs assert len(l_conffp) == 2 openconf = lambda c: config.open_config( c, ex_defaults=[arguments.DEFAULT_CONFIG]) conf1, conf2 = [openconf(c) for c in l_conffp] lv = logging.DEBUG if ns.debug else logging.INFO am_logger = logging.getLogger("amulog") config.set_common_logging(conf1, logger=[_logger, am_logger], lv=lv) from amulog import log_db ld = log_db.LogData(conf1) from . import comp_conf d_ltid = comp_conf.edge_diff_gid(conf1, conf2) for ltid, l_name in sorted(d_ltid.items(), key=lambda x: len(x[1]), reverse=True): print("{0}: {1} ({2})".format(len(l_name), ltid, ld.lt(ltid))) if len(l_name) < 100: print(l_name)
def draw_graph_diff(ns): l_conffp = ns.confs assert len(l_conffp) == 2 openconf = lambda c: config.open_config( c, ex_defaults=[arguments.DEFAULT_CONFIG]) conf1, conf2 = [openconf(c) for c in l_conffp] lv = logging.DEBUG if ns.debug else logging.INFO am_logger = logging.getLogger("amulog") config.set_common_logging(conf1, logger=[_logger, am_logger], lv=lv) dts = dtutil.shortstr2dt(ns.timestr) dte = dts + config.getdur(conf1, "dag", "unit_term") output = ns.filename from . import comp_conf cevmap, cgraph = comp_conf.edge_set_diff(conf1, conf2, (dts, dte)) from . import draw rgraph = draw.relabel_graph(conf1, cgraph, cevmap) draw.graph_nx(output, rgraph) print(output)
def show_graph_lor_edges(ns): conf_fn1, conf_fn2 = ns.confs conf1 = arguments.open_logdag_config(conf_fn1) conf2 = arguments.open_logdag_config(conf_fn2) lv = logging.DEBUG if ns.debug else logging.INFO am_logger = logging.getLogger("amulog") config.set_common_logging(conf1, logger=[_logger, am_logger], lv=lv) from . import comparison edge_sum = 0 d_edges = {} am = arguments.ArgumentManager(conf1) am.load() for dt_range in am.iter_dt_range(): cevmap, cgraph = comparison.edge_set_lor(conf1, conf2, dt_range) edge_sum += cgraph.number_of_edges() d_edges[dt_range[0]] = cgraph.edges() print("logical disjunction edge num: {0}".format(edge_sum)) for k, v in sorted(d_edges.items(), key=lambda x: x[0]): print("{0}: {1}".format(k, len(v)))
def show_graph_lor_edges(ns): l_conffp = ns.confs assert len(l_conffp) == 2 openconf = lambda c: config.open_config( c, ex_defaults=[arguments.DEFAULT_CONFIG]) conf1, conf2 = [openconf(c) for c in l_conffp] lv = logging.DEBUG if ns.debug else logging.INFO am_logger = logging.getLogger("amulog") config.set_common_logging(conf1, logger=[_logger, am_logger], lv=lv) from . import comp_conf edge_sum = 0 d_edges = {} am = arguments.ArgumentManager(conf1) am.load() for dt_range in am.iter_dt_range(): cevmap, cgraph = comp_conf.edge_set_lor(conf1, conf2, dt_range) edge_sum += cgraph.number_of_edges() d_edges[dt_range[0]] = cgraph.edges() print("logical disjunction edge num: {0}".format(edge_sum)) for k, v in sorted(d_edges.items(), key=lambda x: x[0]): print("{0}: {1}".format(k, len(v)))
def show_diff_info(ns): l_conffp = ns.confs assert len(l_conffp) == 2 openconf = lambda c: config.open_config( c, ex_defaults=[arguments.DEFAULT_CONFIG]) conf1, conf2 = [openconf(c) for c in l_conffp] lv = logging.DEBUG if ns.debug else logging.INFO am_logger = logging.getLogger("amulog") config.set_common_logging(conf1, logger=[_logger, am_logger], lv=lv) from . import comp_conf d = defaultdict(int) am = arguments.ArgumentManager(conf1) am.load() for dt_range in am.iter_dt_range(): cevmap_common, cgraph_common = comp_conf.edge_set_common( conf1, conf2, dt_range) d["common"] += cgraph_common.number_of_edges() cevmap_lor, cgraph_lor = comp_conf.edge_set_lor(conf1, conf2, dt_range) d["lor"] += cgraph_lor.number_of_edges() cevmap_diff1, cgraph_diff1 = comp_conf.edge_set_diff(conf1, conf2, dt_range, lor=(cevmap_lor, cgraph_lor)) d["diff1"] += cgraph_diff1.number_of_edges() cevmap_diff2, cgraph_diff2 = comp_conf.edge_set_diff(conf2, conf1, dt_range, lor=(cevmap_lor, cgraph_lor)) d["diff2"] += cgraph_diff2.number_of_edges() print("Logical OR edges: {0}".format(d["lor"])) print("Common edges: {0}".format(d["common"])) print("Edges only found in {0}: {1}".format(ns.confs[0], d["diff1"])) print("Edges only found in {0}: {1}".format(ns.confs[1], d["diff2"]))
# input # - ed_threshold # - d_ed1, d_ed2 import sys from amulog import config from amulog import lt_crf _logger = logging.getLogger("amulog") if len(sys.argv) < 4: sys.exit("usage: {0} CONFIG DISTFILE THRESHOLD".format(sys.argv[0])) conf = config.open_config(sys.argv[1]) config.set_common_logging(conf, logger=_logger, lv=logging.INFO) d_ed = {} with open(sys.argv[2], "r") as f: for line in f: temp = line.rstrip("\n").split() ltid = int(temp[0]) val = float(temp[1]) assert not ltid in d_ed d_ed[ltid] = val threshold = float(sys.argv[3]) s_ltid = set() for ltid, val in d_ed.items(): if val <= threshold: s_ltid.add(ltid)
for w in l_w: label = lw.label(w) if label == POS_UNKNOWN: rpl = w else: _logger.debug( "replace word <{0}>({1}) to **".format(w, label)) rpl = conf.get("log_template", "variable_symbol") tpl.append(rpl) l_tpl.add(tuple(tpl)) for tpl in l_tpl: print(" ".join(tpl)) if __name__ == "__main__": if len(sys.argv) < 4: sys.exit("usage: {0} mode conf input".format(sys.argv[0])) mode = sys.argv[1] conf = config.open_config(sys.argv[2]) lv = logging.DEBUG #lv = logging.INFO config.set_common_logging(conf, logger = _logger, lv = lv) lt_fn = sys.argv[3] if mode == "merge-re": merge_re(conf, lt_fn)
def open_logdag_config(conf_path, debug=False): conf = config.open_config(conf_path, ex_defaults=[DEFAULT_CONFIG]) lv = logging.DEBUG if debug else logging.INFO am_logger = logging.getLogger("amulog") config.set_common_logging(conf, logger=[_logger, am_logger], lv=lv) return conf