def show_diff_info(ns):
conf_fn1, conf_fn2 = ns.confs
conf1 = arguments.open_logdag_config(conf_fn1)
conf2 = arguments.open_logdag_config(conf_fn2)
lv = logging.DEBUG if ns.debug else logging.INFO
am_logger = logging.getLogger("amulog")
config.set_common_logging(conf1, logger=[_logger, am_logger], lv=lv)
from . import comparison
d = defaultdict(int)
am = arguments.ArgumentManager(conf1)
am.load()
for dt_range in am.iter_dt_range():
cevmap_common, cgraph_common = comparison.edge_set_common(
conf1, conf2, dt_range)
d["common"] += cgraph_common.number_of_edges()
cevmap_lor, cgraph_lor = comparison.edge_set_lor(
conf1, conf2, dt_range)
d["lor"] += cgraph_lor.number_of_edges()
cevmap_diff1, cgraph_diff1 = comparison.edge_set_diff(conf1,
conf2,
dt_range,
lor=(cevmap_lor,
cgraph_lor))
d["diff1"] += cgraph_diff1.number_of_edges()
cevmap_diff2, cgraph_diff2 = comparison.edge_set_diff(conf2,
conf1,
dt_range,
lor=(cevmap_lor,
cgraph_lor))
d["diff2"] += cgraph_diff2.number_of_edges()
print("Logical OR edges: {0}".format(d["lor"]))
print("Common edges: {0}".format(d["common"]))
print("Edges only found in {0}: {1}".format(ns.confs[0], d["diff1"]))
print("Edges only found in {0}: {1}".format(ns.confs[1], d["diff2"]))
def show_diff_direction(ns):
l_conffp = ns.confs
assert len(l_conffp) == 2
openconf = lambda c: config.open_config(
c, ex_defaults=[arguments.DEFAULT_CONFIG])
conf1, conf2 = [openconf(c) for c in l_conffp]
lv = logging.DEBUG if ns.debug else logging.INFO
am_logger = logging.getLogger("amulog")
config.set_common_logging(conf1, logger=[_logger, am_logger], lv=lv)
def _print_diff(ret):
for ev1, ev2, di1, di2 in ret:
print("{0} {1} | {2} {3}".format(ev1, di1, di2, ev2))
cnt = 0
from . import comp_conf
am = arguments.ArgumentManager(conf1)
am.load()
if ns.argname is None:
for dt_range in sorted(am.iter_dt_range()):
ret = comp_conf.edge_direction_diff(conf1, conf2, dt_range)
cnt += len(ret)
if len(ret) > 0:
print(dt_range)
_print_diff(ret)
print("")
else:
args = am.jobname2args(ns.argname, conf)
dt_range = args[2]
ret = comp_conf.edge_direction_diff(conf1, conf2, dt_range)
cnt += len(ret)
_print_diff(ret)
print(cnt)
def show_diff_edges(ns):
conf_fn1, conf_fn2 = ns.confs
conf1 = arguments.open_logdag_config(conf_fn1)
conf2 = arguments.open_logdag_config(conf_fn2)
lv = logging.DEBUG if ns.debug else logging.INFO
am_logger = logging.getLogger("amulog")
config.set_common_logging(conf1, logger=[_logger, am_logger], lv=lv)
from . import comparison
am = arguments.ArgumentManager(conf1)
am.load()
for dt_range in sorted(am.iter_dt_range()):
cevmap, cgraph = comparison.edge_set_diff(conf1, conf2, dt_range)
buf_edges = []
for edge in cgraph.edges():
buf = ""
src_info = cevmap.evdef(edge[0])
buf += "[gid={0[0]}, host = {0[1]}]".format(src_info)
if showdag.isdirected(edge, cgraph):
buf += " -> "
else:
buf += " <-> "
dst_info = cevmap.evdef(edge[1])
buf += "[gid={0[0]}, host = {0[1]}]".format(dst_info)
buf_edges.append(buf)
if len(buf_edges) > 0:
print("date: {0}".format(dt_range[0]))
print("\n".join(buf_edges))
def show_diff_edges(ns):
l_conffp = ns.confs
assert len(l_conffp) == 2
openconf = lambda c: config.open_config(
c, ex_defaults=[arguments.DEFAULT_CONFIG])
conf1, conf2 = [openconf(c) for c in l_conffp]
lv = logging.DEBUG if ns.debug else logging.INFO
am_logger = logging.getLogger("amulog")
config.set_common_logging(conf1, logger=[_logger, am_logger], lv=lv)
from . import comp_conf
am = arguments.ArgumentManager(conf1)
am.load()
for dt_range in sorted(am.iter_dt_range()):
cevmap, cgraph = comp_conf.edge_set_diff(conf1, conf2, dt_range)
buf_edges = []
for edge in cgraph.edges():
buf = ""
src_info = cevmap.evdef(edge[0])
buf += "[gid={0[0]}, host = {0[1]}]".format(src_info)
if showdag.isdirected(edge, cgraph):
buf += " -> "
else:
buf += " <-> "
dst_info = cevmap.evdef(edge[1])
buf += "[gid={0[0]}, host = {0[1]}]".format(dst_info)
buf_edges.append(buf)
if len(buf_edges) > 0:
print("date: {0}".format(dt_range[0]))
print("\n".join(buf_edges))
def make_crf_model(ns):
conf_load = config.open_config(ns.config_load)
conf_dump = config.open_config(ns.config_dump)
lv = logging.DEBUG if ns.debug else logging.INFO
config.set_common_logging(conf_dump, logger=_logger, lv=lv)
from . import train
output = ns.output
if ns.train_file is None:
from amulog import log_db
d = parse_condition(ns.conditions)
output_sampled = ns.output_sampled
get_output_sampled = output_sampled is not None
ld = log_db.LogData(conf_load)
iterobj = [lm for lm in ld.iter_lines(**d)]
if get_output_sampled:
fn, l_train = train.make_crf_model(conf_dump, iterobj, output,
return_sampled_messages=True)
import pickle
with open(output_sampled, 'wb') as f:
pickle.dump(l_train, f)
print("> {0}".format(output_sampled))
else:
fn = train.make_crf_model(conf_dump, iterobj, output)
else:
fn = train.make_crf_model_from_trainfile(conf_dump, ns.train_file, output)
print("> {0}".format(fn))
def show_diff_direction(ns):
conf_fn1, conf_fn2 = ns.confs
conf1 = arguments.open_logdag_config(conf_fn1)
conf2 = arguments.open_logdag_config(conf_fn2)
lv = logging.DEBUG if ns.debug else logging.INFO
am_logger = logging.getLogger("amulog")
config.set_common_logging(conf1, logger=[_logger, am_logger], lv=lv)
def _print_diff(_ret):
for ev1, ev2, di1, di2 in _ret:
print("{0} {1} | {2} {3}".format(ev1, di1, di2, ev2))
cnt = 0
from . import comparison
am = arguments.ArgumentManager(conf1)
am.load()
if ns.argname is None:
for dt_range in sorted(am.iter_dt_range()):
ret = comparison.edge_direction_diff(conf1, conf2, dt_range)
cnt += len(ret)
if len(ret) > 0:
print(dt_range)
_print_diff(ret)
print("")
else:
args = am.jobname2args(ns.argname, conf1)
dt_range = args[2]
ret = comparison.edge_direction_diff(conf1, conf2, dt_range)
cnt += len(ret)
_print_diff(ret)
print(cnt)
def show_graph_diff_search(ns):
conf_fn1, conf_fn2 = ns.confs
conf1 = arguments.open_logdag_config(conf_fn1)
conf2 = arguments.open_logdag_config(conf_fn2)
lv = logging.DEBUG if ns.debug else logging.INFO
am_logger = logging.getLogger("amulog")
config.set_common_logging(conf1, logger=[_logger, am_logger], lv=lv)
gid = ns.gid
from . import comparison
comparison.edge_diff_gid_search(conf1, conf2, gid)
def make_crf_train(ns):
conf_load = config.open_config(ns.config_load)
conf_dump = config.open_config(ns.config_dump)
lv = logging.DEBUG if ns.debug else logging.INFO
config.set_common_logging(conf_dump, logger=_logger, lv=lv)
from . import train
from amulog import log_db
d = parse_condition(ns.conditions)
ld = log_db.LogData(conf_load)
iterobj = ld.iter_lines(**d)
print(train.crf_trainfile(conf_dump, iterobj))
def show_graph_diff_search(ns):
l_conffp = ns.confs
assert len(l_conffp) == 2
openconf = lambda c: config.open_config(
c, ex_defaults=[arguments.DEFAULT_CONFIG])
conf1, conf2 = [openconf(c) for c in l_conffp]
lv = logging.DEBUG if ns.debug else logging.INFO
am_logger = logging.getLogger("amulog")
config.set_common_logging(conf1, logger=[_logger, am_logger], lv=lv)
gid = ns.gid
from . import comp_conf
comp_conf.edge_diff_gid_search(conf1, conf2, gid)
def open_logdag_config(conf_path=None, debug=False):
if conf_path is None:
conf = config.open_config(DEFAULT_CONFIG,
env="LOGDAG_CONFIG",
base_default=False)
else:
conf = config.open_config(conf_path,
env="LOGDAG_CONFIG",
base_default=False,
ex_defaults=[DEFAULT_CONFIG])
lv = logging.DEBUG if debug else logging.INFO
am_logger = logging.getLogger("amulog")
config.set_common_logging(conf, logger=[_logger, am_logger], lv=lv)
return conf
def show_match_diff(ns):
l_conffp = ns.confs
assert len(l_conffp) == 2
openconf = lambda c: config.open_config(
c, ex_defaults=[arguments.DEFAULT_CONFIG])
conf1, conf2 = [openconf(c) for c in l_conffp]
lv = logging.DEBUG if ns.debug else logging.INFO
am_logger = logging.getLogger("amulog")
config.set_common_logging(conf1, logger=[_logger, am_logger], lv=lv)
from . import trouble
dirname = conf1.get("eval", "path")
tm = trouble.TroubleManager(dirname)
def _dag_from_name(conf, name):
args = arguments.name2args(name, conf)
r = showdag.LogDAG(args)
r.load()
return r
from logdag import showdag
from . import match_edge
for tr in tm:
d_args1 = match_edge.match_edges(conf1, tr, rule=ns.rule, cond=ns.cond)
cnt1 = sum([len(l_edge) for l_edge in d_args1.values()])
d_args2 = match_edge.match_edges(conf2, tr, rule=ns.rule, cond=ns.cond)
cnt2 = sum([len(l_edge) for l_edge in d_args2.values()])
if cnt1 == cnt2:
pass
else:
print("Trouble {0} {1} ({2})".format(tr.tid, tr.data["date"],
tr.data["group"]))
print("{0}: {1}".format(config.getname(conf1), cnt1))
for key, l_edge in d_args1.items():
r1 = _dag_from_name(conf1, key)
for edge in l_edge:
edgestr = r1.edge_str(edge, graph=r1.graph.to_undirected())
print(key, edgestr)
print("{0}: {1}".format(config.getname(conf2), cnt2))
for key, l_edge in d_args2.items():
r2 = _dag_from_name(conf2, key)
for edge in l_edge:
edgestr = r2.edge_str(edge, graph=r2.graph.to_undirected())
print(key, edgestr)
print("")
def show_graph_diff_lts(ns):
conf_fn1, conf_fn2 = ns.confs
conf1 = arguments.open_logdag_config(conf_fn1)
conf2 = arguments.open_logdag_config(conf_fn2)
lv = logging.DEBUG if ns.debug else logging.INFO
am_logger = logging.getLogger("amulog")
config.set_common_logging(conf1, logger=[_logger, am_logger], lv=lv)
from amulog import log_db
ld = log_db.LogData(conf1)
from . import comparison
d_ltid = comparison.edge_diff_gid(conf1, conf2)
for ltid, l_name in sorted(d_ltid.items(),
key=lambda x: len(x[1]),
reverse=True):
print("{0}: {1} ({2})".format(len(l_name), ltid, ld.lt(ltid)))
if len(l_name) < 100:
print(l_name)
def draw_graph_diff(ns):
conf_fn1, conf_fn2 = ns.confs
conf1 = arguments.open_logdag_config(conf_fn1)
conf2 = arguments.open_logdag_config(conf_fn2)
lv = logging.DEBUG if ns.debug else logging.INFO
am_logger = logging.getLogger("amulog")
config.set_common_logging(conf1, logger=[_logger, am_logger], lv=lv)
dts = dtutil.shortstr2dt(ns.timestr)
dte = dts + config.getdur(conf1, "dag", "unit_term")
output = ns.filename
from . import comparison
cevmap, cgraph = comparison.edge_set_diff(conf1, conf2, (dts, dte))
from . import draw
rgraph = draw.relabel_nodes(cgraph, cevmap)
draw.graph_nx(output, rgraph)
print(output)
def show_graph_diff_lts(ns):
l_conffp = ns.confs
assert len(l_conffp) == 2
openconf = lambda c: config.open_config(
c, ex_defaults=[arguments.DEFAULT_CONFIG])
conf1, conf2 = [openconf(c) for c in l_conffp]
lv = logging.DEBUG if ns.debug else logging.INFO
am_logger = logging.getLogger("amulog")
config.set_common_logging(conf1, logger=[_logger, am_logger], lv=lv)
from amulog import log_db
ld = log_db.LogData(conf1)
from . import comp_conf
d_ltid = comp_conf.edge_diff_gid(conf1, conf2)
for ltid, l_name in sorted(d_ltid.items(),
key=lambda x: len(x[1]),
reverse=True):
print("{0}: {1} ({2})".format(len(l_name), ltid, ld.lt(ltid)))
if len(l_name) < 100:
print(l_name)
def draw_graph_diff(ns):
l_conffp = ns.confs
assert len(l_conffp) == 2
openconf = lambda c: config.open_config(
c, ex_defaults=[arguments.DEFAULT_CONFIG])
conf1, conf2 = [openconf(c) for c in l_conffp]
lv = logging.DEBUG if ns.debug else logging.INFO
am_logger = logging.getLogger("amulog")
config.set_common_logging(conf1, logger=[_logger, am_logger], lv=lv)
dts = dtutil.shortstr2dt(ns.timestr)
dte = dts + config.getdur(conf1, "dag", "unit_term")
output = ns.filename
from . import comp_conf
cevmap, cgraph = comp_conf.edge_set_diff(conf1, conf2, (dts, dte))
from . import draw
rgraph = draw.relabel_graph(conf1, cgraph, cevmap)
draw.graph_nx(output, rgraph)
print(output)
def show_graph_lor_edges(ns):
conf_fn1, conf_fn2 = ns.confs
conf1 = arguments.open_logdag_config(conf_fn1)
conf2 = arguments.open_logdag_config(conf_fn2)
lv = logging.DEBUG if ns.debug else logging.INFO
am_logger = logging.getLogger("amulog")
config.set_common_logging(conf1, logger=[_logger, am_logger], lv=lv)
from . import comparison
edge_sum = 0
d_edges = {}
am = arguments.ArgumentManager(conf1)
am.load()
for dt_range in am.iter_dt_range():
cevmap, cgraph = comparison.edge_set_lor(conf1, conf2, dt_range)
edge_sum += cgraph.number_of_edges()
d_edges[dt_range[0]] = cgraph.edges()
print("logical disjunction edge num: {0}".format(edge_sum))
for k, v in sorted(d_edges.items(), key=lambda x: x[0]):
print("{0}: {1}".format(k, len(v)))
def show_graph_lor_edges(ns):
l_conffp = ns.confs
assert len(l_conffp) == 2
openconf = lambda c: config.open_config(
c, ex_defaults=[arguments.DEFAULT_CONFIG])
conf1, conf2 = [openconf(c) for c in l_conffp]
lv = logging.DEBUG if ns.debug else logging.INFO
am_logger = logging.getLogger("amulog")
config.set_common_logging(conf1, logger=[_logger, am_logger], lv=lv)
from . import comp_conf
edge_sum = 0
d_edges = {}
am = arguments.ArgumentManager(conf1)
am.load()
for dt_range in am.iter_dt_range():
cevmap, cgraph = comp_conf.edge_set_lor(conf1, conf2, dt_range)
edge_sum += cgraph.number_of_edges()
d_edges[dt_range[0]] = cgraph.edges()
print("logical disjunction edge num: {0}".format(edge_sum))
for k, v in sorted(d_edges.items(), key=lambda x: x[0]):
print("{0}: {1}".format(k, len(v)))
def show_diff_info(ns):
l_conffp = ns.confs
assert len(l_conffp) == 2
openconf = lambda c: config.open_config(
c, ex_defaults=[arguments.DEFAULT_CONFIG])
conf1, conf2 = [openconf(c) for c in l_conffp]
lv = logging.DEBUG if ns.debug else logging.INFO
am_logger = logging.getLogger("amulog")
config.set_common_logging(conf1, logger=[_logger, am_logger], lv=lv)
from . import comp_conf
d = defaultdict(int)
am = arguments.ArgumentManager(conf1)
am.load()
for dt_range in am.iter_dt_range():
cevmap_common, cgraph_common = comp_conf.edge_set_common(
conf1, conf2, dt_range)
d["common"] += cgraph_common.number_of_edges()
cevmap_lor, cgraph_lor = comp_conf.edge_set_lor(conf1, conf2, dt_range)
d["lor"] += cgraph_lor.number_of_edges()
cevmap_diff1, cgraph_diff1 = comp_conf.edge_set_diff(conf1,
conf2,
dt_range,
lor=(cevmap_lor,
cgraph_lor))
d["diff1"] += cgraph_diff1.number_of_edges()
cevmap_diff2, cgraph_diff2 = comp_conf.edge_set_diff(conf2,
conf1,
dt_range,
lor=(cevmap_lor,
cgraph_lor))
d["diff2"] += cgraph_diff2.number_of_edges()
print("Logical OR edges: {0}".format(d["lor"]))
print("Common edges: {0}".format(d["common"]))
print("Edges only found in {0}: {1}".format(ns.confs[0], d["diff1"]))
print("Edges only found in {0}: {1}".format(ns.confs[1], d["diff2"]))
# input
# - ed_threshold
# - d_ed1, d_ed2
import sys
from amulog import config
from amulog import lt_crf
_logger = logging.getLogger("amulog")
if len(sys.argv) < 4:
sys.exit("usage: {0} CONFIG DISTFILE THRESHOLD".format(sys.argv[0]))
conf = config.open_config(sys.argv[1])
config.set_common_logging(conf, logger=_logger, lv=logging.INFO)
d_ed = {}
with open(sys.argv[2], "r") as f:
for line in f:
temp = line.rstrip("\n").split()
ltid = int(temp[0])
val = float(temp[1])
assert not ltid in d_ed
d_ed[ltid] = val
threshold = float(sys.argv[3])
s_ltid = set()
for ltid, val in d_ed.items():
if val <= threshold:
s_ltid.add(ltid)
for w in l_w:
label = lw.label(w)
if label == POS_UNKNOWN:
rpl = w
else:
_logger.debug(
"replace word <{0}>({1}) to **".format(w, label))
rpl = conf.get("log_template", "variable_symbol")
tpl.append(rpl)
l_tpl.add(tuple(tpl))
for tpl in l_tpl:
print(" ".join(tpl))
if __name__ == "__main__":
if len(sys.argv) < 4:
sys.exit("usage: {0} mode conf input".format(sys.argv[0]))
mode = sys.argv[1]
conf = config.open_config(sys.argv[2])
lv = logging.DEBUG
#lv = logging.INFO
config.set_common_logging(conf, logger = _logger, lv = lv)
lt_fn = sys.argv[3]
if mode == "merge-re":
merge_re(conf, lt_fn)
def open_logdag_config(conf_path, debug=False):
conf = config.open_config(conf_path, ex_defaults=[DEFAULT_CONFIG])
lv = logging.DEBUG if debug else logging.INFO
am_logger = logging.getLogger("amulog")
config.set_common_logging(conf, logger=[_logger, am_logger], lv=lv)
return conf