def pc_log(conf, top_dt, end_dt, dur, area):
_logger.info("job start ({0} - {1} in {2})".format(top_dt, end_dt, area))
edict, evmap = log2event.get_edict(conf, top_dt, end_dt, dur, area)
_logger.info("{0} events found in given term of log data".format(
len(edict)))
if len(edict) > 2:
threshold = conf.getfloat("dag", "threshold")
ci_func = conf.get("dag", "ci_func")
bin_overlap = conf.getdur("dag", "stat_bin_overlap")
if bin_overlap is None or bin_overlap == "":
bin_overlap = datetime.timedelta(seconds=0)
binarize = pc_input.input_binarize(ci_func)
data = log2event.event2stat(edict, top_dt, end_dt, dur, binarize,
bin_overlap)
skel_method = conf.get("dag", "skeleton_method")
skel_verbose = conf.getboolean("dag", "skeleton_verbose")
pc_depth = conf.getint("dag", "skeleton_depth")
graph = pc_input.pc(data, threshold, ci_func, skel_method, pc_depth,
skel_verbose)
else:
_logger.info("insufficient events({0}), return empty dag".format(\
len(edict)))
graph = pcresult.empty_dag()
output = pcresult.PCOutput(conf)
output.make(graph, evmap, top_dt, end_dt, dur, area)
output.dump()
_logger.info("job done, output {0}".format(output.filename))
return output
def count_edge_label_extype(conf):
ll = init_ltlabel(conf)
ld = log_db.LogData(conf)
import pcresult
s_keys = set()
d_extype = defaultdict(int)
src_dir = conf.get("dag", "output_dir")
for fp in common.rep_dir(src_dir):
r = pcresult.PCOutput(conf).load(fp)
dedges, udedges = r._separate_edges()
for edge in dedges + udedges:
l_group = [
r._label_group_ltg(r.evmap.info(eid).gid) for eid in edge
]
for group in l_group:
s_keys.add(group)
if l_group[0] == l_group[1]:
d_extype[tuple(l_group)] += 1
else:
d_extype[(l_group[0], l_group[1])] += 1
d_extype[(l_group[1], l_group[0])] += 1
table = []
table.append(["group"] + list(s_keys))
for key1 in s_keys:
buf = [key1]
for key2 in s_keys:
cnt = d_extype[(key1, key2)]
buf.append(cnt)
table.append(buf)
print(common.cli_table(table))
def similar_block_dag(conf, top_dt, end_dt, area, method, ignore_same=True):
src_dir = conf.get("dag", "output_dir")
ig_direction = conf.getboolean("search", "dag_ig_direction")
wflag = conf.getboolean("search", "dag_weight")
dur = conf.getdur("dag", "stat_bin")
name = pc_log.thread_name(conf, top_dt, end_dt, dur, area)
if name in common.rep_dir(src_dir):
r_temp = pcresult.PCOutput(conf).load(name)
else:
r_temp = pc_log.pc_log(conf, top_dt, end_dt, dur, area, dump=False)
src_dir = conf.get("dag", "output_dir")
l_r = pcresult.results_in_area(conf, src_dir, area)
weight = None
if wflag:
weight = pcresult.EdgeTFIDF(l_r)
result = []
for r in l_r:
if ignore_same and (r.end_dt > top_dt and r.top_dt < end_dt):
# ignore if common term included
pass
else:
if method == "dag_ed":
dist = pcresult.graph_edit_distance(r_temp, r, ig_direction,
weight)
elif method == "dag_mcs":
dist = pcresult.mcs_size_ratio(r_temp, r, ig_direction, weight)
else:
raise NotImplementedError
result.append((r, dist))
return result
def search_edge_label_extype(conf, label1, label2):
ll = init_ltlabel(conf)
ld = log_db.LogData(conf)
import pcresult
src_dir = conf.get("dag", "output_dir")
for fp in common.rep_dir(src_dir):
r = pcresult.PCOutput(conf).load(fp)
rflag = False
dedges, udedges = r._separate_edges()
for edge in dedges + udedges:
l_group = [r._label_group_ltg(r.evmap.info(eid).gid)
for eid in edge]
if (l_group[0] == label1 and l_group[1] == label2) or \
(l_group[1] == label1 and l_group[0] == label2):
if not rflag:
print("# {0}".format(r.filename))
rflag = True
r._print_edge(edge, False)
r._print_edge_lt(edge)
def pc_log(conf, top_dt, end_dt, dur, area):
_logger.info("job start ({0} - {1} in {2})".format(top_dt, end_dt, area))
edict, evmap = log2event.log2event(conf, top_dt, end_dt, dur, area)
_logger.info("{0} events found in given term of log data".format(
len(edict)))
if len(edict) > 2:
threshold = conf.getfloat("dag", "threshold")
graph = pc_input.pc(edict, threshold)
else:
_logger.info("insufficient events({0}), return empty dag".format(\
len(edict)))
graph = pc_input.empty_dag()
output = pcresult.PCOutput(conf)
output.make(graph, evmap, top_dt, end_dt, dur, area)
output.dump()
_logger.info("job done, output {0}".format(output.filename))
return output
def count_edge_label(conf):
ll = init_ltlabel(conf)
import pcresult
d_cnt_label = defaultdict(int)
d_cnt_group = defaultdict(int)
src_dir = conf.get("dag", "output_dir")
for fp in common.rep_dir(src_dir):
_logger.info("count_edge_label processing {0}".format(fp))
r = pcresult.PCOutput(conf).load(fp)
for edge in r.graph.edges():
for eid in edge:
gid = r.evmap.info(eid).gid
label = r._label_ltg(gid)
d_cnt_label[label] += 1
group = r._label_group_ltg(gid)
d_cnt_group[group] += 1
for group, l_label in ll.d_group.items():
cnt_group = d_cnt_group[group]
print("group {0}: {1} nodes".format(group, cnt_group))
for label in l_label:
cnt_label = d_cnt_label[label]
print(" label {0}: {1} nodes".format(label, cnt_label))
print
def count_edge_label_detail(conf):
ll = init_ltlabel(conf)
ld = log_db.LogData(conf)
import pcresult
d_group = defaultdict(int)
d_group_directed = defaultdict(int)
d_group_intype = defaultdict(int)
d_group_intype_directed = defaultdict(int)
d_group_mean = defaultdict(int)
d_group_mean_directed = defaultdict(int)
import edge_filter
ef = edge_filter.EdgeFilter(conf)
src_dir = conf.get("dag", "output_dir")
for fp in common.rep_dir(src_dir):
_logger.info("count_edge_label_detail processing {0}".format(fp))
r = pcresult.PCOutput(conf).load(fp)
dedges, udedges = r._separate_edges()
for edge in dedges:
cedge = [r.evmap.info(eid) for eid in edge]
fflag = ef.isfiltered(cedge)
l_group = [
r._label_group_ltg(r.evmap.info(eid).gid) for eid in edge
]
iflag = (l_group[0] == l_group[1])
for group in l_group:
d_group[group] += 1
d_group_directed[group] += 1
if iflag:
d_group_intype[group] += 1
d_group_intype_directed[group] += 1
if not fflag:
d_group_mean[group] += 1
d_group_mean_directed[group] += 1
for edge in udedges:
cedge = [r.evmap.info(eid) for eid in edge]
fflag = ef.isfiltered(cedge)
l_group = [
r._label_group_ltg(r.evmap.info(eid).gid) for eid in edge
]
iflag = (l_group[0] == l_group[1])
for group in l_group:
d_group[group] += 1
if iflag:
d_group_intype[group] += 1
if not fflag:
d_group_mean[group] += 1
table = [[
"key", "all", "directed", "intype", "intype_directed", "important",
"important_directed"
]]
for key in d_group.keys():
temp = [key]
temp.append(d_group[key])
temp.append(d_group_directed[key])
temp.append(d_group_intype[key])
temp.append(d_group_intype_directed[key])
temp.append(d_group_mean[key])
temp.append(d_group_mean_directed[key])
table.append(temp)
table.append([
"total",
sum(d_group.values()),
sum(d_group_directed.values()),
sum(d_group_intype.values()),
sum(d_group_intype_directed.values()),
sum(d_group_mean.values()),
sum(d_group_mean_directed.values())
])
print(common.cli_table(table))