def get_enmin_csv_infos(csvfile):
tempinfo = fh.csv_remove_repeat(csvfile, ['sysEname', "triggername"])
nowinfo = tempinfo.iloc[:, 1:4].values.tolist()
minresult = []
for i, one in enumerate(nowinfo):
minresult.append(one + [i + 1])
return minresult
def get_csv_nodesys(csvfile):
infos=fh.csv_remove_repeat(csvfile,['sysEname',"triggername"])
warninglist=infos["triggername"].tolist()
nodelist=[one.split(" ",1)[0][2:] for one in warninglist]
syslist=infos["sysEname"].tolist()
result={nodelist[i]:syslist[i] for i in range(len(nodelist))}
return result
def get_csv_baseinfo(csvfilelist, returnlabels=False):
resultcsvinfos = []
labels = []
allnodelist = nh.get_every_point(pointfile)
for i in csvfilelist:
tempinfo = ac.pdframe_readcsv(i, 0)
# pd.Series(tempinfo["triggername"].value_counts(), index=[''])
counters = tempinfo["triggername"].value_counts()
#去重
tempinfo = fh.csv_remove_repeat(i, ['sysEname', "triggername"])
tempinfo.sort_index()
trechargerinfo = tempinfo.iloc[:, 1:4].values.T
csvweb = nh.create_digraph_bydir(pointfile)
# print(i,list(set(nh.get_every_point(pointfile))-set(trechargerinfo[0])-set([one.split(" ",1)[0][2:] for one in trechargerinfo[2]])))
csvweb.remove_nodes_from(
list(
set(allnodelist) - set(trechargerinfo[0]) -
set([one.split(" ", 1)[0][2:] for one in trechargerinfo[2]])))
if (returnlabels):
labels += tempinfo.loc[:, "is_root"].values.tolist()
nowinfo = tempinfo.iloc[:, 1:4].values.tolist()
for i, one in enumerate(nowinfo):
splittempinfo = one[2].split(" ", 1)
#one[0][4:] 系统
#splittempinfo[0][7:] 主机号
#splittempinfo[1] 告警信息
node = get_warninginfo_node(splittempinfo[1])
pnode = -1
minlen = 0
if (node != -1):
pnode = int(node[5:])
minlen = nx.dijkstra_path_length(csvweb,
source=splittempinfo[0][2:],
target=node)
# print(splittempinfo[1]+":"+str(node))#测试
resultcsvinfos.append([
one[0][4:], #one[0][4:]
splittempinfo[0][7:],
tg.change_time_format(one[1], "%Y-%m-%d %H:%M:%S", "%H"),
splittempinfo[1],
pnode,
minlen,
csvweb.out_degree(splittempinfo[0][2:]),
csvweb.in_degree(splittempinfo[0][2:]),
tg.get_timestr_to_week(one[1], "%Y-%m-%d %H:%M:%S"),
tg.change_time_format(one[1], "%Y-%m-%d %H:%M:%S", "%d"),
pd.Series(counters, index=[one[2]]),
one[1],
i,
])
resultcsvinfos = np.asarray(resultcsvinfos)
if (returnlabels):
return resultcsvinfos, labels
else:
return resultcsvinfos
def csv_waringinfos_fcfs(csvfile):
tempinfo = fh.csv_remove_repeat(csvfile, ['sysEname', "triggername"])
tempinfo.sort_index()
nowinfo = tempinfo.iloc[:, 1:4].values.tolist()
minresult = []
for i, one in enumerate(nowinfo):
minresult.append([one[2], i + 1])
return minresult
def get_basedata(basefiledir):
allinfo=[]
csvfilelist=fh.get_files_by_types(basefiledir,".csv")
for i in csvfilelist:
#去重
tempinfo = fh.csv_remove_repeat(i,['sysEname',"triggername"])
allinfo+=np.array(tempinfo).tolist()
return allinfo
def get_csv_nodedata(csvfile, pointnode, pdir=pointfile):
allnodelist = nh.get_every_point(pdir)
csvweb = nh.create_digraph_bydir(pdir)
tempinfo = fh.csv_remove_repeat(csvfile, ['sysEname', "triggername"])
tempinfo.sort_index()
trechargerinfo = tempinfo.iloc[:, 1:4].values.T
csvweb.remove_nodes_from(
list(
set(allnodelist) - set(trechargerinfo[0]) -
set([one.split(" ", 1)[0][2:] for one in trechargerinfo[2]])))
warningdict = dc.get_csv_warninginfo(csvfile)
predecessors = list(csvweb.predecessors(pointnode))
successors = list(csvweb.successors(pointnode))
strc = ""
strc += "{\"data\":[\n"
for i in allnodelist:
type = -1
if i not in csvweb.nodes():
type = 3
strc += "{\"name\":\"" + i + "\","
typelist = ['系统', '告警涉及节点', '根因节点', '当前无关节点', '根因相连前驱节点', '根因相连后继节点']
if (type != 3):
if (i[:3] == "SYS"):
type = 0
elif i in predecessors:
type = 4
elif i in successors:
type = 5
elif i != pointnode:
type = 1
else:
type = 2
strc += "\"category\":" + str(type) + ","
if i in csvweb.nodes() and i[:3] != "SYS":
strc += "\"value\":[\"" + typelist[type] + "," + warningdict[
i] + "\"],"
else:
strc += "\"value\":[\"" + typelist[type] + "\"],"
strc += "\"target\":["
if (type != 3):
subnodelist = list(csvweb.successors(i))
else:
subnodelist = []
for j in subnodelist:
strc += "{\"name\":\"" + j + "\","
strc += "\"value\":\"" + typelist[type] + "\""
strc += "}"
if (j != subnodelist[-1]):
strc += ","
strc += "]}"
if (i != allnodelist[-1]):
strc += ","
strc += "]}\n"
return strc
def get_csv_nodeinfo_general(csvfile, pointnode, pdir=pointfile):
allnodelist = nh.get_every_point(pdir)
csvweb = nh.create_digraph_bydir(pdir)
tempinfo = fh.csv_remove_repeat(csvfile, ['sysEname', "triggername"])
tempinfo.sort_index()
trechargerinfo = tempinfo.iloc[:, 1:4].values.T
csvweb.remove_nodes_from(
list(
set(allnodelist) - set(trechargerinfo[0]) -
set([one.split(" ", 1)[0][2:] for one in trechargerinfo[2]])))
nodesys = dc.get_csv_nodesys(csvfile)
predecessors = [
one for one in list(csvweb.predecessors(pointnode)) if one[:3] != "SYS"
]
successors = [
one for one in list(csvweb.successors(pointnode)) if one[:3] != "SYS"
]
presys = [nodesys[one] for one in predecessors]
sucsys = [nodesys[one] for one in successors]
return predecessors, presys, successors, sucsys
def create_keypointcsv_onlywarning(dir):
tempsplit = []
labels = []
for i in fh.get_files_by_types(dir, ".csv"):
pdframe = fh.csv_remove_repeat(i,
['sysEname', "triggername", "is_root"])
tempsplit += [
one.split(" ", 1)[1]
for one in pdframe.loc[:, "triggername"].values
]
labels += pdframe.loc[:, "is_root"].values.tolist()
# 警告信息根因表
alldict = {one: 0 for one in set(tempsplit)}
unrootcounts = {one: 0 for one in set(tempsplit)}
for i in range(len(tempsplit)):
init = tempsplit[i]
root = labels[i]
alldict[init] = alldict[init] + 1
if root == 0:
unrootcounts[init] = unrootcounts[init] + 1
unrootlist = []
rootlist = []
unrelation = []
for init in alldict:
if alldict[init] == unrootcounts[init]:
unrootlist.append(init + "|0|" + str(unrootcounts[init]))
elif unrootcounts[init] == 0:
rootlist.append(init + "|" + str(alldict[init]) + "|0")
else:
unrelation.append(init + "|" +
str(alldict[init] - unrootcounts[init]) + "|" +
str(unrootcounts[init]))
return rootlist, unrootlist, unrelation
def create_keypointcsv(dir):
nodetempsilit = []
systempsplit = []
tempsplit = []
labels = []
for i in fh.get_files_by_types(dir, ".csv"):
pdframe = fh.csv_remove_repeat(i,
['sysEname', "triggername", "is_root"])
tempsplit += [
one.split(" ", 1)[1]
for one in pdframe.loc[:, "triggername"].values
]
systempsplit += [one[4:] for one in pdframe.loc[:, "sysEname"].values]
nodetempsilit += [
one.split(" ", 1)[0][7:]
for one in pdframe.loc[:, "triggername"].values
]
labels += pdframe.loc[:, "is_root"].values.tolist()
#系统根因表
sysalldict = {one: 0 for one in set(systempsplit)}
sysunrootcounts = {one: 0 for one in set(systempsplit)}
for i in range(len(systempsplit)):
init = systempsplit[i]
root = labels[i]
sysalldict[init] = sysalldict[init] + 1
if root == 0:
sysunrootcounts[init] = sysunrootcounts[init] + 1
sysunrootlist = []
sysrootlist = []
sysunrelation = []
for init in sysalldict:
if sysalldict[init] == sysunrootcounts[init]:
sysunrootlist.append(init + "|0|" + str(sysunrootcounts[init]))
elif sysunrootcounts[init] == 0:
sysrootlist.append(init + "|" + str(sysalldict[init]) + "|0")
else:
sysunrelation.append(init + "|" + str(sysalldict[init] -
sysunrootcounts[init]) +
"|" + str(sysunrootcounts[init]))
#主机根因表
nodealldict = {one: 0 for one in set(nodetempsilit)}
nodeunrootcounts = {one: 0 for one in set(nodetempsilit)}
for i in range(len(nodetempsilit)):
init = nodetempsilit[i]
root = labels[i]
nodealldict[init] = nodealldict[init] + 1
if root == 0:
nodeunrootcounts[init] = nodeunrootcounts[init] + 1
nodeunrootlist = []
noderootlist = []
nodeunrelation = []
for init in nodealldict:
if nodealldict[init] == nodeunrootcounts[init]:
nodeunrootlist.append(init + "|0|" + str(nodeunrootcounts[init]))
elif nodeunrootcounts[init] == 0:
noderootlist.append(init + "|" + str(nodealldict[init]) + "|0")
else:
nodeunrelation.append(init + "|" + str(nodealldict[init] -
nodeunrootcounts[init]) +
"|" + str(nodeunrootcounts[init]))
#警告信息根因表
alldict = {one: 0 for one in set(tempsplit)}
unrootcounts = {one: 0 for one in set(tempsplit)}
for i in range(len(tempsplit)):
init = tempsplit[i]
root = labels[i]
alldict[init] = alldict[init] + 1
if root == 0:
unrootcounts[init] = unrootcounts[init] + 1
unrootlist = []
rootlist = []
unrelation = []
for init in alldict:
if alldict[init] == unrootcounts[init]:
unrootlist.append(init + "|0|" + str(unrootcounts[init]))
elif unrootcounts[init] == 0:
rootlist.append(init + "|" + str(alldict[init]) + "|0")
else:
unrelation.append(init + "|" +
str(alldict[init] - unrootcounts[init]) + "|" +
str(unrootcounts[init]))
return rootlist, unrootlist, unrelation, noderootlist, nodeunrootlist, nodeunrelation, sysrootlist, sysunrootlist, sysunrelation
def get_csv_warninginfo(csvfile):
infos=fh.csv_remove_repeat(csvfile,['sysEname',"triggername"])
warninglist=infos["triggername"].tolist()
splitlist=[one.split(" ",1) for one in warninglist]
result={one[0][2:]:one[1] for one in splitlist}
return result