def open_network(network_path):
"""Function that imports a network from an html file.
Parameters:
- network_path: path of the network to be imported
Returns:
- G: network object
"""
G = Network()
html = open(network_path, "r")
lines_html = html.readlines()
nodes = ""
edges = ""
for line in lines_html:
if "nodes = new" in line:
nodes = json.loads(line.split('(')[1].split(')')[0])
if "edges = new" in line:
edges = json.loads(line.split('(')[1].split(')')[0])
dictionary_to_nodes(nodes, G)
dictionary_to_edges(edges, G)
G.save_graph("./NetworkGraphs/Temp_Network/temp_network.html")
html_fix(os.path.abspath("./NetworkGraphs/Temp_Network/temp_network.html"))
return G
def on_save(self):
"""Method called when the user presses the save button in the editor configuration. It applies the changes in the temporary network and updates the network graph of the canvas.
Parameters:
- self: current instance of the class;
"""
G = Network()
print(self.routers)
print(self.switches)
print(self.hosts)
print(self.hosts)
network_core.dictionary_to_nodes(self.routers, G)
network_core.dictionary_to_nodes(self.switches, G)
network_core.dictionary_to_nodes(self.hosts, G)
network_core.dictionary_to_nodes(self.others, G)
network_core.dictionary_to_edges(self.temporary_network.edges, G)
self.main_window_object.current_network = G
G.save_graph("./NetworkGraphs/Temp_Network/temp_network.html")
network_core.html_fix(os.path.abspath("./NetworkGraphs/Temp_Network/temp_network.html"))
self.main_window_object.update_canvas_html(os.path.abspath("./NetworkGraphs/Temp_Network/temp_network.html"))
self.main_window_object.ssh_window = ssh_connection(self.main_window_object)
self.main_window_object.edge_window = edge_editors(self.main_window_object)
self.close()
def generate_graph(arbre):
got_net = Network(height="100%",
width="100%",
bgcolor="#222222",
font_color="white")
dico_couleur = {}
for streamer in arbre.streamers:
got_net.add_node(streamer.name,
value=streamer.poids,
shape='image',
image=streamer.photo_profil,
size=100)
dico_couleur[streamer.name] = streamer.couleur
for depart, arrivee in arbre.groupes_migrants:
for viewer in arbre.groupes_migrants[depart, arrivee]:
got_net.add_node(viewer, value=1, color=dico_couleur[depart])
got_net.add_edge(viewer, depart, color=dico_couleur[depart])
got_net.add_edge(viewer, arrivee, color=dico_couleur[arrivee])
got_net.show_buttons()
got_net.set_edge_smooth(smooth_type='dynamic')
got_net.force_atlas_2based(gravity=-200,
central_gravity=0.01,
spring_length=1,
spring_strength=0.08,
damping=0.4,
overlap=0)
got_net.save_graph("./graphs_v1/" + 'streamgame_v1_42' + ".html")
return
def getCourseText():
"""
Visualize courses and their assigned texts.
"""
coursesAndTexts = g.query("""
select distinct ?courseName ?textTitle ?authorLast where {
?a a ccso:Course .
?a ccso:csName ?courseName .
?a ccso:hasLM ?t .
?t res:resource ?doc .
?doc dcterms:title ?textTitle .
?doc dcterms:creator ?author .
?author foaf:surname ?authorLast .
} limit 50""")
for line in coursesAndTexts:
print(line)
net = Network(height='750px', width='100%')
for courseName, textTitle, authorLast in coursesAndTexts:
net.add_node(courseName, shape='square', title=str(courseName))
net.add_node(textTitle,
shape='circle',
label=str(authorLast),
title=str(textTitle))
net.add_edge(courseName, textTitle, title="hasLM")
net.save_graph('../website/graph-viz.html')
def graph_print():
dist = loadCompleteDistro()
matrix = numpy.zeros((len(dist), len(dist)))
i = 0
labels = {}
net = Network()
k=0
caster = {"nld": "Niderlandzki", "gle": "Irlandzki", "spa": "Hiszpański", "fra": "Francuski", "eng": "Angielski", "slk": "Słowacki", "fin": "Fiński", "est": "Estoński", "lit": "Litewski", "hrv": "Chorwacki", "por": "Portugalski", "swe": "Szwedzki", "ron": "Romański", "dan": "Duński", "slv": "Słoweński", "bul": "Bułgarski", "ita": "Włoski", "ell": "Grecki", "mlt": "Maltański", "lav": "Łotewski", "ces": "Czeski", "pol": "Polski", "deu": "Niemiecki", "hun": "Węgierski"}
for lang1 in dist.keys():
print(lang1)
net.add_node(k + 1, label=f"{caster[lang1]}")
k+=1
xD = []
for lang1 in dist.keys():
labels[i] = lang1
j = 0
for lang2 in dist.keys():
if lang1 != lang2:
xD.append([i+1, j+1,round(KullbackLeibnerIteration(dist[lang1].alain(dist[lang2]), dist[lang2].alain(dist[lang1])),2)])
else:
pass
j += 1
i += 1
#G = nx.from_numpy_matrix(matrix)
#H = nx.relabel_nodes(G, labels)
#vg = []
#for u in H.edges.data():
# vg.append(u[2]["weight"])
#
xD.sort(key=lambda x: x[2])
xDD = xD[:40]
for x in xDD:
print(x[2])
net.add_edge(x[0],x[1],value=1.0/x[2], label=str(x[2]))
#
#Z = copy.deepcopy(H)
#for u in Z.edges.data():
# if u[2]["weight"] > median:
# H.remove_edge(u[0], u[1])
#
#pos = nx.spring_layout(H,k=1, iterations=20)
#plt.figure(3, figsize=(10, 10))
#labels = nx.get_edge_attributes(H, 'weight')
#nx.draw_networkx(H,pos=pos, with_labels=True,node_size=500)
#nx.draw_networkx_edge_labels(H, pos, edge_labels=labels)
#plt.show()
net.save_graph("my_graph.html")
net.toggle_physics(True)
net.show_buttons(filter_=['nodes', 'edges', 'physics'])
net.show('mygraph.html')
def plot_bacteria_intraction_network(bacteria, node_list, node_size_list, edge_list, color_list, G_name, folder,
color_by_tax_level=2, directed_G=True, control_color_and_shape=True):
# create the results folder
if not os.path.exists(folder):
os.mkdir(folder)
nodes_colors, nodes_shapes, group_list, tax_to_color_and_shape_map = \
get_nodes_colors_by_bacteria_tax_level(bacteria, G_name, taxnomy_level=color_by_tax_level, folder=folder)
bact_short_names = [shorten_single_bact_name(b) for b in bacteria]
nodes_with_edges = np.unique(np.array(edge_list).flatten()).tolist()
net = Network(height="750px", width="100%", bgcolor="#FFFFFF", font_color="black", directed=directed_G)
#net.barnes_hut(gravity=-120000)
net.force_atlas_2based()
# for the nodes: you can use either only the group option the automatically colors the group in different colors
# shaped like dots - no control, or use color and shape to make it you own, in this case group is irrelevant
for i, node in enumerate(node_list):
if node in nodes_with_edges:
if control_color_and_shape:
net.add_node(node, label=bact_short_names[i], color=nodes_colors[i], value=node_size_list[i],
shape=nodes_shapes[i], group=group_list[i])
else:
net.add_node(node, label=bact_short_names[i], value=node_size_list[i],
group=group_list[i])
# for the edges, the colors are what you decide and send
for i, (u, v) in enumerate(edge_list):
net.add_edge(u, v, color=color_list[i])
net.save_graph(os.path.join(folder, G_name + ".html"))
def generate_graph(arbre):
got_net = Network(height="100%",
width="100%",
bgcolor="#222222",
font_color="white")
got_net.add_node(arbre.name,
value=(1000 * len(arbre.children)**5),
color='#FFFF00')
recursive(arbre, got_net, arbre.name)
got_net.show_buttons()
got_net.save_graph("../data/graphs/" + arbre.name + ".html")
def visualizeGraph(nxGraph, save=False, size=['500px', '500px']):
""" Converts from networkx to a pyvis Network.
"""
subGViz = Network(size[0], size[1])
from_nx(subGViz, nxGraph)
if save:
subGViz.save_graph(save)
else:
subGViz.write_html('graph.html')
#subGViz.show('subGViz.html')
return subGViz
def visualize(edges, error_codes, resouce_pages, args):
G = nx.DiGraph()
G.add_edges_from(edges)
if args.save_txt is not None or args.save_npz is not None:
nodes = list(G.nodes())
adj_matrix = nx.to_numpy_matrix(G, nodelist=nodes, dtype=int)
if args.save_npz is not None:
base_fname = args.save_npz.replace('.npz', '')
scipy.sparse.save_npz(args.save_npz,
scipy.sparse.coo_matrix(adj_matrix))
else:
base_fname = args.save_txt.replace('.txt', '')
np.savetxt(args.save_txt, adj_matrix, fmt='%d')
node_info = get_node_info(nodes, error_codes, resource_pages, args)
with open(base_fname + '_nodes.txt', 'w') as f:
f.write('\n'.join(
[nodes[i] + '\t' + node_info[i] for i in range(len(nodes))]))
net = Network(width=args.width, height=args.height, directed=True)
net.from_nx(G)
if args.show_buttons:
net.show_buttons()
elif args.options is not None:
try:
with open(args.options, 'r') as f:
net.set_options(f.read())
except FileNotFoundError as e:
print('Error: options file', args.options, 'not found.')
except Exception as e:
print('Error applying options:', e)
for node in net.nodes:
node['size'] = 15
node['label'] = ''
if node['id'].startswith(args.site_url):
node['color'] = INTERNAL_COLOR
if node['id'] in resouce_pages:
node['color'] = RESOURCE_COLOR
else:
node['color'] = EXTERNAL_COLOR
if node['id'] in error_codes:
node[
'title'] = f'{error_codes[node["id"]]} Error: <a href="{node["id"]}">{node["id"]}</a>'
node['color'] = ERROR_COLOR
else:
node['title'] = f'<a href="{node["id"]}">{node["id"]}</a>'
net.save_graph(args.vis_file)
def Data_Visualisation (Algor1, Algor2):
net = Network()
inputFile = open(r"inputFile.txt","r+")
firstline = inputFile.readline()
[n,m] = firstline.split()
n = int(n)
m = int(m)
sizes = Algor1
for node in range(n):
net.add_node(node, label=node.__str__(),size=float(sizes[node].split()[0])*10)
input_lines = inputFile.readlines()
for line in input_lines :
edgelist = line.split()
net.add_edge(int(edgelist[0]), int(edgelist[1]), title = int(edgelist[2].__str__()))
net.save_graph('Degree Centrality.html')
net = Network()
inputFile = open(r"inputFile.txt", "r+")
firstline = inputFile.readline()
[n, m] = firstline.split()
n = int(n)
m = int(m)
sizes = Algor2
for node in range(n):
net.add_node(node, label=node.__str__(), size=float(sizes[node].split()[0]) * 40)
input_lines = inputFile.readlines()
for line in input_lines:
edgelist = line.split()
net.add_edge(int(edgelist[0]), int(edgelist[1]), title=int(edgelist[2].__str__()))
net.save_graph('Closeness Centrality.html')
def pyvis_draw(self, graph, name):
cnames = ['blue', 'green', 'red', 'cyan', 'orange',
'black', 'purple', 'purple', 'purple']
G = Network(height="800px", width="100%", directed=True)
for src, edge, dst in graph:
if edge == self.om.type and not self.config.show_url:
continue
src_type = [x for x in graph[src:self.om.type]]
dst_type = [x for x in graph[dst:self.om.type]]
if len(src_type):
if str(Flavor.FUNCTION) == str(src_type[0]) or str(Flavor.METHOD) == str(src_type[0]):
G.add_node(src, title=src, physics=True, color=cnames[0])
elif str(Flavor.CLASS) == str(src_type[0]) or str(Flavor.MODULE) == str(src_type[0]):
G.add_node(src, title=src, physics=True, color=cnames[1])
else:
G.add_node(src, title=src, physics=True, color=cnames[2])
else:
G.add_node(src, title=src, physics=True, color=cnames[3])
if len(dst_type):
if str(Flavor.FUNCTION) == str(dst_type[0]) or str(Flavor.METHOD) == str(dst_type[0]):
G.add_node(dst, title=dst, physics=True, color=cnames[0])
elif str(Flavor.CLASS) == str(dst_type[0]) or str(Flavor.MODULE) == str(dst_type[0]):
G.add_node(dst, title=dst, physics=True, color=cnames[1])
else:
G.add_node(dst, title=dst, physics=True, color=cnames[2])
else:
G.add_node(dst, title=dst, physics=True, color=cnames[3])
G.add_edge(src, dst, width=0.5, title=str(edge), physics=True)
G.hrepulsion(node_distance=120,
central_gravity=0.0,
spring_length=100,
spring_strength=0.01,
damping=0.09)
# G.show_buttons(filter_=['physics'])
G.save_graph("%s.html" % name)
def sequence_sample(self, sess, chars, vocab):
with open('data/review_example_sequence.txt') as f:
data = f.read()
state = sess.run(self.cell.zero_state(1, tf.float32))
x = np.zeros((1, 1))
for activity in data.split("->"):
x[0, 0] = vocab[activity]
feed = {self.input_data: x, self.initial_state: state}
[probs, state] = sess.run([self.probs, self.final_state], feed)
ps = probs[0]
if activity == "time-out 2":
G = Network(height=1000, width=1000, directed=True)
G.add_node(activity, level=0)
index = 0
for p in ps:
if p > 0.01:
if chars[index] == "\n":
index += 1
continue
print(activity + "\t\t" + chars[index] + "\t{%f}" % p)
G.add_node(chars[index], physics=True, level=1)
G.add_edge(activity,
chars[index],
value=int(p * 100),
title=int(p * 100),
physics=True,
arrowStrikethrough=False)
index += 1
print()
G.barnes_hut(gravity=-10000)
# G.show_buttons()
G.save_graph(
os.path.join(
"results",
re.sub('[-=.#/?:$}]', '', activity) +
"_sequence.html"))
def _graph(self):
try:
counter = {}
net = Network(height="100%", width="100%", bgcolor="white", font_color="white")
net.force_atlas_2based(gravity=-250, central_gravity=0.005, spring_length=0, spring_strength=0, damping=1, overlap=1)
exists = []
for entry in self.edge_data:
src = entry['challenger_id']
pokemon = entry['challenger']
if src in exists:
continue
t = "00"
if 'alolan' in pokemon.lower():
t = "61"
my_img = '/static/icons/pokemon_icon_%s_%s.png' % (f'{src:03}', t)
net.add_node(src, ' ', image=my_img, shape='circularImage', size=50, color={'highlight': {'border': entry['color']}}, borderWidthSelected=5)
exists.append(src)
fix_list = [346,26,82]
for src in fix_list:
my_img = '/static/icons/pokemon_icon_%s_%s.png' % (f'{src:03}', t)
net.add_node(src, ' ', image=my_img, shape='circularImage', size=50, color={'highlight': {'border': entry['color']}}, borderWidthSelected=5)
for entry in self.edge_data:
src = entry['challenger_id']
pokemon = entry['challenger']
opponent = entry['opponent']
if opponent.lower() not in self.include:
continue
if pokemon.lower() not in self.include:
continue
dst = entry['opponent_id']
w = entry['weight']/10
net.add_edge(src, dst, value=w, arrows='to', arrowStrikethrough=False, smooth=True, color={"color": entry['color'], "highlight": entry['color'], "opacity": 0.05}, selfReferenceSize=50)
net.set_edge_smooth('continuous')
#net.show_buttons(filter_=['physics'])
os.makedirs("/data/%s" % self.config['settings']['cup'], exist_ok=True)
net.save_graph("/data/%s/%svs%s.html" % (self.config['settings']['cup'], self.config['settings']['my_shield'], self.config['settings']['op_shield']))
return True
except Exception as e:
traceback.print_exc()
logging.info(e)
def render_instance(self,
render_graph_html,
iteration,
circle_bool,
hide_edges_bool,
hide_weight,
h=500,
w=750):
"""
Render a networkx graph in screen
"""
# print("Changed to: ", iteration)
self.iteration = iteration
# If some propriety change, update value and reload the Networx Graph
if self.circle_bool != circle_bool or self.hide_edges_bool != hide_edges_bool or self.hide_weight != hide_weight:
self.circle_bool = circle_bool
self.hide_edges_bool = hide_edges_bool
self.hide_weight = hide_weight
self.load_nx_graph()
self.load_solution(iteration)
# Translate to pyvis network
nt = Network(f'{h}px', f'{w}px', font_color='white')
nt.from_nx(self.nx_graph)
nt.toggle_physics(False)
path = f'network.html'
nt.save_graph(path)
HtmlFile = open(path, 'r') # , encoding='utf-8')
source_code = HtmlFile.read()
# Clean screen before plot
if render_graph_html != None:
render_graph_html.empty()
render_html = components.html(source_code,
height=h * 1.1,
width=w * 1.1)
return render_html
def getCourseText2():
"""
Visualize courses and their assigned texts.
"""
coursesAndTexts = g.query("""
select distinct ?courseName ?textTitle ?authorLast where {
?a a ccso:Course .
?a ccso:csName ?courseName .
?a ccso:hasLM ?t .
?t res:resource ?doc .
?doc dcterms:title ?textTitle .
?doc dcterms:creator ?author .
?author foaf:surname ?authorLast .
} limit 400""")
for line in coursesAndTexts:
print(line)
net = Network(height='750px', width='100%')
for courseName, textTitle, authorLast in coursesAndTexts:
net.add_node(courseName, shape='square', title=str(courseName))
net.add_node(textTitle,
shape='circle',
label=str(authorLast),
title=str(textTitle))
net.add_edge(courseName, textTitle, title="hasLM")
out = ""
nodesJson = json.dumps(net.nodes)
edgesJson = json.dumps(net.edges)
template = f"""
nodes = new vis.DataSet({nodesJson})
edges = new vis.DataSet({edgesJson})
data = {{nodes: nodes, edges: edges}};
"""
print(template)
net.save_graph('../website/graph-vis.html')
def make_graph(self):
graph_data = []
for key in self.ner_dct.keys():
lst = self.ner_dct[key]
for x in lst:
graph_data.append((key, x[0]))
poi_net = Network(height="750px",
width="100%",
bgcolor="#ffffff",
font_color="blue")
poi_net.barnes_hut()
for i in graph_data:
src = i[0]
dst = i[1]
poi_net.add_node(src, label=src, title="src", color="black")
poi_net.add_node(dst, label=dst, title="dst", color="#dd4b39")
poi_net.add_edge(src, dst)
#change this location
poi_net.save_graph(
"/Users/ankitanand/Box/UB/Fall 2019/IR/Proj1/poi.html")
def interactive_plot(self, df):
plt.figure(figsize=(20, 10))
G = nx.from_pandas_edgelist(df,
source='source',
target='target',
create_using=nx.DiGraph())
node_limit = len(G.nodes)
pos = graphviz_layout(G, prog='dot')
nx.draw(G,
pos,
with_labels=True,
node_size=5000,
font_size=20,
font_color='black',
node_color=range(node_limit),
cmap=plt.cm.spring)
plt.title('DOM Tree Visualization')
nt = Network("500px", "500px")
nt.from_nx(G)
nt.toggle_physics(True)
nt.save_graph('dom_tree_inter_viz.html')
def Create_Graph(nodes, connections, output_type='saveshow'):
graph = networkx.Graph()
for node in nodes:
graph.add_node(node['label'], title=node['title'])
node = graph.nodes[node['label']]
print('Woo')
print(node)
#for arg in params:
#graph[label][arg]=params[arg]
for edge in edges:
graph.add_edge(*edge)
for arg in edges[edge]:
graph[edge[0]][edge[1]][arg] = edges[edge][arg]
network = Network(bgcolor='#222222', font_color='white')
network.from_nx(graph)
network.save_graph('graph1.html')
network.show('graph1.html')
def create_graph(graph_structure: Dict[str, Set[Tuple[str, str]]],
graph_structure_attributes: Dict[str,
Set[str]], filepath: str):
"""
Creates and save graph from given processed relationships
:param graph_structure: processed relationships
:param graph_structure_attributes: attributes for every node
:param filepath: path in which we like to save file
:return: None
"""
graph = Network(directed=True, height='800px', width='800px')
for el1, connected_elements in graph_structure.items():
if el1 not in graph.nodes:
graph.add_node(el1)
for connected_element in connected_elements:
el2 = connected_element[1]
predicate = connected_element[0]
if el2 not in graph.nodes:
graph.add_node(el2)
graph.add_edge(el1, el2, title=predicate, label=predicate)
for subject, attributes in graph_structure_attributes.items():
if subject not in graph.nodes:
graph.add_node(subject)
for attribute in attributes:
if attribute not in graph.nodes:
graph.add_node(attribute, color='red')
graph.add_edge(subject, attribute, color='red')
graph.save_graph(filepath)
def pyvis_guanxi():
from pyvis.network import Network
net = Network()
# 添加单个节点
net.add_node(1, label='node_1')
net.add_node(2, label='node_2')
# 添加属性,注意这⾥里里是添加多个节点，使⽤用add_nodes
# 属性值传⼊入的是列列表，要保证列列表⻓长度与节点列列表⻓长度⼀一致
net.add_nodes([3, 4, 5],
value=[10, 20, 30],
title=["节点3", "节点4", "节点5"],
x=[21.4, 54.2, 11.2],
y=[100.2, 23.54, 32.1],
label=["node_3", "node_4", "node_5"],
color=["#00ff1e", "#162347", "#dd4b39"])
# ⼀一次添加⼀一条边
net.add_edge(1, 2)
net.add_edge(1, 3)
net.add_edge(1, 4)
net.add_edge(2, 5)
# 有权重边
net.add_edge(2, 1, wright=0.87)
net.add_edge(2, 3, wright=0.5)
'''
#结合networkx使⽤用
import networkx as nx
# ⽣生成5个节点，每个点与其他所有点相互联系
nxg = nx.complete_graph(5)
G = Network()
G.from_nx(nxg)
# 输出并⽣生成demo2.html
G.show('demo2.html')
'''
# 交互式显示，# 可对打开的html进⾏行行调整
net.show_buttons(filter_=['physics'])
net.save_graph(
'app/templates/render/pyvis_guanxi.html') #输出并⽣生成pyvis_guanxi.html
return render_template('render/pyvis_guanxi.html')
def generate_trustline_graph(data):
'''
This function generates a network graph using pyvis module. The graph is
saved in Analysis folder.
Parameters
----------
data : Filtered transactions data.
Returns
-------
None.
'''
graphNodeList=[] # creating empty nodes list for graph
for acc in data.index: # adding data to graph nodes list
graphNodeList.append(data['account'][acc]) # appending account
graphNodeList.append(data['issuer'][acc]) # appending issuer
graphNodeList=list(set(graphNodeList)) # filtering graph node list for unique node list
graphEdgeList=[] # create graph's edge list
for acc in data.index: # adding data to graph's edges
if(data['value'][acc]!=0): # don't add edge if trustline is set to 0
graphEdgeList.append((data['account'][acc],data['issuer'][acc],data['currency'][acc],data['value'][acc])) # adding edge with account, issuer, currency and value
nx_graph = Network(height="750px", width="100%", bgcolor="#222222", font_color="white",directed="true") # setting parameters for visualising the graph.
for node in graphNodeList: # adding nodes to graph
nx_graph.add_node(node, size=20, title=node)
for edge in graphEdgeList: # adding edges to graph
weigh= str(edge[3])+' '+str(edge[2])
nx_graph.add_edge(edge[0], edge[1], weight=weigh, title=weigh) # adding edge with weights to graph
neighbor_map = nx_graph.get_adj_list()
for node in nx_graph.nodes:
node["title"] += " Trusted:<br>" + "<br>".join(neighbor_map[node["id"]]) # creating title for each node
node["value"] = len(neighbor_map[node["id"]]) # setting value of node
nx_graph.save_graph(r"../Analysis/TrustSetGraph.html") # saving the graph in HTML format
nx_graph.show_buttons(filter_=['physics'])
nx_graph.show("TrustSetGraph.html") # open the graph in browser
for i in range(0, x): #itterate over 0-35 for every HLA in HLAs
if dst in IDs[i]: #if target in one of the HLA lists..
name = IDs[i].split()[0] #get the HLA-type of this target
prob_net.add_node(
dst, dst, title=dst, color=Ncolors[name]
) #add target node to prob_net with color assigned to the hla-type in Ncolors
prob_net.add_edge(src, dst) #add edge between source and target
neighbor_map = prob_net.get_adj_list()
# add neighbor data to node hover data
for node in prob_net.nodes:
node["title"] += " Neighbors:<br>" + "<br>".join(neighbor_map[node["id"]])
node["value"] = len(neighbor_map[node["id"]])
#save graph as TCR.html
prob_net.save_graph("D:/gliph-1.0/gliph/bin/TCR.html")
#make a legend in html language
legenda = "<div> \n<h3>Legend HLA types</h3> \n Patient ID : occurence"
for key, value in Ncolors.items():
legenda += "<p><font color=" + value + ">" + key + "=" + str(
occ[key]) + "</p> \n"
legenda += "</div> \n"
#open the previously saved TCR.html, remove the "</body> </html>" add the legend and add "</body> </html>" again. then write to the appropriate file
with open('D:/gliph-1.0/gliph/bin/TCR.html', 'r') as f:
html = f.read()
htmls = html[-0:-15]
htmll = htmls + legenda
htmlf = htmll + html[-15:]
H = open("D:/gliph-1.0/gliph/bin/TCRL457ID.html", "w+")
def map_data(qajar_data,
name_color='#a0b038',
relative_color='#38b03e',
edge_color='#018786',
name_shape='ellipse',
relative_shape='box',
buttons=False,
name_size=100,
relative_size=100,
name_title='',
name_image='',
relative_image=''):
# make a graph named g
g = Network(height='1500px',
width='100%',
bgcolor='black',
font_color='white')
# show buttons if we call this fuction later with argument buttons=True
if buttons == True:
g.width = '65%'
g.show_buttons()
# make an empty list (qajarnames) to fill it by all the names
qajarnames = []
# parsing the data in json file
for qajar in qajar_data:
nam = (qajar['name'])
namimag = (qajar['image'])
relat = (qajar['relatives'])
# set the number of relatives of each name to size of its node
namessize = len(relat) * 10
# add a node for each name in data
g.add_node(nam,
color=name_color,
shape=name_shape,
image=namimag,
physics=False,
size=namessize,
title=relat)
# fill the list of all names
qajarnames.append(nam)
# add edges between each name and its relative if...
for qajar in qajar_data:
nam = (qajar['name'])
namimag = (qajar['image'])
relat = (qajar['relatives'])
# if the relative has node already...
for eachrelative in relat:
# if the relative doesn't have node, add one(this didn't occur in our case)
if eachrelative not in qajarnames:
g.add_node(eachrelative,
color=relative_color,
shape=relative_shape,
physics=False,
size=relative_size)
# add edges between names and their relatives
g.add_edge(nam, eachrelative, color=edge_color)
# the algorithm for model for visualizing data
g.barnes_hut()
# show an html of this interactive graph
g.show('qajar.html')
g.save_graph('qajar.html')
def draw_graph(self, option="All", to_ip="All", from_ip="All"):
#f = Digraph('network_diagram - '+option, filename=self.filename, engine="dot", format="png")
#f.attr(rankdir='LR', size='8,5')
if len(memory.lan_hosts) > 40:
f = Digraph('network_diagram - ' + option,
filename=self.filename,
engine="sfdp",
format="png")
elif len(memory.lan_hosts) > 20:
f = Digraph('network_diagram - ' + option,
filename=self.filename,
engine="circo",
format="png")
else:
f = Digraph('network_diagram - ' + option,
filename=self.filename,
engine="dot",
format="png")
interactive_graph = Network(directed=True,
height="750px",
width="100%",
bgcolor="#222222",
font_color="white")
interactive_graph.barnes_hut()
vis_nodes = []
vis_edges = []
f.attr('node', shape='doublecircle')
#f.node('defaultGateway')
f.attr('node', shape='circle')
print("Starting Graph Plotting")
edge_present = False
mal, tor, http, https, icmp, dns, clear_text, unknown, covert = 0, 0, 0, 0, 0, 0, 0, 0, 0
if option == "All":
# add nodes
for session in self.sessions:
src, dst, port = session.split("/")
#print(from_ip, to_ip, src, dst)
if (src == from_ip and dst == to_ip) or \
(from_ip == "All" and to_ip == "All") or \
(to_ip == "All" and from_ip == src) or \
(to_ip == dst and from_ip == "All"):
# TODO: Improvise this logic below
# * graphviz graph is not very good with the ":" in strings
if ":" in src:
map_src = src.replace(":", ".")
else:
map_src = src
if ":" in dst:
map_dst = dst.replace(":", ".")
else:
map_dst = dst
# Lan Host
if memory.packet_db[session]["Ethernet"][
"src"] not in memory.lan_hosts:
curr_node = map_src + "\n" + memory.packet_db[session][
"Ethernet"]["src"].replace(":", ".")
f.node(curr_node)
else:
curr_node = memory.lan_hosts[memory.packet_db[session][
"Ethernet"]["src"]]["node"]
f.node(curr_node)
# Destination
if dst in memory.destination_hosts:
if memory.destination_hosts[dst][
"mac"] in memory.lan_hosts:
destination = memory.lan_hosts[
memory.destination_hosts[dst]["mac"]]["node"]
dlabel = memory.destination_hosts[dst][
"domain_name"]
else:
destination = memory.destination_hosts[dst][
"mac"].replace(":", ".")
destination += "\n" + "PossibleGateway"
dlabel = memory.destination_hosts[dst][
"domain_name"]
else:
if memory.packet_db[session]["Ethernet"][
"dst"] in memory.lan_hosts:
destination = memory.lan_hosts[memory.packet_db[
session]["Ethernet"]["dst"]]["node"]
dlabel = ""
else:
destination = memory.packet_db[session][
"Ethernet"]["dst"].replace(":", ".")
destination += "\n" + "PossibleGateway"
dlabel = ""
# Interactive Graph on Beta, so for now add safety checks ( potential failures in python2)
try:
interactive_graph.add_node(str(curr_node),
str(curr_node),
title=str(curr_node),
color="yellow")
interactive_graph.add_node(str(destination),
str(destination),
title=str(destination),
color="yellow")
except Exception as e:
print("Interactive graph error occurred: " + str(e))
#if (curr_node, curr_node, title=curr_node, color="yellow") not in vis_nodes:
# vis_nodes.append((curr_node, curr_node, title=curr_node, color="yellow"))
#if (destination, destination, title=destination, color="yellow") not in vis_nodes:
# vis_nodes.append((destination, destination, title=destination, color="yellow"))
if curr_node != destination:
if session in memory.possible_tor_traffic:
f.edge(curr_node,
destination,
label='TOR: ' + str(map_dst),
color="white")
tor += 1
#interactive_graph.add_edge(curr_node, destination, color="white", value=tor/100, smooth={type: "curvedCCW", roundness: 0.4})
interactive_graph.add_edge(curr_node,
destination,
color="white",
title='TOR: ' +
str(map_dst),
smooth={
"type": "curvedCW",
"roundness": tor / 9
})
#if edge not in vis_edges:toor
# vis_edges.append(edge)
if edge_present == False:
edge_present = True
elif memory.packet_db[session]["covert"]:
if port == "53":
protocol = "DNS"
else:
protocol = port
f.edge(curr_node,
destination,
label='Covert/' + protocol + ': ' +
str(map_dst) + ": " + str(dlabel),
color="cyan")
covert += 1
interactive_graph.add_edge(
curr_node,
destination,
color="cyan",
title='Covert: ' + str(map_dst) + ": " +
dlabel,
smooth={
"type": "curvedCCW",
"roundness": covert / 12
})
if edge_present == False:
edge_present = True
else:
if port == "443":
f.edge(curr_node,
destination,
label='HTTPS: ' + str(map_dst) + ": " +
str(dlabel),
color="blue")
https += 1
interactive_graph.add_edge(
curr_node,
destination,
color="blue",
title='HTTPS: ' + str(map_dst) + ": " +
dlabel,
smooth={
"type": "curvedCCW",
"roundness": https / 10
})
#if edge not in vis_edges:
# vis_edges.append(edge)
if edge_present == False:
edge_present = True
elif port == "80":
f.edge(curr_node,
destination,
label='HTTP: ' + str(map_dst) + ": " +
str(dlabel),
color="green")
http += 1
interactive_graph.add_edge(
curr_node,
destination,
color="green",
title='HTTP: ' + str(map_dst) + ": " +
dlabel,
smooth={
"type": "curvedCW",
"roundness": http / 12
})
#if edge not in vis_edges:
# vis_edges.append(edge)
if edge_present == False:
edge_present = True
elif port == "ICMP":
f.edge(curr_node,
destination,
label='ICMP: ' + str(map_dst),
color="black")
icmp += 1
interactive_graph.add_edge(curr_node,
destination,
color="purple",
title='ICMP: ' +
str(map_dst),
smooth={
"type":
"curvedCCW",
"roundness":
icmp / 6
})
#if edge not in vis_edges:
# vis_edges.append(edge)
if edge_present == False:
edge_present = True
elif port == "53":
f.edge(curr_node,
destination,
label='DNS: ' + str(map_dst),
color="orange")
dns += 1
interactive_graph.add_edge(
curr_node,
destination,
color="pink",
title='DNS: ' + str(map_dst),
smooth={
"type": "curvedCW",
"roundness": dns / 5
})
#if edge not in vis_edges:
# vis_edges.append(edge)
if edge_present == False:
edge_present = True
elif int(port) in [
20, 21, 23, 25, 110, 143, 139, 69, 161,
162, 1521
]:
f.edge(curr_node,
destination,
label='ClearTextProtocol/' + port +
': ' + str(map_dst),
color="violet")
clear_text += 1
interactive_graph.add_edge(
curr_node,
destination,
color="#9A2EFE",
title='ClearTextProtocol/' + port + ': ' +
str(map_dst),
smooth={
"type": "curvedCW",
"roundness": clear_text / 4
})
if edge_present == False:
edge_present = True
else:
f.edge(curr_node,
destination,
label='UnknownProtocol/' + port + ': ' +
str(map_dst),
color="brown")
unknown += 1
interactive_graph.add_edge(
curr_node,
destination,
color="brown",
title='UnknownProtocol/' + port + ': ' +
str(map_dst),
smooth={
"type": "curvedCW",
"roundness": unknown / 3
})
if edge_present == False:
edge_present = True
else:
# This block was just added to handle MAC SPOOF scenario
# * Most of the CTF Challenges have fake identical MACs that need to be displayed
if map_src in curr_node:
other_node = map_dst + "\n" + memory.packet_db[
session]["Ethernet"]["dst"].replace(":", ".")
else:
other_node = map_src + "\n" + memory.packet_db[
session]["Ethernet"]["src"].replace(":", ".")
f.node(other_node)
interactive_graph.add_node(str(other_node),
str(other_node),
title=str(other_node),
color="yellow")
f.edge(curr_node,
other_node,
label='WeirdTraffic/' + port,
color="pink")
unknown += 1
interactive_graph.add_edge(curr_node,
other_node,
color="pink",
title='WeirdTraffic/' +
port,
smooth={
"type": "curvedCW",
"roundness": unknown / 3
})
if edge_present == False:
edge_present = True
elif option == "HTTP":
for session in self.sessions:
src, dst, port = session.split("/")
if (src == from_ip and dst == to_ip) or \
(from_ip == "All" and to_ip == "All") or \
(to_ip == "All" and from_ip == src) or \
(to_ip == dst and from_ip == "All"):
# TODO: Improvise this logic below
# * graphviz graph is not very good with the ":" in strings
if ":" in src:
map_src = src.replace(":", ".")
else:
map_src = src
if ":" in dst:
map_dst = dst.replace(":", ".")
else:
map_dst = dst
# Lan Host
if memory.packet_db[session]["Ethernet"][
"src"] not in memory.lan_hosts:
curr_node = map_src + "\n" + memory.packet_db[session][
"Ethernet"]["src"].replace(":", ".")
f.node(curr_node)
else:
curr_node = memory.lan_hosts[memory.packet_db[session][
"Ethernet"]["src"]]["node"]
f.node(curr_node)
# Destination
if dst in memory.destination_hosts:
if memory.destination_hosts[dst][
"mac"] in memory.lan_hosts:
destination = memory.lan_hosts[
memory.destination_hosts[dst]["mac"]]["node"]
dlabel = memory.destination_hosts[dst][
"domain_name"]
else:
destination = memory.destination_hosts[dst][
"mac"].replace(":", ".")
destination += "\n" + "PossibleGateway"
dlabel = memory.destination_hosts[dst][
"domain_name"]
else:
if memory.packet_db[session]["Ethernet"][
"dst"] in memory.lan_hosts:
destination = memory.lan_hosts[memory.packet_db[
session]["Ethernet"]["dst"]]["node"]
dlabel = ""
else:
destination = memory.packet_db[session][
"Ethernet"]["dst"].replace(":", ".")
destination += "\n" + "PossibleGateway"
dlabel = ""
# Interactive Graph on Beta, so for now add safety checks ( potential failures in python2)
try:
interactive_graph.add_node(str(curr_node),
str(curr_node),
title=str(curr_node),
color="yellow")
interactive_graph.add_node(str(destination),
str(destination),
title=str(destination),
color="yellow")
except Exception as e:
print("Interactive graph error occurred: " + str(e))
if port == "80" and curr_node != destination:
f.edge(curr_node,
destination,
label='HTTP: ' + str(map_dst) + ": " + dlabel,
color="green")
http += 1
interactive_graph.add_edge(curr_node,
destination,
color="green",
smooth={
"type": "curvedCW",
"roundness": http / 10
})
if edge_present == False:
edge_present = True
elif option == "HTTPS":
for session in self.sessions:
src, dst, port = session.split("/")
if (src == from_ip and dst == to_ip) or \
(from_ip == "All" and to_ip == "All") or \
(to_ip == "All" and from_ip == src) or \
(to_ip == dst and from_ip == "All"):
# TODO: Improvise this logic below
# * graphviz graph is not very good with the ":" in strings
if ":" in src:
map_src = src.replace(":", ".")
else:
map_src = src
if ":" in dst:
map_dst = dst.replace(":", ".")
else:
map_dst = dst
# Lan Host
if memory.packet_db[session]["Ethernet"][
"src"] not in memory.lan_hosts:
curr_node = map_src + "\n" + memory.packet_db[session][
"Ethernet"]["src"].replace(":", ".")
f.node(curr_node)
else:
curr_node = memory.lan_hosts[memory.packet_db[session][
"Ethernet"]["src"]]["node"]
f.node(curr_node)
# Destination
if dst in memory.destination_hosts:
if memory.destination_hosts[dst][
"mac"] in memory.lan_hosts:
destination = memory.lan_hosts[
memory.destination_hosts[dst]["mac"]]["node"]
dlabel = memory.destination_hosts[dst][
"domain_name"]
else:
destination = memory.destination_hosts[dst][
"mac"].replace(":", ".")
destination += "\n" + "PossibleGateway"
dlabel = memory.destination_hosts[dst][
"domain_name"]
else:
if memory.packet_db[session]["Ethernet"][
"dst"] in memory.lan_hosts:
destination = memory.lan_hosts[memory.packet_db[
session]["Ethernet"]["dst"]]["node"]
dlabel = ""
else:
destination = memory.packet_db[session][
"Ethernet"]["dst"].replace(":", ".")
destination += "\n" + "PossibleGateway"
dlabel = ""
# Interactive Graph on Beta, so for now add safety checks ( potential failures in python2)
try:
interactive_graph.add_node(str(curr_node),
str(curr_node),
title=str(curr_node),
color="yellow")
interactive_graph.add_node(str(destination),
str(destination),
title=str(destination),
color="yellow")
except Exception as e:
print("Interactive graph error occurred: " + str(e))
if port == "443" and curr_node != destination:
f.edge(curr_node,
destination,
label='HTTPS: ' + str(map_dst) + ": " + dlabel,
color="blue")
https += 1
interactive_graph.add_edge(curr_node,
destination,
color="blue",
smooth={
"type": "curvedCCW",
"roundness": https / 10
})
if edge_present == False:
edge_present = True
elif option == "Tor":
for session in self.sessions:
src, dst, port = session.split("/")
if (src == from_ip and dst == to_ip) or \
(from_ip == "All" and to_ip == "All") or \
(to_ip == "All" and from_ip == src) or \
(to_ip == dst and from_ip == "All"):
# TODO: Improvise this logic below
# * graphviz graph is not very good with the ":" in strings
if ":" in src:
map_src = src.replace(":", ".")
else:
map_src = src
if ":" in dst:
map_dst = dst.replace(":", ".")
else:
map_dst = dst
# Lan Host
if memory.packet_db[session]["Ethernet"][
"src"] not in memory.lan_hosts:
curr_node = map_src + "\n" + memory.packet_db[session][
"Ethernet"]["src"].replace(":", ".")
f.node(curr_node)
else:
curr_node = memory.lan_hosts[memory.packet_db[session][
"Ethernet"]["src"]]["node"]
f.node(curr_node)
# Destination
if dst in memory.destination_hosts:
if memory.destination_hosts[dst][
"mac"] in memory.lan_hosts:
destination = memory.lan_hosts[
memory.destination_hosts[dst]["mac"]]["node"]
dlabel = memory.destination_hosts[dst][
"domain_name"]
else:
destination = memory.destination_hosts[dst][
"mac"].replace(":", ".")
destination += "\n" + "PossibleGateway"
dlabel = memory.destination_hosts[dst][
"domain_name"]
else:
if memory.packet_db[session]["Ethernet"][
"dst"] in memory.lan_hosts:
destination = memory.lan_hosts[memory.packet_db[
session]["Ethernet"]["dst"]]["node"]
dlabel = ""
else:
destination = memory.packet_db[session][
"Ethernet"]["dst"].replace(":", ".")
destination += "\n" + "PossibleGateway"
dlabel = ""
# Interactive Graph on Beta, so for now add safety checks ( potential failures in python2)
try:
interactive_graph.add_node(str(curr_node),
str(curr_node),
title=str(curr_node),
color="yellow")
interactive_graph.add_node(str(destination),
str(destination),
title=str(destination),
color="yellow")
except Exception as e:
print("Interactive graph error occurred: " + str(e))
if session in memory.possible_tor_traffic and curr_node != destination:
f.edge(curr_node,
destination,
label='TOR: ' + str(map_dst),
color="white")
tor += 1
interactive_graph.add_edge(curr_node,
destination,
color="white",
smooth={
"type": "curvedCW",
"roundness": tor / 10
})
if edge_present == False:
edge_present = True
elif option == "Malicious":
# TODO: would we need to iterate over and over all the session irrespective of the properties
for session in self.sessions:
src, dst, port = session.split("/")
if (src == from_ip and dst == to_ip) or \
(from_ip == "All" and to_ip == "All") or \
(to_ip == "All" and from_ip == src) or \
(to_ip == dst and from_ip == "All"):
# TODO: Improvise this logic below
# * graphviz graph is not very good with the ":" in strings
if ":" in src:
map_src = src.replace(":", ".")
else:
map_src = src
if ":" in dst:
map_dst = dst.replace(":", ".")
else:
map_dst = dst
# Lan Host
if memory.packet_db[session]["Ethernet"][
"src"] not in memory.lan_hosts:
curr_node = map_src + "\n" + memory.packet_db[session][
"Ethernet"]["src"].replace(":", ".")
f.node(curr_node)
else:
curr_node = memory.lan_hosts[memory.packet_db[session][
"Ethernet"]["src"]]["node"]
f.node(curr_node)
# Destination
if dst in memory.destination_hosts:
if memory.destination_hosts[dst][
"mac"] in memory.lan_hosts:
destination = memory.lan_hosts[
memory.destination_hosts[dst]["mac"]]["node"]
dlabel = memory.destination_hosts[dst][
"domain_name"]
else:
destination = memory.destination_hosts[dst][
"mac"].replace(":", ".")
destination += "\n" + "PossibleGateway"
dlabel = memory.destination_hosts[dst][
"domain_name"]
else:
if memory.packet_db[session]["Ethernet"][
"dst"] in memory.lan_hosts:
destination = memory.lan_hosts[memory.packet_db[
session]["Ethernet"]["dst"]]["node"]
dlabel = ""
else:
destination = memory.packet_db[session][
"Ethernet"]["dst"].replace(":", ".")
destination += "\n" + "PossibleGateway"
dlabel = ""
# Interactive Graph on Beta, so for now add safety checks ( potential failures in python2)
try:
interactive_graph.add_node(str(curr_node),
str(curr_node),
title=str(curr_node),
color="yellow")
interactive_graph.add_node(str(destination),
str(destination),
title=str(destination),
color="yellow")
except Exception as e:
print("Interactive graph error occurred: " + str(e))
if session in memory.possible_mal_traffic and curr_node != destination:
f.edge(curr_node,
destination,
label='Malicious: ' + str(map_dst),
color="red")
mal += 1
interactive_graph.add_edge(curr_node,
destination,
color="red",
smooth={
"type": "curvedCW",
"roundness": mal / 10
})
if edge_present == False:
edge_present = True
elif option == "ICMP":
for session in self.sessions:
src, dst, protocol = session.split("/")
if (src == from_ip and dst == to_ip) or \
(from_ip == "All" and to_ip == "All") or \
(to_ip == "All" and from_ip == src) or \
(to_ip == dst and from_ip == "All"):
if ":" in src:
map_src = src.replace(":", ".")
else:
map_src = src
if ":" in dst:
map_dst = dst.replace(":", ".")
else:
map_dst = dst
# Lan Host
if memory.packet_db[session]["Ethernet"][
"src"] not in memory.lan_hosts:
curr_node = map_src + "\n" + memory.packet_db[session][
"Ethernet"]["src"].replace(":", ".")
f.node(curr_node)
else:
curr_node = memory.lan_hosts[memory.packet_db[session][
"Ethernet"]["src"]]["node"]
f.node(curr_node)
# Destination
if dst in memory.destination_hosts:
if memory.destination_hosts[dst][
"mac"] in memory.lan_hosts:
destination = memory.lan_hosts[
memory.destination_hosts[dst]["mac"]]["node"]
dlabel = memory.destination_hosts[dst][
"domain_name"]
else:
destination = memory.destination_hosts[dst][
"mac"].replace(":", ".")
destination += "\n" + "PossibleGateway"
dlabel = memory.destination_hosts[dst][
"domain_name"]
else:
if memory.packet_db[session]["Ethernet"][
"dst"] in memory.lan_hosts:
destination = memory.lan_hosts[memory.packet_db[
session]["Ethernet"]["dst"]]["node"]
dlabel = ""
else:
destination = memory.packet_db[session][
"Ethernet"]["dst"].replace(":", ".")
destination += "\n" + "PossibleGateway"
dlabel = ""
# Interactive Graph on Beta, so for now add safety checks ( potential failures in python2)
try:
interactive_graph.add_node(str(curr_node),
str(curr_node),
title=str(curr_node),
color="yellow")
interactive_graph.add_node(str(destination),
str(destination),
title=str(destination),
color="yellow")
except Exception as e:
print("Interactive graph error occurred: " + str(e))
if protocol == "ICMP" and curr_node != destination:
f.edge(curr_node,
destination,
label='ICMP: ' + str(map_dst),
color="black")
icmp += 1
interactive_graph.add_edge(curr_node,
destination,
color="purple",
smooth={
"type": "curvedCCW",
"roundness": icmp / 10
})
if edge_present == False:
edge_present = True
elif option == "DNS":
for session in self.sessions:
src, dst, port = session.split("/")
if (src == from_ip and dst == to_ip) or \
(from_ip == "All" and to_ip == "All") or \
(to_ip == "All" and from_ip == src) or \
(to_ip == dst and from_ip == "All"):
if ":" in src:
map_src = src.replace(":", ".")
else:
map_src = src
if ":" in dst:
map_dst = dst.replace(":", ".")
else:
map_dst = dst
# Lan Host
if memory.packet_db[session]["Ethernet"][
"src"] not in memory.lan_hosts:
curr_node = map_src + "\n" + memory.packet_db[session][
"Ethernet"]["src"].replace(":", ".")
f.node(curr_node)
else:
curr_node = memory.lan_hosts[memory.packet_db[session][
"Ethernet"]["src"]]["node"]
f.node(curr_node)
# Destination
if dst in memory.destination_hosts:
if memory.destination_hosts[dst][
"mac"] in memory.lan_hosts:
destination = memory.lan_hosts[
memory.destination_hosts[dst]["mac"]]["node"]
dlabel = memory.destination_hosts[dst][
"domain_name"]
else:
destination = memory.destination_hosts[dst][
"mac"].replace(":", ".")
destination += "\n" + "PossibleGateway"
dlabel = memory.destination_hosts[dst][
"domain_name"]
else:
if memory.packet_db[session]["Ethernet"][
"dst"] in memory.lan_hosts:
destination = memory.lan_hosts[memory.packet_db[
session]["Ethernet"]["dst"]]["node"]
dlabel = ""
else:
destination = memory.packet_db[session][
"Ethernet"]["dst"].replace(":", ".")
destination += "\n" + "PossibleGateway"
dlabel = ""
# Interactive Graph on Beta, so for now add safety checks ( potential failures in python2)
try:
interactive_graph.add_node(str(curr_node),
str(curr_node),
title=str(curr_node),
color="yellow")
interactive_graph.add_node(str(destination),
str(destination),
title=str(destination),
color="yellow")
except Exception as e:
print("Interactive graph error occurred: " + str(e))
if port == "53" and curr_node != destination:
f.edge(curr_node,
destination,
label='DNS: ' + str(map_dst),
color="orange")
dns += 1
interactive_graph.add_edge(curr_node,
destination,
color="pink",
smooth={
"type": "curvedCW",
"roundness": dns / 10
})
if edge_present == False:
edge_present = True
if edge_present == False:
f.attr(label="No " + option + " Traffic between nodes!",
engine='circo',
size="5, 5",
dpi="300")
self.apply_styles(f, self.styles)
f.render()
interactive_graph.show_buttons(filter_=['physics'])
interactive_graph.save_graph(self.filename + ".html")
ret.append(
f"<div>closeness: {closeness_centrality[asn]:.4f}</div>")
ret.append(
f"<div>betweenness: {betweenness_centrality[asn]:.4f}</div>")
ret.append(f"<div>peer count: {len(fullasmap[asn])}</div>")
ret.append(f"<p></p><div>Peer list:</div><p></p>")
for peeras in fullasmap[asn]:
ret.append(
f"<div onclick=\"select_node({peeras});\">{asname(peeras)}</div>"
)
ret.append("<p></p><div>Registry:</div>")
_registry = asinfo(asn) #.replace("\n", "<br/>")
ret.append(f"<pre>{_registry}</pre>")
return "".join(ret)
for asn, peers in fullasmap.items():
size = node_centrality[asn]
net.add_node(asn,
label=asname(asn),
size=size,
title=gentitle(asn))
for asn, peers in asmap.items():
for p in peers:
net.add_edge(asn, p)
net.save_graph('index.html')
if __name__ == "__main__":
main()
def simulfunc():
try:
df_i = pd.read_csv('static/inputfile_arm.csv', header=None)
df_i = df_i.astype(str)
except FileNotFoundError:
df_i = pd.read_csv('static/store_data.csv', header=None)
df_i = df_i.astype(str)
# -----------------------------------------------------------------------------------------------------------------
records = []
for i in range(0, len(df_i)):
records.append([
str(df_i.values[i, j]) for j in range(0, 20)
if df_i.loc[i, j] != 'nan'
])
# -----------------------------------------------------------------------------------------------------------------
# Explore the data: use different support and confidence levels to see how number of apssociations
# change and what are the associated metrics: support, confidence, conviction, lift
itemsets, r = efficient_apriori.apriori(records,
min_support=0.02,
min_confidence=0.3)
s = []
cv = []
c = []
lift = []
nn = []
nd = {}
count = 0
for i in r:
s.append(i.support) # support measure
cv.append(i.conviction) # conviction measure
c.append(i.confidence) # condidence measure
lift.append(i.lift)
n = []
for m in i.lhs:
n.append(m)
for l in i.rhs:
n.append(l)
nd[count] = n
count += 1
nn.append(n)
figbar, axbar = plt.subplots(1, 1, figsize=(15, 15))
ax = axbar
x = range(len(s))
jn = [', '.join(a) for a in nn]
x_labels = jn
y = s
ax.barh(x, y, color=(0.7, 0.3, 0.3, 0.6), align='center') # , color=color
plt.yticks(x, x_labels)
plt.tick_params(axis='both', labelsize=18)
plt.xlabel('Support level', fontsize=15)
plt.legend()
plt.savefig('static/barchart.png', bbox_inches='tight')
mpld3.save_html(figbar, 'static/barchart.html')
# ----------------------------------------------------------------------------------------------------------
# You can look at the trajectory of number of frequent combinations at different selected support level (normalized frequency):
cat_vars = [x * 0.01 for x in range(0, 20)]
allrules = {}
allsets = {}
lenrules = []
lensets = []
itemsets = {}
for var in cat_vars[1:]:
allsets[var] = 0
ti = efficient_apriori.itemsets_from_transactions(records,
min_support=var)
itemsets[var] = ti
for ii in range(len(ti[0])):
allsets[var] = allsets[var] + len(ti[0][ii + 1])
lensets.append(allsets[var])
# figsupp, axsupp = plt.subplots(1, 1, figsize=(10, 8))
# ax = axsupp
# ax.plot(cat_vars[1:], lensets, '-o', color='#e6194b')
# plt.legend(fontsize=18, loc='upper right')
# plt.xlabel('Support Level', fontsize=18)
# plt.ylabel('Number of Frequent Baskets', fontsize=18)
# plt.tick_params(axis='both', labelsize=18)
# plt.xlim(0, 0.2)
# mpld3.save_html(figsupp, 'support.html')
# ----------------------------------------------------------------------------------------------------------
# You can look at the trajectory of number of associations found between combinations at your selected support level:
# if you are not sure about entering min_support, min_confidence you can set up support and confidence as a
# function of var (in the following for loop) such that a certain level of your desired metric
# (conviction, lift, ..) is maintained
con_vars = [x * 0.10 for x in range(1, 11)]
con_vars = np.round(con_vars, 2)
allrules = {}
lenrules = []
for var in con_vars:
_, rules = efficient_apriori.apriori(records,
min_support=0.01,
min_confidence=var)
allrules[var] = rules
lenrules.append(len(allrules[var]))
# figcon, axcon = plt.subplots(1, 1) # , figsize=(8, 6)
# ax = axcon
# ax.plot(con_vars, lenrules, '-o', label='numer of association rules ', color=(0.2, 0.7, 0.9, 0.9))
# plt.legend(fontsize=35, loc='upper right')
# plt.xlabel('Min required confidence level based on support of 1%', fontsize=35)
# plt.ylabel('Number of association rules', fontsize=35)
# plt.tick_params(axis='both', labelsize=35)
# mpld3.save_html(figcon, 'static/confidence.html')
# -----------------------------------------------------------
# gall: is the network of items that are associated with low confidence level, in order to see most available associations
itemsets, r = efficient_apriori.apriori(records,
min_support=0.01,
min_confidence=0.01)
Gall = nx.Graph()
c = 1
for i in r:
node_list = []
for j in i.lhs:
node_list.append(j)
for j in i.rhs:
node_list.append(j)
for j in range(len(node_list) - 1):
for k in range(j + 1, len(node_list)):
Gall.add_edge(node_list[j], node_list[k])
Gallnodes = list(Gall.nodes)
gall = Network()
d = nx.degree(Gall)
dd = dict(d)
# ------------------------------------------------------------------------------------------------------------------
c = 1
# node_list = []
for i in r:
node_list = []
for j in i.lhs:
node_list.append(j)
for j in i.rhs:
node_list.append(j)
for j in range(len(node_list) - 1):
for k in range(j + 1, len(node_list)):
gall.add_node(node_list[j],
value=dd[node_list[j]],
color='#e6194b'
) # (0.8,0.2,0.8,0.9) 46f0f0'#ffe119' '#f032e6'
gall.add_node(node_list[k],
value=dd[node_list[k]],
color='#e6194b')
gall.add_edge(node_list[j], node_list[k], color='#fabebe')
# ------------------------------------------------------------------------------------------------------------------
# choose appropiriate min_support, min_confidence based on data: elbow in figures support.html and confidence.html
# min_support=0.01, min_confidence=0.3
g = nx.Graph()
c = 1
algeallrules = list(allrules.keys())[2]
for i in allrules[algeallrules]:
node_list = []
for j in i.lhs:
node_list.append(j)
for j in i.rhs:
node_list.append(j)
for j in range(len(node_list) - 1):
for k in range(j + 1, len(node_list)):
g.add_edge(node_list[j], node_list[k])
d = nx.degree(g)
dd = dict(d)
# -----------------------------------------------------------------------------------------------------------------
# if you want to get the red graph in slides:
# gn = Network()
# c = 1
# algeallrules = list(allrules.keys())[2]
# for i in allrules[algeallrules]:
# node_list = []
# for j in i.lhs:
# node_list.append(j)
# for j in i.rhs:
# node_list.append(j)
# for j in range(len(node_list) - 1):
# for k in range(j + 1, len(node_list)):
# # gg.add_edge(node_list[j], node_list[k])
# gn.add_node(node_list[j], value=dd[node_list[j]], color='#e6194b')
# gn.add_node(node_list[k], value=dd[node_list[k]], color='#e6194b')
# gn.add_edge(node_list[j], node_list[k])
#
# node_list_g = list(set(node_list))
# gn.save_graph('static/ggrule3percentconf.html')
# -----------------------------------------------------------------------------------------------------------------
# community.girvan_newman uses edgebetween ness to detect communities in graph, and finds hierachical clusters
# final communities are disjoint clusters, meaning they do not have any common items
com = community.girvan_newman(g)
communities = list(com)
node_com = {}
nb_com = len(communities)
cl = np.ceil(nb_com / 4)
clusters = communities[int(cl)]
for i in range(len(list(clusters))):
for j in list(clusters)[i]:
node_com[j] = i
col = [
'#e6194b', 'orangered', '#46f0f0', '#bcf60c', '#808080', '#911eb4',
'#000000', '#46f0f0', '#f032e6', '#bcf60c', '#3cb44b', '#ffe119',
'#4363d8', '#fabebe', '#008080', '#e6beff', '#9a6324', '#fffac8',
'#800000', '#aaffc3', '#808000', '#ffd8b1', '#000075', '#e6194b',
'#bcf60c', '#808080', '#911eb4', '#46f0f0', '#f032e6', '#bcf60c',
'#3cb44b', '#ffe119', '#4363d8', '#fabebe', '#008080', '#e6beff',
'#9a6324', '#fffac8', '#800000', '#aaffc3', '#808000', '#ffd8b1',
'#000075'
]
node_colors = []
node_colorsg = {}
for node in g.nodes():
for j in range(len(clusters)):
if node_com[node] == j:
node_colors.append(col[j])
node_colorsg[node] = col[j]
# -----------------------------------------------------------------------------------------------------------------
c = 0
clusters_out = {}
for i in range(len(clusters)):
if len(clusters[i]) > 1:
clusters_out[c] = clusters[i]
c += 1
clusters_out_df = pd.DataFrame.from_dict(clusters_out, orient='index')
clusters_out_df = clusters_out_df.T
clusters_out_df = clusters_out_df.fillna(' ')
colnames = []
for i in clusters_out_df.columns:
col = 'Recommended Basket ' + str(i + 1)
colnames.append(col)
clusters_out_df.columns = colnames
dg = nx.degree(g)
ddg = dict(d)
# -----------------------------------------------------------------------------------------------------------------
gg = Network()
c = 1
algeallrules = list(allrules.keys())[2]
for i in allrules[algeallrules]:
node_list = []
for j in i.lhs:
node_list.append(j)
for j in i.rhs:
node_list.append(j)
for j in range(len(node_list) - 1):
for k in range(j + 1, len(node_list)):
# gg.add_edge(node_list[j], node_list[k])
gg.add_node(node_list[j],
value=ddg[node_list[j]],
color=node_colorsg[node_list[j]])
gg.add_node(node_list[k],
value=ddg[node_list[k]],
color=node_colorsg[node_list[k]])
gg.add_edge(node_list[j], node_list[k])
node_list_g = list(set(node_list))
gg.save_graph('static/ggrule3percentconf_c.html')
# -----------------------------------------------------------------------------------------------------------------
# if you need nto export graph in json file for better visualization, here you go:
# nodes = [{'name': str(i), 'club': gg.node[i]}
# for i in g.nodes()]
# links = [{'source': u[0], 'target': u[1]}
# for u in gg.edges()]
# with open('graph.json', 'w') as f:
# json.dump({'nodes': nodes, 'links': links},
# f, indent=4,)
# ------------------------------------------------------------------------------------------------------------------
cc = {}
for i in range(3, int(len(g.nodes) / 2)):
cc[i] = []
nb_clus = len(list(k_clique_communities(g, i)))
for c in range(nb_clus):
tt = list(list(k_clique_communities(g, i))[c])
cc[i].append(tt)
cc[i] = sorted(cc[i])
soft_clusters = pd.DataFrame.from_dict(cc, orient='index')
soft_clusters.reset_index(inplace=True, drop=True)
# ------------------------------------------------------------------------------------------------------------------
# notice in graph that mineral water, spagetti and .. are densly connected woth other clusters:
# we may want to construct clusters that allow having shred items
# k_clique_communities uses cliques of different size to detect communities in graph,
# final communities are overlapping clusters, meaning they are allowed to have common items
cc = {}
final_package = {}
count = 0
for i in range(3, int(len(g.nodes) / 3)):
cc[i] = []
nb_clus = len(list(k_clique_communities(g, i)))
if count < nb_clus:
count = i
for c in range(nb_clus):
tt = list(list(k_clique_communities(g, i))[c])
cc[i].append(tt)
cc[i] = sorted(cc[i])
# final_package = cc[count]
soft_clusters = soft_clusters.dropna(axis=0)
# -----------------------------------------------------------------------------------------------------------------
overlapping_clusters = pd.DataFrame(columns=[0])
for i in range(len(soft_clusters)):
l = []
temp = soft_clusters.iloc[i]
for j in temp:
l.append(j)
df1 = pd.DataFrame(l).T
overlapping_clusters = pd.concat([overlapping_clusters, df1],
ignore_index=True,
axis=1)
overlapping_clusters = overlapping_clusters.drop([0], axis=1)
overlapping_clusters = overlapping_clusters.fillna(' ')
colnames_over = []
for i in overlapping_clusters.columns:
col = 'Recommended Basket' + str(i)
colnames_over.append(col)
overlapping_clusters.columns = colnames_over
overlapping_clusters = overlapping_clusters[
overlapping_clusters.columns[:3]]
overlapping_clusters.to_csv('static/soft_clusters.csv', index=False)
# final_package_df.to_csv('static/final_package_df.csv')
clusters_out_df.to_csv('static/clusters_out_df.csv', index=False)
def network(request, n, f):
fileType = f
inputFile = n
htmlFile = "./NetworkViewer/networks/" + n.split(".")[0] + ".html"
# The network parameters to be displayed are initiated. All parameters can not be calculated for all types of networks
diameter = 0
clusteringCoeff = 0
cliques = 0
degree = 0
connectedComponents = 0
stronglyConnectedomponents = 0
# Different file types needs to be handeled in different ways
ngx = None
if fileType == 'geneSpider':
# Gene spider matrix is read as a matrix that is then transposed, in order to get the directions right
networkDf = pd.read_csv(os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
'media/' + inputFile),
sep=',',
header=None)
networkDfTransposed = networkDf.T
nxg = nx.from_numpy_matrix(np.array(networkDfTransposed),
create_using=nx.MultiDiGraph())
elif fileType == 'adjacencyList':
# Adjacency lists/Edge lists are red as undirected pandas edgelists
networkDf = pd.read_csv(os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
'media/' + inputFile),
sep='\s+',
header=None)
nxg = nx.from_pandas_edgelist(networkDf, source=0, target=1)
elif fileType == 'directedAdjacencyList':
# Directed edge lists need to be read as directed pandas edgelists
networkDf = pd.read_csv(os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
'media/' + inputFile),
sep='\s+',
header=None)
nxg = nx.from_pandas_edgelist(networkDf,
source=0,
target=1,
create_using=nx.MultiDiGraph())
elif fileType == 'adjacencyMatrix':
# Adjacency matrix need to be read as an undirected matrix
networkDf = pd.read_csv(os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
'media/' + inputFile),
sep=',',
header=None)
nxg = nx.from_numpy_matrix(np.array(networkDf))
elif fileType == 'directedAdjacencyMatrix':
# Directed Adjacency matrix need to be read as a directed matrix
networkDf = pd.read_csv(os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
'media/' + inputFile),
sep=',',
header=None)
nxg = nx.from_numpy_matrix(np.array(networkDf),
create_using=nx.MultiDiGraph())
elif fileType == 'funCoup':
# FunCoup-network-files contains several columns defining the evidence types and scores. Here, they are red as edge lists, only taking the genes and their edges into account
networkDf = pd.read_csv(os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
'media/' + inputFile),
sep='\t')
networkDfGenes = networkDf[['2:Gene1', '3:Gene2']]
nxg = nx.from_pandas_edgelist(networkDfGenes,
source='2:Gene1',
target='3:Gene2')
# The networkx-networks are drawn as png-images, since the pyvis can not handle other formats than html.
pos = nx.spring_layout(nxg)
nx.draw(nxg, pos, with_labels=True)
# nx.draw_networkx_edge_labels(nxg, pos, with_labels = True)
plt.savefig("./NetworkViewer/templates/NetworkViewer/networks/" +
inputFile.split(".")[0] + ".png",
format="PNG")
plt.clf()
# Initiating pyvis interactive network with customized graphics
g = Network()
g.barnes_hut(gravity=-2000,
central_gravity=0.02,
spring_length=1,
spring_strength=0.000001,
damping=0.09,
overlap=0)
g.toggle_physics(True)
# Calculating network prooperties from the networkx graphs
if nx.is_directed(nxg):
g.directed = True
stronglyConnectedomponents = nx.number_strongly_connected_components(
nxg)
else:
connectedComponents = nx.number_connected_components(nxg)
if connectedComponents == 1:
diameter = nx.diameter(nxg, e=None)
clusteringCoeffs = nx.clustering(nxg)
clusteringCoeff = np.mean(list(clusteringCoeffs.values()))
allCliques = nx.find_cliques(nxg)
cliques = len(list(allCliques))
degrees = nxg.degree()
degreesOnly = []
for node, degree in degrees:
degreesOnly.append(degree)
degree = np.mean(degreesOnly)
maxDegree = np.max(degreesOnly)
# Filling the pyvis graph with nodes and edges from the networkx graph.
allNodes = list(nxg.nodes)
allSizes = []
# The nodes gets sizes according to their degree
for d in degreesOnly:
allSizes.append(40 * (d / maxDegree))
g.add_nodes(allNodes, size=allSizes)
# The edges gets width according to their weights
allEdges = nxg.edges(data=True)
edges = []
for a, b, w in allEdges:
edges.append((a, b, w.get('weight')))
g.add_edges(edges)
g.height = "100%"
g.width = "100%"
# The pyvis graph is saved as an html file, that is embedded in the network viewer-vindow
g.save_graph("./NetworkViewer/templates/NetworkViewer/networks/" +
inputFile.split(".")[0] + ".html")
# Exporting network properties to the html-view
context = {
"diameter": ("%.2f" % diameter),
"clustering": ("%.2f" % clusteringCoeff),
"cliques": ("%.2f" % cliques),
"degree": ("%.2f" % degree),
"connectedComponents": connectedComponents,
"stronglyConnectedomponents": stronglyConnectedomponents,
"inputFile": inputFile,
"htmlFile": htmlFile
}
return render(request, 'NetworkViewer/viewNetwork.html', context)
def render_graph(self, graph, filename="graph.html"):
net = Network("100%", "100%")
source_files = list()
for node in graph.nodes(data=True):
node_shape = "ellipse"
node_id = node[0]
node_label = self.format_node_label(node[0],
graph.in_degree(node[0]),
graph.out_degree(node[0]))
if "Scenario:" in node[0]:
node_color = self.get_node_color("#2589BD")
node_level = 1 if not self.config.show_src_file else 2
if self.config.show_src_file:
source_files.append((node[1]["source_file"], node[0]))
elif graph.out_degree(node[0]) > 0:
node_color = self.get_node_color("#009B72")
node_level = 2 if not self.config.show_src_file else 3
else:
node_color = self.get_node_color("#EC9A29")
node_level = 3 if not self.config.show_src_file else 4
if node[1]["smell"]:
node_color = self.get_node_color("red")
node_title = "\n".join(node[1]["smell_names"])
net.add_node(n_id=node_id,
label=node_label,
level=node_level,
color=node_color,
shape=node_shape,
title=node_title)
else:
net.add_node(n_id=node_id,
label=node_label,
level=node_level,
color=node_color,
shape=node_shape)
if self.config.show_src_file:
for pair in source_files:
src = pair[0]
dst = pair[1]
net.add_node(src,
src,
shape=node_shape,
level=1,
color=self.get_node_color("#9FA08E"))
net.add_edge(src, dst)
edges = list(graph.edges())
for src, dst in edges:
edge_dashes = False
if src.startswith("Step:") and dst.startswith("Step:"):
edge_dashes = True
edge_width = edges.count((src, dst))
if self.config.edge_labels:
graph_data = graph.get_edge_data(src, dst)
labels = list()
for key in graph_data:
labels.append(str(graph_data[key]["label"]))
edge_label = ",".join(labels)
net.add_edge(src,
dst,
dashes=edge_dashes,
width=edge_width,
label=edge_label)
else:
net.add_edge(src, dst, dashes=edge_dashes, width=edge_width)
net.set_options("""
var options = {
"edges": {
"font":{
"size": """ + str(self.config.font_size) + """
},
"arrows": {
"to": {
"enabled": true
}
},
"color": {
"inherit": true,
"highlight": "rgba(0,0,0,1)"
},
"smooth": false
},
"nodes": {
"borderWidth": 0.5,
"borderWidthSelected":1,
"font": {
"color": "rgba(255,255,255,1)",
"size": """ + str(self.config.font_size) + """
},
"fixed": {
"x": false,
"y": false
}
},
"layout": {
"improvedLayout": true,
"hierarchical": {
"enabled": true,
"levelSeparation": """ +
str(self.config.level_separation) + """,
"nodeSpacing": """ + str(self.config.node_spacing) + """,
"treeSpacing": 5,
"direction": "LR",
"blockShifting": true,
"edgeMinimization": false,
"parentCentralization": true,
"sortMethod": "directed"
}
},
"physics": {
"hierarchicalRepulsion": {
"centralGravity": 0,
"springLength": 0,
"springConstant": 0,
"damping": 1,
"nodeDistance": """ + str(self.config.node_spacing) + """
},
"minVelocity": 0.00,
"maxVelocity": 0.00,
"solver": "hierarchicalRepulsion"
}
}
""")
net.save_graph(f"{self.config.OUTPUT_DIR}/{filename}")
return f"{self.config.OUTPUT_DIR}/{filename}"
def fnc_build_graphnx(df_system, dfConnFinal, global_dict, router_mode, filename):
from pyvis.network import Network
phyList = ['High-Ran','High-Ran-ABR','FronteraPE','Mid-Ran','AggEthernet','FronteraSDH','BBIP']
#plt.figure(figsize=(15,15))
#G = nx.Graph()
G = Network("800px", "1300px", heading="")
#G.set_edge_smooth('dynamic')
G.set_options("""
var options = {
"nodes": {
"size":13,
"font": {
"size": 11
}
},
"edges": {
"arrowStrikethrough": false,
"color": {
"inherit": false
},
"font": {
"size": 10,
"align": "top"
},
"smooth": false
},
"manipulation": {
"enabled": true,
"initiallyActive": true
},
"physics": {
"barnesHut": {
"centralGravity": 0.2,
"springLength": 100,
"springConstant": 0.01,
"damping": 0.7,
"avoidOverlap": 1
},
"maxVelocity": 5,
"minVelocity": 0.47,
"solver": "barnesHut"
}
}
""")
for i in df_system.itertuples():
router_name = i[1]
id = i[2]
ip_system = i[3]
if global_dict[router_name]['HWfunction'] in phyList:
physics = False
else:
physics = True
G.add_node(router_name,
label = str(router_name) + "\n" + str(ip_system),
chassis = global_dict[router_name]['HWtype'],
ip = str(ip_system),
color = get_attribute(router_name,"color",global_dict)['yEd'],
physics = physics,
)
for i in dfConnFinal.itertuples():
router_name_A = i[1]
port_name_A = i[2]
ip_A = i[12]
ip_system_A = global_dict[router_name_A]['system']
router_name_B = i[5]
port_name_B = i[4]
ip_B = i[13]
ip_system_B = global_dict[router_name_B]['system']
cable_id = i[3]
if global_dict[router_name_A]['HWfunction'] in phyList and global_dict[router_name_B]['HWfunction'] in phyList:
physics = False
else:
physics = True
edgeLabel = port_name_A + "--" + port_name_B
if cable_id in ["N/A",None]:
G.add_edge(source=router_name_A, to=router_name_B, label=edgeLabel, physics=physics)
else:
G.add_edge(source=router_name_A, to=router_name_B, label=cable_id + "\n" + edgeLabel, physics=physics)
#G.barnes_hut(overlap=1)
#G.force_atlas_2based(overlap=1)
#G.show_buttons()
G.save_graph("plugins/topo.html")
