def open_network(network_path): """Function that imports a network from an html file. Parameters: - network_path: path of the network to be imported Returns: - G: network object """ G = Network() html = open(network_path, "r") lines_html = html.readlines() nodes = "" edges = "" for line in lines_html: if "nodes = new" in line: nodes = json.loads(line.split('(')[1].split(')')[0]) if "edges = new" in line: edges = json.loads(line.split('(')[1].split(')')[0]) dictionary_to_nodes(nodes, G) dictionary_to_edges(edges, G) G.save_graph("./NetworkGraphs/Temp_Network/temp_network.html") html_fix(os.path.abspath("./NetworkGraphs/Temp_Network/temp_network.html")) return G
def on_save(self): """Method called when the user presses the save button in the editor configuration. It applies the changes in the temporary network and updates the network graph of the canvas. Parameters: - self: current instance of the class; """ G = Network() print(self.routers) print(self.switches) print(self.hosts) print(self.hosts) network_core.dictionary_to_nodes(self.routers, G) network_core.dictionary_to_nodes(self.switches, G) network_core.dictionary_to_nodes(self.hosts, G) network_core.dictionary_to_nodes(self.others, G) network_core.dictionary_to_edges(self.temporary_network.edges, G) self.main_window_object.current_network = G G.save_graph("./NetworkGraphs/Temp_Network/temp_network.html") network_core.html_fix(os.path.abspath("./NetworkGraphs/Temp_Network/temp_network.html")) self.main_window_object.update_canvas_html(os.path.abspath("./NetworkGraphs/Temp_Network/temp_network.html")) self.main_window_object.ssh_window = ssh_connection(self.main_window_object) self.main_window_object.edge_window = edge_editors(self.main_window_object) self.close()
def generate_graph(arbre): got_net = Network(height="100%", width="100%", bgcolor="#222222", font_color="white") dico_couleur = {} for streamer in arbre.streamers: got_net.add_node(streamer.name, value=streamer.poids, shape='image', image=streamer.photo_profil, size=100) dico_couleur[streamer.name] = streamer.couleur for depart, arrivee in arbre.groupes_migrants: for viewer in arbre.groupes_migrants[depart, arrivee]: got_net.add_node(viewer, value=1, color=dico_couleur[depart]) got_net.add_edge(viewer, depart, color=dico_couleur[depart]) got_net.add_edge(viewer, arrivee, color=dico_couleur[arrivee]) got_net.show_buttons() got_net.set_edge_smooth(smooth_type='dynamic') got_net.force_atlas_2based(gravity=-200, central_gravity=0.01, spring_length=1, spring_strength=0.08, damping=0.4, overlap=0) got_net.save_graph("./graphs_v1/" + 'streamgame_v1_42' + ".html") return
def getCourseText(): """ Visualize courses and their assigned texts. """ coursesAndTexts = g.query(""" select distinct ?courseName ?textTitle ?authorLast where { ?a a ccso:Course . ?a ccso:csName ?courseName . ?a ccso:hasLM ?t . ?t res:resource ?doc . ?doc dcterms:title ?textTitle . ?doc dcterms:creator ?author . ?author foaf:surname ?authorLast . } limit 50""") for line in coursesAndTexts: print(line) net = Network(height='750px', width='100%') for courseName, textTitle, authorLast in coursesAndTexts: net.add_node(courseName, shape='square', title=str(courseName)) net.add_node(textTitle, shape='circle', label=str(authorLast), title=str(textTitle)) net.add_edge(courseName, textTitle, title="hasLM") net.save_graph('../website/graph-viz.html')
def graph_print(): dist = loadCompleteDistro() matrix = numpy.zeros((len(dist), len(dist))) i = 0 labels = {} net = Network() k=0 caster = {"nld": "Niderlandzki", "gle": "Irlandzki", "spa": "Hiszpański", "fra": "Francuski", "eng": "Angielski", "slk": "Słowacki", "fin": "Fiński", "est": "Estoński", "lit": "Litewski", "hrv": "Chorwacki", "por": "Portugalski", "swe": "Szwedzki", "ron": "Romański", "dan": "Duński", "slv": "Słoweński", "bul": "Bułgarski", "ita": "Włoski", "ell": "Grecki", "mlt": "Maltański", "lav": "Łotewski", "ces": "Czeski", "pol": "Polski", "deu": "Niemiecki", "hun": "Węgierski"} for lang1 in dist.keys(): print(lang1) net.add_node(k + 1, label=f"{caster[lang1]}") k+=1 xD = [] for lang1 in dist.keys(): labels[i] = lang1 j = 0 for lang2 in dist.keys(): if lang1 != lang2: xD.append([i+1, j+1,round(KullbackLeibnerIteration(dist[lang1].alain(dist[lang2]), dist[lang2].alain(dist[lang1])),2)]) else: pass j += 1 i += 1 #G = nx.from_numpy_matrix(matrix) #H = nx.relabel_nodes(G, labels) #vg = [] #for u in H.edges.data(): # vg.append(u[2]["weight"]) # xD.sort(key=lambda x: x[2]) xDD = xD[:40] for x in xDD: print(x[2]) net.add_edge(x[0],x[1],value=1.0/x[2], label=str(x[2])) # #Z = copy.deepcopy(H) #for u in Z.edges.data(): # if u[2]["weight"] > median: # H.remove_edge(u[0], u[1]) # #pos = nx.spring_layout(H,k=1, iterations=20) #plt.figure(3, figsize=(10, 10)) #labels = nx.get_edge_attributes(H, 'weight') #nx.draw_networkx(H,pos=pos, with_labels=True,node_size=500) #nx.draw_networkx_edge_labels(H, pos, edge_labels=labels) #plt.show() net.save_graph("my_graph.html") net.toggle_physics(True) net.show_buttons(filter_=['nodes', 'edges', 'physics']) net.show('mygraph.html')
def plot_bacteria_intraction_network(bacteria, node_list, node_size_list, edge_list, color_list, G_name, folder, color_by_tax_level=2, directed_G=True, control_color_and_shape=True): # create the results folder if not os.path.exists(folder): os.mkdir(folder) nodes_colors, nodes_shapes, group_list, tax_to_color_and_shape_map = \ get_nodes_colors_by_bacteria_tax_level(bacteria, G_name, taxnomy_level=color_by_tax_level, folder=folder) bact_short_names = [shorten_single_bact_name(b) for b in bacteria] nodes_with_edges = np.unique(np.array(edge_list).flatten()).tolist() net = Network(height="750px", width="100%", bgcolor="#FFFFFF", font_color="black", directed=directed_G) #net.barnes_hut(gravity=-120000) net.force_atlas_2based() # for the nodes: you can use either only the group option the automatically colors the group in different colors # shaped like dots - no control, or use color and shape to make it you own, in this case group is irrelevant for i, node in enumerate(node_list): if node in nodes_with_edges: if control_color_and_shape: net.add_node(node, label=bact_short_names[i], color=nodes_colors[i], value=node_size_list[i], shape=nodes_shapes[i], group=group_list[i]) else: net.add_node(node, label=bact_short_names[i], value=node_size_list[i], group=group_list[i]) # for the edges, the colors are what you decide and send for i, (u, v) in enumerate(edge_list): net.add_edge(u, v, color=color_list[i]) net.save_graph(os.path.join(folder, G_name + ".html"))
def generate_graph(arbre): got_net = Network(height="100%", width="100%", bgcolor="#222222", font_color="white") got_net.add_node(arbre.name, value=(1000 * len(arbre.children)**5), color='#FFFF00') recursive(arbre, got_net, arbre.name) got_net.show_buttons() got_net.save_graph("../data/graphs/" + arbre.name + ".html")
def visualizeGraph(nxGraph, save=False, size=['500px', '500px']): """ Converts from networkx to a pyvis Network. """ subGViz = Network(size[0], size[1]) from_nx(subGViz, nxGraph) if save: subGViz.save_graph(save) else: subGViz.write_html('graph.html') #subGViz.show('subGViz.html') return subGViz
def visualize(edges, error_codes, resouce_pages, args): G = nx.DiGraph() G.add_edges_from(edges) if args.save_txt is not None or args.save_npz is not None: nodes = list(G.nodes()) adj_matrix = nx.to_numpy_matrix(G, nodelist=nodes, dtype=int) if args.save_npz is not None: base_fname = args.save_npz.replace('.npz', '') scipy.sparse.save_npz(args.save_npz, scipy.sparse.coo_matrix(adj_matrix)) else: base_fname = args.save_txt.replace('.txt', '') np.savetxt(args.save_txt, adj_matrix, fmt='%d') node_info = get_node_info(nodes, error_codes, resource_pages, args) with open(base_fname + '_nodes.txt', 'w') as f: f.write('\n'.join( [nodes[i] + '\t' + node_info[i] for i in range(len(nodes))])) net = Network(width=args.width, height=args.height, directed=True) net.from_nx(G) if args.show_buttons: net.show_buttons() elif args.options is not None: try: with open(args.options, 'r') as f: net.set_options(f.read()) except FileNotFoundError as e: print('Error: options file', args.options, 'not found.') except Exception as e: print('Error applying options:', e) for node in net.nodes: node['size'] = 15 node['label'] = '' if node['id'].startswith(args.site_url): node['color'] = INTERNAL_COLOR if node['id'] in resouce_pages: node['color'] = RESOURCE_COLOR else: node['color'] = EXTERNAL_COLOR if node['id'] in error_codes: node[ 'title'] = f'{error_codes[node["id"]]} Error: <a href="{node["id"]}">{node["id"]}</a>' node['color'] = ERROR_COLOR else: node['title'] = f'<a href="{node["id"]}">{node["id"]}</a>' net.save_graph(args.vis_file)
def Data_Visualisation (Algor1, Algor2): net = Network() inputFile = open(r"inputFile.txt","r+") firstline = inputFile.readline() [n,m] = firstline.split() n = int(n) m = int(m) sizes = Algor1 for node in range(n): net.add_node(node, label=node.__str__(),size=float(sizes[node].split()[0])*10) input_lines = inputFile.readlines() for line in input_lines : edgelist = line.split() net.add_edge(int(edgelist[0]), int(edgelist[1]), title = int(edgelist[2].__str__())) net.save_graph('Degree Centrality.html') net = Network() inputFile = open(r"inputFile.txt", "r+") firstline = inputFile.readline() [n, m] = firstline.split() n = int(n) m = int(m) sizes = Algor2 for node in range(n): net.add_node(node, label=node.__str__(), size=float(sizes[node].split()[0]) * 40) input_lines = inputFile.readlines() for line in input_lines: edgelist = line.split() net.add_edge(int(edgelist[0]), int(edgelist[1]), title=int(edgelist[2].__str__())) net.save_graph('Closeness Centrality.html')
def pyvis_draw(self, graph, name): cnames = ['blue', 'green', 'red', 'cyan', 'orange', 'black', 'purple', 'purple', 'purple'] G = Network(height="800px", width="100%", directed=True) for src, edge, dst in graph: if edge == self.om.type and not self.config.show_url: continue src_type = [x for x in graph[src:self.om.type]] dst_type = [x for x in graph[dst:self.om.type]] if len(src_type): if str(Flavor.FUNCTION) == str(src_type[0]) or str(Flavor.METHOD) == str(src_type[0]): G.add_node(src, title=src, physics=True, color=cnames[0]) elif str(Flavor.CLASS) == str(src_type[0]) or str(Flavor.MODULE) == str(src_type[0]): G.add_node(src, title=src, physics=True, color=cnames[1]) else: G.add_node(src, title=src, physics=True, color=cnames[2]) else: G.add_node(src, title=src, physics=True, color=cnames[3]) if len(dst_type): if str(Flavor.FUNCTION) == str(dst_type[0]) or str(Flavor.METHOD) == str(dst_type[0]): G.add_node(dst, title=dst, physics=True, color=cnames[0]) elif str(Flavor.CLASS) == str(dst_type[0]) or str(Flavor.MODULE) == str(dst_type[0]): G.add_node(dst, title=dst, physics=True, color=cnames[1]) else: G.add_node(dst, title=dst, physics=True, color=cnames[2]) else: G.add_node(dst, title=dst, physics=True, color=cnames[3]) G.add_edge(src, dst, width=0.5, title=str(edge), physics=True) G.hrepulsion(node_distance=120, central_gravity=0.0, spring_length=100, spring_strength=0.01, damping=0.09) # G.show_buttons(filter_=['physics']) G.save_graph("%s.html" % name)
def sequence_sample(self, sess, chars, vocab): with open('data/review_example_sequence.txt') as f: data = f.read() state = sess.run(self.cell.zero_state(1, tf.float32)) x = np.zeros((1, 1)) for activity in data.split("->"): x[0, 0] = vocab[activity] feed = {self.input_data: x, self.initial_state: state} [probs, state] = sess.run([self.probs, self.final_state], feed) ps = probs[0] if activity == "time-out 2": G = Network(height=1000, width=1000, directed=True) G.add_node(activity, level=0) index = 0 for p in ps: if p > 0.01: if chars[index] == "\n": index += 1 continue print(activity + "\t\t" + chars[index] + "\t{%f}" % p) G.add_node(chars[index], physics=True, level=1) G.add_edge(activity, chars[index], value=int(p * 100), title=int(p * 100), physics=True, arrowStrikethrough=False) index += 1 print() G.barnes_hut(gravity=-10000) # G.show_buttons() G.save_graph( os.path.join( "results", re.sub('[-=.#/?:$}]', '', activity) + "_sequence.html"))
def _graph(self): try: counter = {} net = Network(height="100%", width="100%", bgcolor="white", font_color="white") net.force_atlas_2based(gravity=-250, central_gravity=0.005, spring_length=0, spring_strength=0, damping=1, overlap=1) exists = [] for entry in self.edge_data: src = entry['challenger_id'] pokemon = entry['challenger'] if src in exists: continue t = "00" if 'alolan' in pokemon.lower(): t = "61" my_img = '/static/icons/pokemon_icon_%s_%s.png' % (f'{src:03}', t) net.add_node(src, ' ', image=my_img, shape='circularImage', size=50, color={'highlight': {'border': entry['color']}}, borderWidthSelected=5) exists.append(src) fix_list = [346,26,82] for src in fix_list: my_img = '/static/icons/pokemon_icon_%s_%s.png' % (f'{src:03}', t) net.add_node(src, ' ', image=my_img, shape='circularImage', size=50, color={'highlight': {'border': entry['color']}}, borderWidthSelected=5) for entry in self.edge_data: src = entry['challenger_id'] pokemon = entry['challenger'] opponent = entry['opponent'] if opponent.lower() not in self.include: continue if pokemon.lower() not in self.include: continue dst = entry['opponent_id'] w = entry['weight']/10 net.add_edge(src, dst, value=w, arrows='to', arrowStrikethrough=False, smooth=True, color={"color": entry['color'], "highlight": entry['color'], "opacity": 0.05}, selfReferenceSize=50) net.set_edge_smooth('continuous') #net.show_buttons(filter_=['physics']) os.makedirs("/data/%s" % self.config['settings']['cup'], exist_ok=True) net.save_graph("/data/%s/%svs%s.html" % (self.config['settings']['cup'], self.config['settings']['my_shield'], self.config['settings']['op_shield'])) return True except Exception as e: traceback.print_exc() logging.info(e)
def render_instance(self, render_graph_html, iteration, circle_bool, hide_edges_bool, hide_weight, h=500, w=750): """ Render a networkx graph in screen """ # print("Changed to: ", iteration) self.iteration = iteration # If some propriety change, update value and reload the Networx Graph if self.circle_bool != circle_bool or self.hide_edges_bool != hide_edges_bool or self.hide_weight != hide_weight: self.circle_bool = circle_bool self.hide_edges_bool = hide_edges_bool self.hide_weight = hide_weight self.load_nx_graph() self.load_solution(iteration) # Translate to pyvis network nt = Network(f'{h}px', f'{w}px', font_color='white') nt.from_nx(self.nx_graph) nt.toggle_physics(False) path = f'network.html' nt.save_graph(path) HtmlFile = open(path, 'r') # , encoding='utf-8') source_code = HtmlFile.read() # Clean screen before plot if render_graph_html != None: render_graph_html.empty() render_html = components.html(source_code, height=h * 1.1, width=w * 1.1) return render_html
def getCourseText2(): """ Visualize courses and their assigned texts. """ coursesAndTexts = g.query(""" select distinct ?courseName ?textTitle ?authorLast where { ?a a ccso:Course . ?a ccso:csName ?courseName . ?a ccso:hasLM ?t . ?t res:resource ?doc . ?doc dcterms:title ?textTitle . ?doc dcterms:creator ?author . ?author foaf:surname ?authorLast . } limit 400""") for line in coursesAndTexts: print(line) net = Network(height='750px', width='100%') for courseName, textTitle, authorLast in coursesAndTexts: net.add_node(courseName, shape='square', title=str(courseName)) net.add_node(textTitle, shape='circle', label=str(authorLast), title=str(textTitle)) net.add_edge(courseName, textTitle, title="hasLM") out = "" nodesJson = json.dumps(net.nodes) edgesJson = json.dumps(net.edges) template = f""" nodes = new vis.DataSet({nodesJson}) edges = new vis.DataSet({edgesJson}) data = {{nodes: nodes, edges: edges}}; """ print(template) net.save_graph('../website/graph-vis.html')
def make_graph(self): graph_data = [] for key in self.ner_dct.keys(): lst = self.ner_dct[key] for x in lst: graph_data.append((key, x[0])) poi_net = Network(height="750px", width="100%", bgcolor="#ffffff", font_color="blue") poi_net.barnes_hut() for i in graph_data: src = i[0] dst = i[1] poi_net.add_node(src, label=src, title="src", color="black") poi_net.add_node(dst, label=dst, title="dst", color="#dd4b39") poi_net.add_edge(src, dst) #change this location poi_net.save_graph( "/Users/ankitanand/Box/UB/Fall 2019/IR/Proj1/poi.html")
def interactive_plot(self, df): plt.figure(figsize=(20, 10)) G = nx.from_pandas_edgelist(df, source='source', target='target', create_using=nx.DiGraph()) node_limit = len(G.nodes) pos = graphviz_layout(G, prog='dot') nx.draw(G, pos, with_labels=True, node_size=5000, font_size=20, font_color='black', node_color=range(node_limit), cmap=plt.cm.spring) plt.title('DOM Tree Visualization') nt = Network("500px", "500px") nt.from_nx(G) nt.toggle_physics(True) nt.save_graph('dom_tree_inter_viz.html')
def Create_Graph(nodes, connections, output_type='saveshow'): graph = networkx.Graph() for node in nodes: graph.add_node(node['label'], title=node['title']) node = graph.nodes[node['label']] print('Woo') print(node) #for arg in params: #graph[label][arg]=params[arg] for edge in edges: graph.add_edge(*edge) for arg in edges[edge]: graph[edge[0]][edge[1]][arg] = edges[edge][arg] network = Network(bgcolor='#222222', font_color='white') network.from_nx(graph) network.save_graph('graph1.html') network.show('graph1.html')
def create_graph(graph_structure: Dict[str, Set[Tuple[str, str]]], graph_structure_attributes: Dict[str, Set[str]], filepath: str): """ Creates and save graph from given processed relationships :param graph_structure: processed relationships :param graph_structure_attributes: attributes for every node :param filepath: path in which we like to save file :return: None """ graph = Network(directed=True, height='800px', width='800px') for el1, connected_elements in graph_structure.items(): if el1 not in graph.nodes: graph.add_node(el1) for connected_element in connected_elements: el2 = connected_element[1] predicate = connected_element[0] if el2 not in graph.nodes: graph.add_node(el2) graph.add_edge(el1, el2, title=predicate, label=predicate) for subject, attributes in graph_structure_attributes.items(): if subject not in graph.nodes: graph.add_node(subject) for attribute in attributes: if attribute not in graph.nodes: graph.add_node(attribute, color='red') graph.add_edge(subject, attribute, color='red') graph.save_graph(filepath)
def pyvis_guanxi(): from pyvis.network import Network net = Network() # 添加单个节点 net.add_node(1, label='node_1') net.add_node(2, label='node_2') # 添加属性,注意这⾥里里是添加多个节点,使⽤用add_nodes # 属性值传⼊入的是列列表,要保证列列表⻓长度与节点列列表⻓长度⼀一致 net.add_nodes([3, 4, 5], value=[10, 20, 30], title=["节点3", "节点4", "节点5"], x=[21.4, 54.2, 11.2], y=[100.2, 23.54, 32.1], label=["node_3", "node_4", "node_5"], color=["#00ff1e", "#162347", "#dd4b39"]) # ⼀一次添加⼀一条边 net.add_edge(1, 2) net.add_edge(1, 3) net.add_edge(1, 4) net.add_edge(2, 5) # 有权重边 net.add_edge(2, 1, wright=0.87) net.add_edge(2, 3, wright=0.5) ''' #结合networkx使⽤用 import networkx as nx # ⽣生成5个节点,每个点与其他所有点相互联系 nxg = nx.complete_graph(5) G = Network() G.from_nx(nxg) # 输出并⽣生成demo2.html G.show('demo2.html') ''' # 交互式显示,# 可对打开的html进⾏行行调整 net.show_buttons(filter_=['physics']) net.save_graph( 'app/templates/render/pyvis_guanxi.html') #输出并⽣生成pyvis_guanxi.html return render_template('render/pyvis_guanxi.html')
def generate_trustline_graph(data): ''' This function generates a network graph using pyvis module. The graph is saved in Analysis folder. Parameters ---------- data : Filtered transactions data. Returns ------- None. ''' graphNodeList=[] # creating empty nodes list for graph for acc in data.index: # adding data to graph nodes list graphNodeList.append(data['account'][acc]) # appending account graphNodeList.append(data['issuer'][acc]) # appending issuer graphNodeList=list(set(graphNodeList)) # filtering graph node list for unique node list graphEdgeList=[] # create graph's edge list for acc in data.index: # adding data to graph's edges if(data['value'][acc]!=0): # don't add edge if trustline is set to 0 graphEdgeList.append((data['account'][acc],data['issuer'][acc],data['currency'][acc],data['value'][acc])) # adding edge with account, issuer, currency and value nx_graph = Network(height="750px", width="100%", bgcolor="#222222", font_color="white",directed="true") # setting parameters for visualising the graph. for node in graphNodeList: # adding nodes to graph nx_graph.add_node(node, size=20, title=node) for edge in graphEdgeList: # adding edges to graph weigh= str(edge[3])+' '+str(edge[2]) nx_graph.add_edge(edge[0], edge[1], weight=weigh, title=weigh) # adding edge with weights to graph neighbor_map = nx_graph.get_adj_list() for node in nx_graph.nodes: node["title"] += " Trusted:<br>" + "<br>".join(neighbor_map[node["id"]]) # creating title for each node node["value"] = len(neighbor_map[node["id"]]) # setting value of node nx_graph.save_graph(r"../Analysis/TrustSetGraph.html") # saving the graph in HTML format nx_graph.show_buttons(filter_=['physics']) nx_graph.show("TrustSetGraph.html") # open the graph in browser
for i in range(0, x): #itterate over 0-35 for every HLA in HLAs if dst in IDs[i]: #if target in one of the HLA lists.. name = IDs[i].split()[0] #get the HLA-type of this target prob_net.add_node( dst, dst, title=dst, color=Ncolors[name] ) #add target node to prob_net with color assigned to the hla-type in Ncolors prob_net.add_edge(src, dst) #add edge between source and target neighbor_map = prob_net.get_adj_list() # add neighbor data to node hover data for node in prob_net.nodes: node["title"] += " Neighbors:<br>" + "<br>".join(neighbor_map[node["id"]]) node["value"] = len(neighbor_map[node["id"]]) #save graph as TCR.html prob_net.save_graph("D:/gliph-1.0/gliph/bin/TCR.html") #make a legend in html language legenda = "<div> \n<h3>Legend HLA types</h3> \n Patient ID : occurence" for key, value in Ncolors.items(): legenda += "<p><font color=" + value + ">" + key + "=" + str( occ[key]) + "</p> \n" legenda += "</div> \n" #open the previously saved TCR.html, remove the "</body> </html>" add the legend and add "</body> </html>" again. then write to the appropriate file with open('D:/gliph-1.0/gliph/bin/TCR.html', 'r') as f: html = f.read() htmls = html[-0:-15] htmll = htmls + legenda htmlf = htmll + html[-15:] H = open("D:/gliph-1.0/gliph/bin/TCRL457ID.html", "w+")
def map_data(qajar_data, name_color='#a0b038', relative_color='#38b03e', edge_color='#018786', name_shape='ellipse', relative_shape='box', buttons=False, name_size=100, relative_size=100, name_title='', name_image='', relative_image=''): # make a graph named g g = Network(height='1500px', width='100%', bgcolor='black', font_color='white') # show buttons if we call this fuction later with argument buttons=True if buttons == True: g.width = '65%' g.show_buttons() # make an empty list (qajarnames) to fill it by all the names qajarnames = [] # parsing the data in json file for qajar in qajar_data: nam = (qajar['name']) namimag = (qajar['image']) relat = (qajar['relatives']) # set the number of relatives of each name to size of its node namessize = len(relat) * 10 # add a node for each name in data g.add_node(nam, color=name_color, shape=name_shape, image=namimag, physics=False, size=namessize, title=relat) # fill the list of all names qajarnames.append(nam) # add edges between each name and its relative if... for qajar in qajar_data: nam = (qajar['name']) namimag = (qajar['image']) relat = (qajar['relatives']) # if the relative has node already... for eachrelative in relat: # if the relative doesn't have node, add one(this didn't occur in our case) if eachrelative not in qajarnames: g.add_node(eachrelative, color=relative_color, shape=relative_shape, physics=False, size=relative_size) # add edges between names and their relatives g.add_edge(nam, eachrelative, color=edge_color) # the algorithm for model for visualizing data g.barnes_hut() # show an html of this interactive graph g.show('qajar.html') g.save_graph('qajar.html')
def draw_graph(self, option="All", to_ip="All", from_ip="All"): #f = Digraph('network_diagram - '+option, filename=self.filename, engine="dot", format="png") #f.attr(rankdir='LR', size='8,5') if len(memory.lan_hosts) > 40: f = Digraph('network_diagram - ' + option, filename=self.filename, engine="sfdp", format="png") elif len(memory.lan_hosts) > 20: f = Digraph('network_diagram - ' + option, filename=self.filename, engine="circo", format="png") else: f = Digraph('network_diagram - ' + option, filename=self.filename, engine="dot", format="png") interactive_graph = Network(directed=True, height="750px", width="100%", bgcolor="#222222", font_color="white") interactive_graph.barnes_hut() vis_nodes = [] vis_edges = [] f.attr('node', shape='doublecircle') #f.node('defaultGateway') f.attr('node', shape='circle') print("Starting Graph Plotting") edge_present = False mal, tor, http, https, icmp, dns, clear_text, unknown, covert = 0, 0, 0, 0, 0, 0, 0, 0, 0 if option == "All": # add nodes for session in self.sessions: src, dst, port = session.split("/") #print(from_ip, to_ip, src, dst) if (src == from_ip and dst == to_ip) or \ (from_ip == "All" and to_ip == "All") or \ (to_ip == "All" and from_ip == src) or \ (to_ip == dst and from_ip == "All"): # TODO: Improvise this logic below # * graphviz graph is not very good with the ":" in strings if ":" in src: map_src = src.replace(":", ".") else: map_src = src if ":" in dst: map_dst = dst.replace(":", ".") else: map_dst = dst # Lan Host if memory.packet_db[session]["Ethernet"][ "src"] not in memory.lan_hosts: curr_node = map_src + "\n" + memory.packet_db[session][ "Ethernet"]["src"].replace(":", ".") f.node(curr_node) else: curr_node = memory.lan_hosts[memory.packet_db[session][ "Ethernet"]["src"]]["node"] f.node(curr_node) # Destination if dst in memory.destination_hosts: if memory.destination_hosts[dst][ "mac"] in memory.lan_hosts: destination = memory.lan_hosts[ memory.destination_hosts[dst]["mac"]]["node"] dlabel = memory.destination_hosts[dst][ "domain_name"] else: destination = memory.destination_hosts[dst][ "mac"].replace(":", ".") destination += "\n" + "PossibleGateway" dlabel = memory.destination_hosts[dst][ "domain_name"] else: if memory.packet_db[session]["Ethernet"][ "dst"] in memory.lan_hosts: destination = memory.lan_hosts[memory.packet_db[ session]["Ethernet"]["dst"]]["node"] dlabel = "" else: destination = memory.packet_db[session][ "Ethernet"]["dst"].replace(":", ".") destination += "\n" + "PossibleGateway" dlabel = "" # Interactive Graph on Beta, so for now add safety checks ( potential failures in python2) try: interactive_graph.add_node(str(curr_node), str(curr_node), title=str(curr_node), color="yellow") interactive_graph.add_node(str(destination), str(destination), title=str(destination), color="yellow") except Exception as e: print("Interactive graph error occurred: " + str(e)) #if (curr_node, curr_node, title=curr_node, color="yellow") not in vis_nodes: # vis_nodes.append((curr_node, curr_node, title=curr_node, color="yellow")) #if (destination, destination, title=destination, color="yellow") not in vis_nodes: # vis_nodes.append((destination, destination, title=destination, color="yellow")) if curr_node != destination: if session in memory.possible_tor_traffic: f.edge(curr_node, destination, label='TOR: ' + str(map_dst), color="white") tor += 1 #interactive_graph.add_edge(curr_node, destination, color="white", value=tor/100, smooth={type: "curvedCCW", roundness: 0.4}) interactive_graph.add_edge(curr_node, destination, color="white", title='TOR: ' + str(map_dst), smooth={ "type": "curvedCW", "roundness": tor / 9 }) #if edge not in vis_edges:toor # vis_edges.append(edge) if edge_present == False: edge_present = True elif memory.packet_db[session]["covert"]: if port == "53": protocol = "DNS" else: protocol = port f.edge(curr_node, destination, label='Covert/' + protocol + ': ' + str(map_dst) + ": " + str(dlabel), color="cyan") covert += 1 interactive_graph.add_edge( curr_node, destination, color="cyan", title='Covert: ' + str(map_dst) + ": " + dlabel, smooth={ "type": "curvedCCW", "roundness": covert / 12 }) if edge_present == False: edge_present = True else: if port == "443": f.edge(curr_node, destination, label='HTTPS: ' + str(map_dst) + ": " + str(dlabel), color="blue") https += 1 interactive_graph.add_edge( curr_node, destination, color="blue", title='HTTPS: ' + str(map_dst) + ": " + dlabel, smooth={ "type": "curvedCCW", "roundness": https / 10 }) #if edge not in vis_edges: # vis_edges.append(edge) if edge_present == False: edge_present = True elif port == "80": f.edge(curr_node, destination, label='HTTP: ' + str(map_dst) + ": " + str(dlabel), color="green") http += 1 interactive_graph.add_edge( curr_node, destination, color="green", title='HTTP: ' + str(map_dst) + ": " + dlabel, smooth={ "type": "curvedCW", "roundness": http / 12 }) #if edge not in vis_edges: # vis_edges.append(edge) if edge_present == False: edge_present = True elif port == "ICMP": f.edge(curr_node, destination, label='ICMP: ' + str(map_dst), color="black") icmp += 1 interactive_graph.add_edge(curr_node, destination, color="purple", title='ICMP: ' + str(map_dst), smooth={ "type": "curvedCCW", "roundness": icmp / 6 }) #if edge not in vis_edges: # vis_edges.append(edge) if edge_present == False: edge_present = True elif port == "53": f.edge(curr_node, destination, label='DNS: ' + str(map_dst), color="orange") dns += 1 interactive_graph.add_edge( curr_node, destination, color="pink", title='DNS: ' + str(map_dst), smooth={ "type": "curvedCW", "roundness": dns / 5 }) #if edge not in vis_edges: # vis_edges.append(edge) if edge_present == False: edge_present = True elif int(port) in [ 20, 21, 23, 25, 110, 143, 139, 69, 161, 162, 1521 ]: f.edge(curr_node, destination, label='ClearTextProtocol/' + port + ': ' + str(map_dst), color="violet") clear_text += 1 interactive_graph.add_edge( curr_node, destination, color="#9A2EFE", title='ClearTextProtocol/' + port + ': ' + str(map_dst), smooth={ "type": "curvedCW", "roundness": clear_text / 4 }) if edge_present == False: edge_present = True else: f.edge(curr_node, destination, label='UnknownProtocol/' + port + ': ' + str(map_dst), color="brown") unknown += 1 interactive_graph.add_edge( curr_node, destination, color="brown", title='UnknownProtocol/' + port + ': ' + str(map_dst), smooth={ "type": "curvedCW", "roundness": unknown / 3 }) if edge_present == False: edge_present = True else: # This block was just added to handle MAC SPOOF scenario # * Most of the CTF Challenges have fake identical MACs that need to be displayed if map_src in curr_node: other_node = map_dst + "\n" + memory.packet_db[ session]["Ethernet"]["dst"].replace(":", ".") else: other_node = map_src + "\n" + memory.packet_db[ session]["Ethernet"]["src"].replace(":", ".") f.node(other_node) interactive_graph.add_node(str(other_node), str(other_node), title=str(other_node), color="yellow") f.edge(curr_node, other_node, label='WeirdTraffic/' + port, color="pink") unknown += 1 interactive_graph.add_edge(curr_node, other_node, color="pink", title='WeirdTraffic/' + port, smooth={ "type": "curvedCW", "roundness": unknown / 3 }) if edge_present == False: edge_present = True elif option == "HTTP": for session in self.sessions: src, dst, port = session.split("/") if (src == from_ip and dst == to_ip) or \ (from_ip == "All" and to_ip == "All") or \ (to_ip == "All" and from_ip == src) or \ (to_ip == dst and from_ip == "All"): # TODO: Improvise this logic below # * graphviz graph is not very good with the ":" in strings if ":" in src: map_src = src.replace(":", ".") else: map_src = src if ":" in dst: map_dst = dst.replace(":", ".") else: map_dst = dst # Lan Host if memory.packet_db[session]["Ethernet"][ "src"] not in memory.lan_hosts: curr_node = map_src + "\n" + memory.packet_db[session][ "Ethernet"]["src"].replace(":", ".") f.node(curr_node) else: curr_node = memory.lan_hosts[memory.packet_db[session][ "Ethernet"]["src"]]["node"] f.node(curr_node) # Destination if dst in memory.destination_hosts: if memory.destination_hosts[dst][ "mac"] in memory.lan_hosts: destination = memory.lan_hosts[ memory.destination_hosts[dst]["mac"]]["node"] dlabel = memory.destination_hosts[dst][ "domain_name"] else: destination = memory.destination_hosts[dst][ "mac"].replace(":", ".") destination += "\n" + "PossibleGateway" dlabel = memory.destination_hosts[dst][ "domain_name"] else: if memory.packet_db[session]["Ethernet"][ "dst"] in memory.lan_hosts: destination = memory.lan_hosts[memory.packet_db[ session]["Ethernet"]["dst"]]["node"] dlabel = "" else: destination = memory.packet_db[session][ "Ethernet"]["dst"].replace(":", ".") destination += "\n" + "PossibleGateway" dlabel = "" # Interactive Graph on Beta, so for now add safety checks ( potential failures in python2) try: interactive_graph.add_node(str(curr_node), str(curr_node), title=str(curr_node), color="yellow") interactive_graph.add_node(str(destination), str(destination), title=str(destination), color="yellow") except Exception as e: print("Interactive graph error occurred: " + str(e)) if port == "80" and curr_node != destination: f.edge(curr_node, destination, label='HTTP: ' + str(map_dst) + ": " + dlabel, color="green") http += 1 interactive_graph.add_edge(curr_node, destination, color="green", smooth={ "type": "curvedCW", "roundness": http / 10 }) if edge_present == False: edge_present = True elif option == "HTTPS": for session in self.sessions: src, dst, port = session.split("/") if (src == from_ip and dst == to_ip) or \ (from_ip == "All" and to_ip == "All") or \ (to_ip == "All" and from_ip == src) or \ (to_ip == dst and from_ip == "All"): # TODO: Improvise this logic below # * graphviz graph is not very good with the ":" in strings if ":" in src: map_src = src.replace(":", ".") else: map_src = src if ":" in dst: map_dst = dst.replace(":", ".") else: map_dst = dst # Lan Host if memory.packet_db[session]["Ethernet"][ "src"] not in memory.lan_hosts: curr_node = map_src + "\n" + memory.packet_db[session][ "Ethernet"]["src"].replace(":", ".") f.node(curr_node) else: curr_node = memory.lan_hosts[memory.packet_db[session][ "Ethernet"]["src"]]["node"] f.node(curr_node) # Destination if dst in memory.destination_hosts: if memory.destination_hosts[dst][ "mac"] in memory.lan_hosts: destination = memory.lan_hosts[ memory.destination_hosts[dst]["mac"]]["node"] dlabel = memory.destination_hosts[dst][ "domain_name"] else: destination = memory.destination_hosts[dst][ "mac"].replace(":", ".") destination += "\n" + "PossibleGateway" dlabel = memory.destination_hosts[dst][ "domain_name"] else: if memory.packet_db[session]["Ethernet"][ "dst"] in memory.lan_hosts: destination = memory.lan_hosts[memory.packet_db[ session]["Ethernet"]["dst"]]["node"] dlabel = "" else: destination = memory.packet_db[session][ "Ethernet"]["dst"].replace(":", ".") destination += "\n" + "PossibleGateway" dlabel = "" # Interactive Graph on Beta, so for now add safety checks ( potential failures in python2) try: interactive_graph.add_node(str(curr_node), str(curr_node), title=str(curr_node), color="yellow") interactive_graph.add_node(str(destination), str(destination), title=str(destination), color="yellow") except Exception as e: print("Interactive graph error occurred: " + str(e)) if port == "443" and curr_node != destination: f.edge(curr_node, destination, label='HTTPS: ' + str(map_dst) + ": " + dlabel, color="blue") https += 1 interactive_graph.add_edge(curr_node, destination, color="blue", smooth={ "type": "curvedCCW", "roundness": https / 10 }) if edge_present == False: edge_present = True elif option == "Tor": for session in self.sessions: src, dst, port = session.split("/") if (src == from_ip and dst == to_ip) or \ (from_ip == "All" and to_ip == "All") or \ (to_ip == "All" and from_ip == src) or \ (to_ip == dst and from_ip == "All"): # TODO: Improvise this logic below # * graphviz graph is not very good with the ":" in strings if ":" in src: map_src = src.replace(":", ".") else: map_src = src if ":" in dst: map_dst = dst.replace(":", ".") else: map_dst = dst # Lan Host if memory.packet_db[session]["Ethernet"][ "src"] not in memory.lan_hosts: curr_node = map_src + "\n" + memory.packet_db[session][ "Ethernet"]["src"].replace(":", ".") f.node(curr_node) else: curr_node = memory.lan_hosts[memory.packet_db[session][ "Ethernet"]["src"]]["node"] f.node(curr_node) # Destination if dst in memory.destination_hosts: if memory.destination_hosts[dst][ "mac"] in memory.lan_hosts: destination = memory.lan_hosts[ memory.destination_hosts[dst]["mac"]]["node"] dlabel = memory.destination_hosts[dst][ "domain_name"] else: destination = memory.destination_hosts[dst][ "mac"].replace(":", ".") destination += "\n" + "PossibleGateway" dlabel = memory.destination_hosts[dst][ "domain_name"] else: if memory.packet_db[session]["Ethernet"][ "dst"] in memory.lan_hosts: destination = memory.lan_hosts[memory.packet_db[ session]["Ethernet"]["dst"]]["node"] dlabel = "" else: destination = memory.packet_db[session][ "Ethernet"]["dst"].replace(":", ".") destination += "\n" + "PossibleGateway" dlabel = "" # Interactive Graph on Beta, so for now add safety checks ( potential failures in python2) try: interactive_graph.add_node(str(curr_node), str(curr_node), title=str(curr_node), color="yellow") interactive_graph.add_node(str(destination), str(destination), title=str(destination), color="yellow") except Exception as e: print("Interactive graph error occurred: " + str(e)) if session in memory.possible_tor_traffic and curr_node != destination: f.edge(curr_node, destination, label='TOR: ' + str(map_dst), color="white") tor += 1 interactive_graph.add_edge(curr_node, destination, color="white", smooth={ "type": "curvedCW", "roundness": tor / 10 }) if edge_present == False: edge_present = True elif option == "Malicious": # TODO: would we need to iterate over and over all the session irrespective of the properties for session in self.sessions: src, dst, port = session.split("/") if (src == from_ip and dst == to_ip) or \ (from_ip == "All" and to_ip == "All") or \ (to_ip == "All" and from_ip == src) or \ (to_ip == dst and from_ip == "All"): # TODO: Improvise this logic below # * graphviz graph is not very good with the ":" in strings if ":" in src: map_src = src.replace(":", ".") else: map_src = src if ":" in dst: map_dst = dst.replace(":", ".") else: map_dst = dst # Lan Host if memory.packet_db[session]["Ethernet"][ "src"] not in memory.lan_hosts: curr_node = map_src + "\n" + memory.packet_db[session][ "Ethernet"]["src"].replace(":", ".") f.node(curr_node) else: curr_node = memory.lan_hosts[memory.packet_db[session][ "Ethernet"]["src"]]["node"] f.node(curr_node) # Destination if dst in memory.destination_hosts: if memory.destination_hosts[dst][ "mac"] in memory.lan_hosts: destination = memory.lan_hosts[ memory.destination_hosts[dst]["mac"]]["node"] dlabel = memory.destination_hosts[dst][ "domain_name"] else: destination = memory.destination_hosts[dst][ "mac"].replace(":", ".") destination += "\n" + "PossibleGateway" dlabel = memory.destination_hosts[dst][ "domain_name"] else: if memory.packet_db[session]["Ethernet"][ "dst"] in memory.lan_hosts: destination = memory.lan_hosts[memory.packet_db[ session]["Ethernet"]["dst"]]["node"] dlabel = "" else: destination = memory.packet_db[session][ "Ethernet"]["dst"].replace(":", ".") destination += "\n" + "PossibleGateway" dlabel = "" # Interactive Graph on Beta, so for now add safety checks ( potential failures in python2) try: interactive_graph.add_node(str(curr_node), str(curr_node), title=str(curr_node), color="yellow") interactive_graph.add_node(str(destination), str(destination), title=str(destination), color="yellow") except Exception as e: print("Interactive graph error occurred: " + str(e)) if session in memory.possible_mal_traffic and curr_node != destination: f.edge(curr_node, destination, label='Malicious: ' + str(map_dst), color="red") mal += 1 interactive_graph.add_edge(curr_node, destination, color="red", smooth={ "type": "curvedCW", "roundness": mal / 10 }) if edge_present == False: edge_present = True elif option == "ICMP": for session in self.sessions: src, dst, protocol = session.split("/") if (src == from_ip and dst == to_ip) or \ (from_ip == "All" and to_ip == "All") or \ (to_ip == "All" and from_ip == src) or \ (to_ip == dst and from_ip == "All"): if ":" in src: map_src = src.replace(":", ".") else: map_src = src if ":" in dst: map_dst = dst.replace(":", ".") else: map_dst = dst # Lan Host if memory.packet_db[session]["Ethernet"][ "src"] not in memory.lan_hosts: curr_node = map_src + "\n" + memory.packet_db[session][ "Ethernet"]["src"].replace(":", ".") f.node(curr_node) else: curr_node = memory.lan_hosts[memory.packet_db[session][ "Ethernet"]["src"]]["node"] f.node(curr_node) # Destination if dst in memory.destination_hosts: if memory.destination_hosts[dst][ "mac"] in memory.lan_hosts: destination = memory.lan_hosts[ memory.destination_hosts[dst]["mac"]]["node"] dlabel = memory.destination_hosts[dst][ "domain_name"] else: destination = memory.destination_hosts[dst][ "mac"].replace(":", ".") destination += "\n" + "PossibleGateway" dlabel = memory.destination_hosts[dst][ "domain_name"] else: if memory.packet_db[session]["Ethernet"][ "dst"] in memory.lan_hosts: destination = memory.lan_hosts[memory.packet_db[ session]["Ethernet"]["dst"]]["node"] dlabel = "" else: destination = memory.packet_db[session][ "Ethernet"]["dst"].replace(":", ".") destination += "\n" + "PossibleGateway" dlabel = "" # Interactive Graph on Beta, so for now add safety checks ( potential failures in python2) try: interactive_graph.add_node(str(curr_node), str(curr_node), title=str(curr_node), color="yellow") interactive_graph.add_node(str(destination), str(destination), title=str(destination), color="yellow") except Exception as e: print("Interactive graph error occurred: " + str(e)) if protocol == "ICMP" and curr_node != destination: f.edge(curr_node, destination, label='ICMP: ' + str(map_dst), color="black") icmp += 1 interactive_graph.add_edge(curr_node, destination, color="purple", smooth={ "type": "curvedCCW", "roundness": icmp / 10 }) if edge_present == False: edge_present = True elif option == "DNS": for session in self.sessions: src, dst, port = session.split("/") if (src == from_ip and dst == to_ip) or \ (from_ip == "All" and to_ip == "All") or \ (to_ip == "All" and from_ip == src) or \ (to_ip == dst and from_ip == "All"): if ":" in src: map_src = src.replace(":", ".") else: map_src = src if ":" in dst: map_dst = dst.replace(":", ".") else: map_dst = dst # Lan Host if memory.packet_db[session]["Ethernet"][ "src"] not in memory.lan_hosts: curr_node = map_src + "\n" + memory.packet_db[session][ "Ethernet"]["src"].replace(":", ".") f.node(curr_node) else: curr_node = memory.lan_hosts[memory.packet_db[session][ "Ethernet"]["src"]]["node"] f.node(curr_node) # Destination if dst in memory.destination_hosts: if memory.destination_hosts[dst][ "mac"] in memory.lan_hosts: destination = memory.lan_hosts[ memory.destination_hosts[dst]["mac"]]["node"] dlabel = memory.destination_hosts[dst][ "domain_name"] else: destination = memory.destination_hosts[dst][ "mac"].replace(":", ".") destination += "\n" + "PossibleGateway" dlabel = memory.destination_hosts[dst][ "domain_name"] else: if memory.packet_db[session]["Ethernet"][ "dst"] in memory.lan_hosts: destination = memory.lan_hosts[memory.packet_db[ session]["Ethernet"]["dst"]]["node"] dlabel = "" else: destination = memory.packet_db[session][ "Ethernet"]["dst"].replace(":", ".") destination += "\n" + "PossibleGateway" dlabel = "" # Interactive Graph on Beta, so for now add safety checks ( potential failures in python2) try: interactive_graph.add_node(str(curr_node), str(curr_node), title=str(curr_node), color="yellow") interactive_graph.add_node(str(destination), str(destination), title=str(destination), color="yellow") except Exception as e: print("Interactive graph error occurred: " + str(e)) if port == "53" and curr_node != destination: f.edge(curr_node, destination, label='DNS: ' + str(map_dst), color="orange") dns += 1 interactive_graph.add_edge(curr_node, destination, color="pink", smooth={ "type": "curvedCW", "roundness": dns / 10 }) if edge_present == False: edge_present = True if edge_present == False: f.attr(label="No " + option + " Traffic between nodes!", engine='circo', size="5, 5", dpi="300") self.apply_styles(f, self.styles) f.render() interactive_graph.show_buttons(filter_=['physics']) interactive_graph.save_graph(self.filename + ".html")
ret.append( f"<div>closeness: {closeness_centrality[asn]:.4f}</div>") ret.append( f"<div>betweenness: {betweenness_centrality[asn]:.4f}</div>") ret.append(f"<div>peer count: {len(fullasmap[asn])}</div>") ret.append(f"<p></p><div>Peer list:</div><p></p>") for peeras in fullasmap[asn]: ret.append( f"<div onclick=\"select_node({peeras});\">{asname(peeras)}</div>" ) ret.append("<p></p><div>Registry:</div>") _registry = asinfo(asn) #.replace("\n", "<br/>") ret.append(f"<pre>{_registry}</pre>") return "".join(ret) for asn, peers in fullasmap.items(): size = node_centrality[asn] net.add_node(asn, label=asname(asn), size=size, title=gentitle(asn)) for asn, peers in asmap.items(): for p in peers: net.add_edge(asn, p) net.save_graph('index.html') if __name__ == "__main__": main()
def simulfunc(): try: df_i = pd.read_csv('static/inputfile_arm.csv', header=None) df_i = df_i.astype(str) except FileNotFoundError: df_i = pd.read_csv('static/store_data.csv', header=None) df_i = df_i.astype(str) # ----------------------------------------------------------------------------------------------------------------- records = [] for i in range(0, len(df_i)): records.append([ str(df_i.values[i, j]) for j in range(0, 20) if df_i.loc[i, j] != 'nan' ]) # ----------------------------------------------------------------------------------------------------------------- # Explore the data: use different support and confidence levels to see how number of apssociations # change and what are the associated metrics: support, confidence, conviction, lift itemsets, r = efficient_apriori.apriori(records, min_support=0.02, min_confidence=0.3) s = [] cv = [] c = [] lift = [] nn = [] nd = {} count = 0 for i in r: s.append(i.support) # support measure cv.append(i.conviction) # conviction measure c.append(i.confidence) # condidence measure lift.append(i.lift) n = [] for m in i.lhs: n.append(m) for l in i.rhs: n.append(l) nd[count] = n count += 1 nn.append(n) figbar, axbar = plt.subplots(1, 1, figsize=(15, 15)) ax = axbar x = range(len(s)) jn = [', '.join(a) for a in nn] x_labels = jn y = s ax.barh(x, y, color=(0.7, 0.3, 0.3, 0.6), align='center') # , color=color plt.yticks(x, x_labels) plt.tick_params(axis='both', labelsize=18) plt.xlabel('Support level', fontsize=15) plt.legend() plt.savefig('static/barchart.png', bbox_inches='tight') mpld3.save_html(figbar, 'static/barchart.html') # ---------------------------------------------------------------------------------------------------------- # You can look at the trajectory of number of frequent combinations at different selected support level (normalized frequency): cat_vars = [x * 0.01 for x in range(0, 20)] allrules = {} allsets = {} lenrules = [] lensets = [] itemsets = {} for var in cat_vars[1:]: allsets[var] = 0 ti = efficient_apriori.itemsets_from_transactions(records, min_support=var) itemsets[var] = ti for ii in range(len(ti[0])): allsets[var] = allsets[var] + len(ti[0][ii + 1]) lensets.append(allsets[var]) # figsupp, axsupp = plt.subplots(1, 1, figsize=(10, 8)) # ax = axsupp # ax.plot(cat_vars[1:], lensets, '-o', color='#e6194b') # plt.legend(fontsize=18, loc='upper right') # plt.xlabel('Support Level', fontsize=18) # plt.ylabel('Number of Frequent Baskets', fontsize=18) # plt.tick_params(axis='both', labelsize=18) # plt.xlim(0, 0.2) # mpld3.save_html(figsupp, 'support.html') # ---------------------------------------------------------------------------------------------------------- # You can look at the trajectory of number of associations found between combinations at your selected support level: # if you are not sure about entering min_support, min_confidence you can set up support and confidence as a # function of var (in the following for loop) such that a certain level of your desired metric # (conviction, lift, ..) is maintained con_vars = [x * 0.10 for x in range(1, 11)] con_vars = np.round(con_vars, 2) allrules = {} lenrules = [] for var in con_vars: _, rules = efficient_apriori.apriori(records, min_support=0.01, min_confidence=var) allrules[var] = rules lenrules.append(len(allrules[var])) # figcon, axcon = plt.subplots(1, 1) # , figsize=(8, 6) # ax = axcon # ax.plot(con_vars, lenrules, '-o', label='numer of association rules ', color=(0.2, 0.7, 0.9, 0.9)) # plt.legend(fontsize=35, loc='upper right') # plt.xlabel('Min required confidence level based on support of 1%', fontsize=35) # plt.ylabel('Number of association rules', fontsize=35) # plt.tick_params(axis='both', labelsize=35) # mpld3.save_html(figcon, 'static/confidence.html') # ----------------------------------------------------------- # gall: is the network of items that are associated with low confidence level, in order to see most available associations itemsets, r = efficient_apriori.apriori(records, min_support=0.01, min_confidence=0.01) Gall = nx.Graph() c = 1 for i in r: node_list = [] for j in i.lhs: node_list.append(j) for j in i.rhs: node_list.append(j) for j in range(len(node_list) - 1): for k in range(j + 1, len(node_list)): Gall.add_edge(node_list[j], node_list[k]) Gallnodes = list(Gall.nodes) gall = Network() d = nx.degree(Gall) dd = dict(d) # ------------------------------------------------------------------------------------------------------------------ c = 1 # node_list = [] for i in r: node_list = [] for j in i.lhs: node_list.append(j) for j in i.rhs: node_list.append(j) for j in range(len(node_list) - 1): for k in range(j + 1, len(node_list)): gall.add_node(node_list[j], value=dd[node_list[j]], color='#e6194b' ) # (0.8,0.2,0.8,0.9) 46f0f0'#ffe119' '#f032e6' gall.add_node(node_list[k], value=dd[node_list[k]], color='#e6194b') gall.add_edge(node_list[j], node_list[k], color='#fabebe') # ------------------------------------------------------------------------------------------------------------------ # choose appropiriate min_support, min_confidence based on data: elbow in figures support.html and confidence.html # min_support=0.01, min_confidence=0.3 g = nx.Graph() c = 1 algeallrules = list(allrules.keys())[2] for i in allrules[algeallrules]: node_list = [] for j in i.lhs: node_list.append(j) for j in i.rhs: node_list.append(j) for j in range(len(node_list) - 1): for k in range(j + 1, len(node_list)): g.add_edge(node_list[j], node_list[k]) d = nx.degree(g) dd = dict(d) # ----------------------------------------------------------------------------------------------------------------- # if you want to get the red graph in slides: # gn = Network() # c = 1 # algeallrules = list(allrules.keys())[2] # for i in allrules[algeallrules]: # node_list = [] # for j in i.lhs: # node_list.append(j) # for j in i.rhs: # node_list.append(j) # for j in range(len(node_list) - 1): # for k in range(j + 1, len(node_list)): # # gg.add_edge(node_list[j], node_list[k]) # gn.add_node(node_list[j], value=dd[node_list[j]], color='#e6194b') # gn.add_node(node_list[k], value=dd[node_list[k]], color='#e6194b') # gn.add_edge(node_list[j], node_list[k]) # # node_list_g = list(set(node_list)) # gn.save_graph('static/ggrule3percentconf.html') # ----------------------------------------------------------------------------------------------------------------- # community.girvan_newman uses edgebetween ness to detect communities in graph, and finds hierachical clusters # final communities are disjoint clusters, meaning they do not have any common items com = community.girvan_newman(g) communities = list(com) node_com = {} nb_com = len(communities) cl = np.ceil(nb_com / 4) clusters = communities[int(cl)] for i in range(len(list(clusters))): for j in list(clusters)[i]: node_com[j] = i col = [ '#e6194b', 'orangered', '#46f0f0', '#bcf60c', '#808080', '#911eb4', '#000000', '#46f0f0', '#f032e6', '#bcf60c', '#3cb44b', '#ffe119', '#4363d8', '#fabebe', '#008080', '#e6beff', '#9a6324', '#fffac8', '#800000', '#aaffc3', '#808000', '#ffd8b1', '#000075', '#e6194b', '#bcf60c', '#808080', '#911eb4', '#46f0f0', '#f032e6', '#bcf60c', '#3cb44b', '#ffe119', '#4363d8', '#fabebe', '#008080', '#e6beff', '#9a6324', '#fffac8', '#800000', '#aaffc3', '#808000', '#ffd8b1', '#000075' ] node_colors = [] node_colorsg = {} for node in g.nodes(): for j in range(len(clusters)): if node_com[node] == j: node_colors.append(col[j]) node_colorsg[node] = col[j] # ----------------------------------------------------------------------------------------------------------------- c = 0 clusters_out = {} for i in range(len(clusters)): if len(clusters[i]) > 1: clusters_out[c] = clusters[i] c += 1 clusters_out_df = pd.DataFrame.from_dict(clusters_out, orient='index') clusters_out_df = clusters_out_df.T clusters_out_df = clusters_out_df.fillna(' ') colnames = [] for i in clusters_out_df.columns: col = 'Recommended Basket ' + str(i + 1) colnames.append(col) clusters_out_df.columns = colnames dg = nx.degree(g) ddg = dict(d) # ----------------------------------------------------------------------------------------------------------------- gg = Network() c = 1 algeallrules = list(allrules.keys())[2] for i in allrules[algeallrules]: node_list = [] for j in i.lhs: node_list.append(j) for j in i.rhs: node_list.append(j) for j in range(len(node_list) - 1): for k in range(j + 1, len(node_list)): # gg.add_edge(node_list[j], node_list[k]) gg.add_node(node_list[j], value=ddg[node_list[j]], color=node_colorsg[node_list[j]]) gg.add_node(node_list[k], value=ddg[node_list[k]], color=node_colorsg[node_list[k]]) gg.add_edge(node_list[j], node_list[k]) node_list_g = list(set(node_list)) gg.save_graph('static/ggrule3percentconf_c.html') # ----------------------------------------------------------------------------------------------------------------- # if you need nto export graph in json file for better visualization, here you go: # nodes = [{'name': str(i), 'club': gg.node[i]} # for i in g.nodes()] # links = [{'source': u[0], 'target': u[1]} # for u in gg.edges()] # with open('graph.json', 'w') as f: # json.dump({'nodes': nodes, 'links': links}, # f, indent=4,) # ------------------------------------------------------------------------------------------------------------------ cc = {} for i in range(3, int(len(g.nodes) / 2)): cc[i] = [] nb_clus = len(list(k_clique_communities(g, i))) for c in range(nb_clus): tt = list(list(k_clique_communities(g, i))[c]) cc[i].append(tt) cc[i] = sorted(cc[i]) soft_clusters = pd.DataFrame.from_dict(cc, orient='index') soft_clusters.reset_index(inplace=True, drop=True) # ------------------------------------------------------------------------------------------------------------------ # notice in graph that mineral water, spagetti and .. are densly connected woth other clusters: # we may want to construct clusters that allow having shred items # k_clique_communities uses cliques of different size to detect communities in graph, # final communities are overlapping clusters, meaning they are allowed to have common items cc = {} final_package = {} count = 0 for i in range(3, int(len(g.nodes) / 3)): cc[i] = [] nb_clus = len(list(k_clique_communities(g, i))) if count < nb_clus: count = i for c in range(nb_clus): tt = list(list(k_clique_communities(g, i))[c]) cc[i].append(tt) cc[i] = sorted(cc[i]) # final_package = cc[count] soft_clusters = soft_clusters.dropna(axis=0) # ----------------------------------------------------------------------------------------------------------------- overlapping_clusters = pd.DataFrame(columns=[0]) for i in range(len(soft_clusters)): l = [] temp = soft_clusters.iloc[i] for j in temp: l.append(j) df1 = pd.DataFrame(l).T overlapping_clusters = pd.concat([overlapping_clusters, df1], ignore_index=True, axis=1) overlapping_clusters = overlapping_clusters.drop([0], axis=1) overlapping_clusters = overlapping_clusters.fillna(' ') colnames_over = [] for i in overlapping_clusters.columns: col = 'Recommended Basket' + str(i) colnames_over.append(col) overlapping_clusters.columns = colnames_over overlapping_clusters = overlapping_clusters[ overlapping_clusters.columns[:3]] overlapping_clusters.to_csv('static/soft_clusters.csv', index=False) # final_package_df.to_csv('static/final_package_df.csv') clusters_out_df.to_csv('static/clusters_out_df.csv', index=False)
def network(request, n, f): fileType = f inputFile = n htmlFile = "./NetworkViewer/networks/" + n.split(".")[0] + ".html" # The network parameters to be displayed are initiated. All parameters can not be calculated for all types of networks diameter = 0 clusteringCoeff = 0 cliques = 0 degree = 0 connectedComponents = 0 stronglyConnectedomponents = 0 # Different file types needs to be handeled in different ways ngx = None if fileType == 'geneSpider': # Gene spider matrix is read as a matrix that is then transposed, in order to get the directions right networkDf = pd.read_csv(os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'media/' + inputFile), sep=',', header=None) networkDfTransposed = networkDf.T nxg = nx.from_numpy_matrix(np.array(networkDfTransposed), create_using=nx.MultiDiGraph()) elif fileType == 'adjacencyList': # Adjacency lists/Edge lists are red as undirected pandas edgelists networkDf = pd.read_csv(os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'media/' + inputFile), sep='\s+', header=None) nxg = nx.from_pandas_edgelist(networkDf, source=0, target=1) elif fileType == 'directedAdjacencyList': # Directed edge lists need to be read as directed pandas edgelists networkDf = pd.read_csv(os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'media/' + inputFile), sep='\s+', header=None) nxg = nx.from_pandas_edgelist(networkDf, source=0, target=1, create_using=nx.MultiDiGraph()) elif fileType == 'adjacencyMatrix': # Adjacency matrix need to be read as an undirected matrix networkDf = pd.read_csv(os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'media/' + inputFile), sep=',', header=None) nxg = nx.from_numpy_matrix(np.array(networkDf)) elif fileType == 'directedAdjacencyMatrix': # Directed Adjacency matrix need to be read as a directed matrix networkDf = pd.read_csv(os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'media/' + inputFile), sep=',', header=None) nxg = nx.from_numpy_matrix(np.array(networkDf), create_using=nx.MultiDiGraph()) elif fileType == 'funCoup': # FunCoup-network-files contains several columns defining the evidence types and scores. Here, they are red as edge lists, only taking the genes and their edges into account networkDf = pd.read_csv(os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'media/' + inputFile), sep='\t') networkDfGenes = networkDf[['2:Gene1', '3:Gene2']] nxg = nx.from_pandas_edgelist(networkDfGenes, source='2:Gene1', target='3:Gene2') # The networkx-networks are drawn as png-images, since the pyvis can not handle other formats than html. pos = nx.spring_layout(nxg) nx.draw(nxg, pos, with_labels=True) # nx.draw_networkx_edge_labels(nxg, pos, with_labels = True) plt.savefig("./NetworkViewer/templates/NetworkViewer/networks/" + inputFile.split(".")[0] + ".png", format="PNG") plt.clf() # Initiating pyvis interactive network with customized graphics g = Network() g.barnes_hut(gravity=-2000, central_gravity=0.02, spring_length=1, spring_strength=0.000001, damping=0.09, overlap=0) g.toggle_physics(True) # Calculating network prooperties from the networkx graphs if nx.is_directed(nxg): g.directed = True stronglyConnectedomponents = nx.number_strongly_connected_components( nxg) else: connectedComponents = nx.number_connected_components(nxg) if connectedComponents == 1: diameter = nx.diameter(nxg, e=None) clusteringCoeffs = nx.clustering(nxg) clusteringCoeff = np.mean(list(clusteringCoeffs.values())) allCliques = nx.find_cliques(nxg) cliques = len(list(allCliques)) degrees = nxg.degree() degreesOnly = [] for node, degree in degrees: degreesOnly.append(degree) degree = np.mean(degreesOnly) maxDegree = np.max(degreesOnly) # Filling the pyvis graph with nodes and edges from the networkx graph. allNodes = list(nxg.nodes) allSizes = [] # The nodes gets sizes according to their degree for d in degreesOnly: allSizes.append(40 * (d / maxDegree)) g.add_nodes(allNodes, size=allSizes) # The edges gets width according to their weights allEdges = nxg.edges(data=True) edges = [] for a, b, w in allEdges: edges.append((a, b, w.get('weight'))) g.add_edges(edges) g.height = "100%" g.width = "100%" # The pyvis graph is saved as an html file, that is embedded in the network viewer-vindow g.save_graph("./NetworkViewer/templates/NetworkViewer/networks/" + inputFile.split(".")[0] + ".html") # Exporting network properties to the html-view context = { "diameter": ("%.2f" % diameter), "clustering": ("%.2f" % clusteringCoeff), "cliques": ("%.2f" % cliques), "degree": ("%.2f" % degree), "connectedComponents": connectedComponents, "stronglyConnectedomponents": stronglyConnectedomponents, "inputFile": inputFile, "htmlFile": htmlFile } return render(request, 'NetworkViewer/viewNetwork.html', context)
def render_graph(self, graph, filename="graph.html"): net = Network("100%", "100%") source_files = list() for node in graph.nodes(data=True): node_shape = "ellipse" node_id = node[0] node_label = self.format_node_label(node[0], graph.in_degree(node[0]), graph.out_degree(node[0])) if "Scenario:" in node[0]: node_color = self.get_node_color("#2589BD") node_level = 1 if not self.config.show_src_file else 2 if self.config.show_src_file: source_files.append((node[1]["source_file"], node[0])) elif graph.out_degree(node[0]) > 0: node_color = self.get_node_color("#009B72") node_level = 2 if not self.config.show_src_file else 3 else: node_color = self.get_node_color("#EC9A29") node_level = 3 if not self.config.show_src_file else 4 if node[1]["smell"]: node_color = self.get_node_color("red") node_title = "\n".join(node[1]["smell_names"]) net.add_node(n_id=node_id, label=node_label, level=node_level, color=node_color, shape=node_shape, title=node_title) else: net.add_node(n_id=node_id, label=node_label, level=node_level, color=node_color, shape=node_shape) if self.config.show_src_file: for pair in source_files: src = pair[0] dst = pair[1] net.add_node(src, src, shape=node_shape, level=1, color=self.get_node_color("#9FA08E")) net.add_edge(src, dst) edges = list(graph.edges()) for src, dst in edges: edge_dashes = False if src.startswith("Step:") and dst.startswith("Step:"): edge_dashes = True edge_width = edges.count((src, dst)) if self.config.edge_labels: graph_data = graph.get_edge_data(src, dst) labels = list() for key in graph_data: labels.append(str(graph_data[key]["label"])) edge_label = ",".join(labels) net.add_edge(src, dst, dashes=edge_dashes, width=edge_width, label=edge_label) else: net.add_edge(src, dst, dashes=edge_dashes, width=edge_width) net.set_options(""" var options = { "edges": { "font":{ "size": """ + str(self.config.font_size) + """ }, "arrows": { "to": { "enabled": true } }, "color": { "inherit": true, "highlight": "rgba(0,0,0,1)" }, "smooth": false }, "nodes": { "borderWidth": 0.5, "borderWidthSelected":1, "font": { "color": "rgba(255,255,255,1)", "size": """ + str(self.config.font_size) + """ }, "fixed": { "x": false, "y": false } }, "layout": { "improvedLayout": true, "hierarchical": { "enabled": true, "levelSeparation": """ + str(self.config.level_separation) + """, "nodeSpacing": """ + str(self.config.node_spacing) + """, "treeSpacing": 5, "direction": "LR", "blockShifting": true, "edgeMinimization": false, "parentCentralization": true, "sortMethod": "directed" } }, "physics": { "hierarchicalRepulsion": { "centralGravity": 0, "springLength": 0, "springConstant": 0, "damping": 1, "nodeDistance": """ + str(self.config.node_spacing) + """ }, "minVelocity": 0.00, "maxVelocity": 0.00, "solver": "hierarchicalRepulsion" } } """) net.save_graph(f"{self.config.OUTPUT_DIR}/{filename}") return f"{self.config.OUTPUT_DIR}/{filename}"
def fnc_build_graphnx(df_system, dfConnFinal, global_dict, router_mode, filename): from pyvis.network import Network phyList = ['High-Ran','High-Ran-ABR','FronteraPE','Mid-Ran','AggEthernet','FronteraSDH','BBIP'] #plt.figure(figsize=(15,15)) #G = nx.Graph() G = Network("800px", "1300px", heading="") #G.set_edge_smooth('dynamic') G.set_options(""" var options = { "nodes": { "size":13, "font": { "size": 11 } }, "edges": { "arrowStrikethrough": false, "color": { "inherit": false }, "font": { "size": 10, "align": "top" }, "smooth": false }, "manipulation": { "enabled": true, "initiallyActive": true }, "physics": { "barnesHut": { "centralGravity": 0.2, "springLength": 100, "springConstant": 0.01, "damping": 0.7, "avoidOverlap": 1 }, "maxVelocity": 5, "minVelocity": 0.47, "solver": "barnesHut" } } """) for i in df_system.itertuples(): router_name = i[1] id = i[2] ip_system = i[3] if global_dict[router_name]['HWfunction'] in phyList: physics = False else: physics = True G.add_node(router_name, label = str(router_name) + "\n" + str(ip_system), chassis = global_dict[router_name]['HWtype'], ip = str(ip_system), color = get_attribute(router_name,"color",global_dict)['yEd'], physics = physics, ) for i in dfConnFinal.itertuples(): router_name_A = i[1] port_name_A = i[2] ip_A = i[12] ip_system_A = global_dict[router_name_A]['system'] router_name_B = i[5] port_name_B = i[4] ip_B = i[13] ip_system_B = global_dict[router_name_B]['system'] cable_id = i[3] if global_dict[router_name_A]['HWfunction'] in phyList and global_dict[router_name_B]['HWfunction'] in phyList: physics = False else: physics = True edgeLabel = port_name_A + "--" + port_name_B if cable_id in ["N/A",None]: G.add_edge(source=router_name_A, to=router_name_B, label=edgeLabel, physics=physics) else: G.add_edge(source=router_name_A, to=router_name_B, label=cable_id + "\n" + edgeLabel, physics=physics) #G.barnes_hut(overlap=1) #G.force_atlas_2based(overlap=1) #G.show_buttons() G.save_graph("plugins/topo.html")