def route_manager_reader(reaction, result_dir, tree_size, threshold, reference, methods, id_to_string, mode='enumeration'): smarts_dict = {} G = None prev = "" f = open(os.path.join(result_dir, "pt.txt"), 'r') for i in range(tree_size): l = f.readline() if l == '': break prev = l c = 0 f.close() f = open("./tmp.dot", 'w') f.write(prev) f.close() G = read_dot("./tmp.dot") linecache.clearcache() route_manager = RouteManager(G, id_to_string, result_dir, reference, methods) return route_manager
def graph_mining(dotfiles): dataset=[] for dotf in dotfiles: dotnx=nx.Graph(read_dot(dotf)) dataset.append(dotnx) gsp=GSpan(dataset) gsp.GraphSet_Projection()
def get_rts_from_pot_file(pot_file): """ return an rts dict and networkx graph """ from networkx.drawing.nx_pydot import from_pydot, read_dot import networkx as nx with open(pot_file) as file: pot = read_dot(file) graph = nx.DiGraph(pot) rts = [] for node, data in graph.nodes(data=True): task = {"id":get_int(node)} for k, v in data.items(): task[str(k)] = get_int(v) for successor in graph.successors(node): task["p"] = [] for k, v in graph.get_edge_data(node, successor).items(): task["p"].append({k:get_int(v),"id":get_int(successor)}) rts.append(task) # sort by id rts.sort(key=lambda t: t["id"]) return (rts, graph)
def load_graph(path): g = read_dot(path) for v in g.nodes: g.nodes[v]['weight'] = int(g.nodes[v]['weight']) for e in g.edges: g.edges[e]['weight'] = int(g.edges[e]['weight']) return g
def main(): parser = argparse.ArgumentParser(description="Finds cycles in dot file graphs, such as those from Puppet. " "By Jason Antman <http://blog.jasonantman.com>") parser.add_argument('dotfile', metavar='DOTFILE', nargs='?', type=argparse.FileType('r'), default=sys.stdin, help="the dotfile to process. Uses standard input if argument is '-' or not present") parser.add_argument("--only-shortest", action='store_true', help="only show the shortest cycles. Example: if both A->C and A->B->C exist, only show the former. " "This vastly reduces the amount of output when analysing dependency issues.") parser.add_argument("--print-labels", action='store_true', help="print the node labels instead of their ids.") args = parser.parse_args() # read in the specified file, create a networkx DiGraph G = nx.DiGraph(read_dot(args.dotfile)) C = nx.simple_cycles(G) if args.only_shortest: C = remove_super_cycles(C) if args.print_labels: C = extract_node_labels(C, G) for i in C: # append the first node again so that the cycle is complete i.append(i[0]) print(" -> ".join(i))
def read_write_legend(dot): with TempTestDir("tst") as dir_name: path = os.path.join(dir_name, "graph.dot") with open(path, 'w') as graph_file: graph_file.write(dot.legend) graph = read_dot(path) return graph
def cflow_dir(a): index = nx.DiGraph() for c in glob.glob(os.path.join(a, "*.c")): g = None dot = str(Path(c).with_suffix(".dot")) if not os.path.isfile(dot): g = import_cflow(c, Path(c).with_suffix(".cflow")) write_dot(g, dot) print(dot, popen("ctags -x %s | wc -l" % (c))[0], len(set(e[0] for e in g.edges()))) else: print(dot) try: # g = nx.drawing.nx_agraph.read_dot(dot) g = read_dot(dot) except (TypeError, pygraphviz.agraph.DotError): print('nx_pydot <- nx_agraph') g = nx.drawing.nx_pydot.read_dot(dot) # digraph_print(g, [], Path(c).with_suffix(".tree")) # index.add_nodes_from(g.nodes()) index.add_edges_from(g.edges()) write_dot(index, str(os.path.join(a, 'index.dot'))) digraph_print(digraph_tree(index), [], os.path.join(a, 'index.tree')) return index
def load_config(self, config, configtype="json"): try: if configtype == "dot": self.graph = read_dot(config) elif configtype == "json": self.graph = json_graph.node_link_graph(json.load(open(config))) elif configtype == "gml": self.graph = read_gml(config) except Exception,e: print "Config read error: {}".format(str(e)) self.logger.error("Error reading configuration: {}".format(str(e))) sys.exit(-1)
def load_config(self, config, configtype="json"): try: if configtype == "dot": self.graph = read_dot(config) elif configtype == "json": self.graph = json_graph.node_link_graph(json.load( open(config))) elif configtype == "gml": self.graph = read_gml(config) except Exception, e: print "Config read error: {}".format(str(e)) self.logger.error("Error reading configuration: {}".format(str(e))) sys.exit(-1)
def dot_to_graph(d: str) -> networkx.Graph: """Convert a DOT formatted string into a Graph Args: d: str, the DOT formatted string. See [1] for more about the DOT language and formatting. Returns: Graph, the NetworkX Graph """ with io.StringIO(d) as f: return read_dot(path=f)
def load_graph(path): G = nx.DiGraph(read_dot(path)) for n in G.nodes: G.nodes[n]["name"] = n G.nodes[n]["out_edges"] = [ edge_operator_operands(G.edges[e]) for e in G.edges if e[0] == n ] G.nodes[n]["in_edges"] = [ edge_operator_operands(G.edges[e]) for e in G.edges if e[1] == n ] for e in G.edges: u, v = e G.edges[e]["src"] = G.nodes[u] G.edges[e]["dst"] = G.nodes[v] return G
def main(): try: graph = read_dot(sys.argv[1]) root = next(filter(lambda R: graph.in_degree(R) == 0, graph.nodes())) paths = list( map( lambda P: list(all_simple_paths(graph, source=root, target=P)), list((filter(lambda L: graph.out_degree(L) == 0, graph.nodes))))) list(map(lambda R: print("".join(R)), list(chain.from_iterable(paths)))) except: print("Unexpected error")
def test_from_shower(): # will need an eventwise with Parents, Children, MCPID # layer -1 0 1 1 -1 2 2 3 3 3 -1 # idx 0 1 2 3 4 5 6 7 8 9 10 children = [[], [0, 2, 3], [5], [6, 5, 4], [], [], [7, 8, 9], [], [], []] parents = [[1], [], [1], [1], [3], [2, 3], [3], [6], [6], [6]] mcpid = [4, 5, 5, 3, 2, 1, -5, -1, 7, 11] n_nodes = len(children) shower = FormShower.Shower(list(range(n_nodes)), parents, children, mcpid) dot = DrawTrees.DotGraph(shower) with TempTestDir("tst") as dir_name: path = os.path.join(dir_name, "graph.dot") with open(path, 'w') as graph_file: graph_file.write(str(dot)) graph = read_dot(path) assert len(graph.nodes) == n_nodes assert len(graph.edges) == 10
def main(args): # create a random graph # G=nx.gnp_random_graph(n=10,p=0.6) G = read_dot(args.dot_file) # remember the coordinates of the vertices if args.layout == "shell": coords = nx.shell_layout(G) else: coords = pydot_layout(G) # remove "len(clique)>2" if you're interested in maxcliques with 2 edges cliques = [clique for clique in nx.find_cliques(G) if len(clique) > 2] # #draw the graph nx.draw(G, pos=coords, node_size=200) print(coords) in_more_than_one_clique = set() seen = set() for clique in cliques: for node in clique: if node not in seen: seen.add(node) else: in_more_than_one_clique.add(node) # already_processed_nodes = set() for clique in cliques: print("Clique to appear: ", clique) color = draw_circle_around_clique(clique, coords) node_colors = [] for node in clique: if node not in in_more_than_one_clique: node_colors.append(color) else: node_colors.append("grey") nx.draw_networkx_nodes(G, pos=coords, nodelist=clique, node_color=node_colors, node_size=200) # already_processed_nodes.update(clique) nx.draw_networkx_labels(G, coords, font_size=8) # nx.draw_networkx_nodes(G,pos=coords, nodelist=clique) plt.savefig(args.outfile) plt.close()
def test_create_dot(self): """ Verify that a dot file is correctly created from pstats data stored in a file field. """ with self._stats_file(): try: # create dot with tempfile.NamedTemporaryFile(delete=False) as dotfile: dot = _create_dot(self._profile(), 5) dotfile.write(dot.encode('utf-8')) # verify generated dot is valid G = read_dot(dotfile.name) self.assertGreater(len(G.nodes()), 0) finally: os.unlink(dotfile.name)
def main(): parser = argparse.ArgumentParser(description="Finds cycles in dot file graphs, such as those from Puppet. " "By Jason Antman <http://blog.jasonantman.com>") parser.add_argument('dotfile', metavar='DOTFILE', nargs='?', type=argparse.FileType('r'), default=sys.stdin, help="the dotfile to process. Uses standard input if argument is '-' or not present") parser.add_argument("--only-shortest", action='store_true', help="only show the shortest cycles. Example: if both A->C and A->B->C exist, only show the former. " "This vastly reduces the amount of output when analysing dependency issues.") args = parser.parse_args() # read in the specified file, create a networkx DiGraph G = nx.DiGraph(read_dot(args.dotfile)) C = nx.simple_cycles(G) if args.only_shortest: C = remove_super_cycles(C) for i in C: print(i)
def test_create_dot(self): """ Verify that a dot file is correctly created from pstats data stored in a file field. """ with self._stats_file() as filename: try: # create dot with tempfile.NamedTemporaryFile(delete=False) as dotfile: dot = _create_dot(self._profile(), 5) dot = dot.encode('utf-8') if PY3 else dot dotfile.write(dot) # verify generated dot is valid G = read_dot(dotfile.name) self.assertGreater(len(G.nodes()), 0) finally: os.unlink(dotfile.name)
def main(): path = "" if (len(sys.argv) > 1): path = sys.argv[1] else: usage() sys.exit(1) try: fh = open(path) except IOError as e: sys.stderr.write("ERROR: could not read file " + path + "\n") usage() sys.exit(1) # read in the specified file, create a networkx DiGraph G = nx.DiGraph(read_dot(path)) C = nx.simple_cycles(G) for i in C: print i
def main(input_file, k, output_file=None): G = read_dot(input_file) dic = _gen_vars(G, k) f1 = _each_v_has_c(dic) f2 = _each_v_only_one_c(dic) f3 = _adj_not_same_c(G, dic) solver = Solver() solver.add(f1 + f2 + f3) if solver.check().r > 0: m = solver.model() colors = [p for p in chain(*dic.values()) if m[p]] palette = color_palette('pastel', k).as_hex() for c in colors: s = str(c) cc = s.split('_') G.nodes[cc[0]]['fillcolor'] = palette[int(cc[1])] G.nodes[cc[0]]['style'] = 'filled' if output_file: write_dot(G, output_file) else: print(to_pydot(G).to_string()) else: print(f"No se pudo encontrar una {k}-coloración para la gráfica", file=sys.stderr)
]) label.extend([ "betweeness_min", "betweeness_max", "betweeness_median", "betweeness_mean", "betweeness_std" ]) label.extend([ "shortest_path_min", "shortest_path_max", "shortest_path_median", "shortest_path_mean", "shortest_path_std" ]) label.append("diameter") label.append("radius") csvwriter.writerow(label) for fname in os.listdir("output"): print(fname) try: G = read_dot(os.path.join("output", fname)) nx.draw(G) except: print("cannot load graph") continue if G.number_of_nodes() == 0: print("Cannot read binary file") continue data = [] data.append(fname) data.append(G.number_of_nodes()) data.append(G.number_of_edges()) data.append(density(G)) deg_centrality = degree_centrality(G) data.extend(properties_of_array(deg_centrality)) cln_centrality = closeness_centrality(G)
import networkx as nx from networkx.drawing.nx_pydot import read_dot, write_dot from networkx.algorithms.dag import descendants G = nx.DiGraph(read_dot("/home/daniel/kernel-fullprop.dot")) DAG = nx.DiGraph() # Need to convert the graph to a DAG components = nx.strongly_connected_components(G) contraction = dict() for c in components: component_iter = iter(c) contract_to = next(component_iter) contraction[contract_to] = contract_to while True: try: element = next(component_iter) assert element not in contraction contraction[element] = contract_to except StopIteration: break DAG.add_node(contract_to) for (u, v) in G.edges: DAG.add_edge(contraction[u], contraction[v]) blacklist = { "vsnprintf(bottom)", "sprintf(bottom)", "snprintf(bottom)",
def convert_from_dot(dot_path, app=None): """ Create a DAG object from a graph stored as a dot file. Parameters ------------------------ dot_path - string Where the dot file is located. app - None/string The application that the graph represents, e.g., "Cholesky". Returns ------------------------ dag - DAG object Converted version of the graph described by the dot file. Notes ------------------------ 1. This is very slow so isn't recommended for even medium-sized DAGs. The vast majority of the time seems to be taken by read_dot (from Networkx) itself, which surely shouldn't be so slow, so I may investigate this further in the future. """ # Use read_dot from Networkx to load the graph. graph = nx.DiGraph(read_dot(dot_path)) # Check if it's actually a DAG and make the graph directed if it isn't already. if graph.is_directed(): G = graph else: G = nx.DiGraph() G.name = graph.name G.add_nodes_from(graph) done = set() for u, v in graph.edges(): if (v, u) not in done: G.add_edge(u, v) done.add((u, v)) G.graph = deepcopy(graph.graph) G.node = deepcopy(graph.node) # Look for cycles. try: nx.topological_sort(G) except nx.NetworkXUnfeasible: raise ValueError( 'Input graph in convert_from_dot has at least one cycle so is not a DAG!' ) # Get app name from the filename (if not input). if not app: filename = dot_path.split('/')[-1] app = filename.split('.')[0] # Create the DAG object. dag = DAG(app=app) done = set() for t in nx.topological_sort(G): if t not in done: nd = Task() nd.ID = int(t) nd.entry = True done.add(t) else: for n in dag.DAG: if n.ID == int(t): nd = n break count = 0 for s in G.successors(t): count += 1 if s not in done: nd1 = Task() nd1.ID = int(s) done.add(s) else: for n in dag.DAG: if n.ID == int(s): nd1 = n break dag.DAG.add_edge(nd, nd1) if not count: nd.exit = True dag.num_tasks = len(dag.DAG) return dag
import os from networkx.drawing.nx_pydot import read_dot from map import utils BASE_DIR = os.path.dirname(os.path.abspath(__file__)) current_map = read_dot(os.path.join(BASE_DIR, "dot_map.dot")) tables = [n for n in current_map.nodes if n.lower().startswith('t')] CHEF = 'chef' adjacency = utils.get_adjacency_with_direction(current_map) all_locations = current_map.nodes
import networkx as nx from networkx.drawing.nx_pydot import write_dot from networkx.drawing.nx_pydot import read_dot fileName = sys.argv[1] outputPath = sys.argv[2] if outputPath[-1] != "/": outputPath += "/" # Slicing path to get the file name i = sys.argv[1].rfind("/") + 1 outputFile = sys.argv[1][i:] # Building graph and converting string names like 0_2_1 to integers while keeping the topology G = read_dot(fileName) G = nx.convert_node_labels_to_integers(G) # Full file path - including name ERRADO! CORRIGIR. RSTRIP NAO E SUFIXO OU PREFIXO E SIM COMB DE VAL fileName = outputPath + outputFile.split(".")[0] + ".txt" V = G.number_of_nodes() E = G.number_of_edges() # Output .dot in .txt as an edge list, with number of vertices and edges on the first line nx.write_edgelist(G, fileName, data=False) with open(fileName, 'r') as original: data = original.read() with open(fileName, 'w') as modified: modified.write(str(V) + " " + str(E) + "\n" + data)
def update_figure(n_points): # This example shows how to draw a NetworkX graph in Plotly. import plotly.graph_objects as go import networkx as nx from networkx.drawing.nx_pydot import read_dot G = nx.DiGraph(read_dot('generated.gv')) # pos = nx.nx_pydot.graphviz_layout(G) ''' pos = nx.nx_pydot.pydot_layout(G, prog='neato') pos = nx.circular_layout(G) pos = nx.planar_layout(G) #nice pos = nx.spiral_layout(G) #nice ''' pos = nx.nx_pydot.graphviz_layout(G, prog='dot') # nice # Retrieve the coordinates of the nodes and store them # in two separate edge lists: one for X coordinates # and one for Y coordinates. edge_x = [] edge_y = [] for edge in G.edges(): # Get the X and Y coordinates from pos. x0 = pos[edge[0]][0] y0 = pos[edge[0]][1] x1 = pos[edge[1]][0] y1 = pos[edge[1]][1] edge_x.append(x0) edge_x.append(x1) # Plotly will connect every node on the edge list in sequence. # Inserting a "None" node to prevent Plotly from connecting nodes # that should not be connected. edge_x.append(None) edge_y.append(y0) edge_y.append(y1) edge_y.append(None) # Create a line plot to draw all the edges. edge_trace = go.Scatter( x=edge_x, y=edge_y, mode='lines', line=dict(width=0.7)) # Create a node list node_x = [] node_y = [] descriptions = [] for node in G.nodes(): # Saving node coordinates to the node list. x = pos[node][0] y = pos[node][1] node_x.append(x) node_y.append(y) descriptions.append( str(df[df['code'] == node]['name'].values)[2:-2] + '<br>Departement:' + str(df[df['code'] == node]['department'].values)[2:-2] + '<br>Code:' + str(node) + '<br>Credit Hour:' + str(df[df['code'] == node]['Credit Hours'].values)[2:-2] + '<br>Prerequistite:' + add_br_to_long_string(str(df[df['code'] == node]['Prerequisites'].values))[2:-2] + '<br>Description:<br>' + add_br_to_long_string(str(df[df['code'] == node]['Description'].values))[2:-2] ) # print(list(G.nodes)) # Create a scatter plot to draw all the nodes. node_trace = go.Scatter( x=node_x, y=node_y, mode="markers+text", # Show both markers and labels text=list(G.nodes), # The texts will be the labels of the nodes textposition="middle left", # Place the text to the left of the node hovertext=list(descriptions), hoverinfo='text', # Tooltip will be from the "text" argument marker=dict( size=10, color="white", line_width=2) ) fig = go.Figure(data=[edge_trace, node_trace], layout=go.Layout( # title="A NetworkX Graph Rendered with Plotly", titlefont_size=16, showlegend=False, xaxis=dict(showgrid=False, zeroline=False, showticklabels=False), yaxis=dict(showgrid=False, zeroline=False, showticklabels=False)) ) fig.update_layout( autosize=False, width=1900, height=1000 ) return fig
def load_data(path): network = read_dot(path) return network
def compose_query_by_dot_path(self, dot_path: str): g = read_dot(dot_path) self.compose_query_by_digraph(g)
def __dotfile2json(self, dot_file): g = nx.Graph(read_dot(dot_file)) json_data = json_graph.node_link_data(g) return json_data
if __name__=="__main__": G1=nx.Graph() G2=nx.Graph() G3=nx.Graph() G4=nx.Graph() G5=nx.Graph() G6=nx.Graph() G7=nx.Graph() G8=nx.Graph() G9=nx.Graph() G10=nx.Graph() G11=nx.Graph() G12=nx.Graph() G13=nx.Graph() G14=nx.Graph(read_dot("../cpp-src/EventNet/EventNet_BoostGL.dot")) G1.add_edges_from([(1,2),(2,3),(2,4),(1,5),(2,5)]) G2.add_edges_from([(1,3),(4,3),(5,4),(2,5),(1,5)]) G3.add_edges_from([(1,4),(5,3),(5,2),(4,5),(1,2)]) G4.add_edges_from([(1,5),(2,3),(4,2),(2,5),(3,2),(1,6)]) G5.add_edges_from([(1,4),(2,1),(5,4),(4,1),(4,2),(5,6),(7,1)]) G6.add_edges_from([(5,6),(7,1)]) G7.add_edges_from([(5,4),(7,1)]) G8.add_edges_from([(1,4),(5,6),(7,1)]) rgof=open("./InterviewAlgorithm/graphmining/RecursiveGlossOverlapGraph.WebSpider-HTML.out.1") G9edges=json.load(rgof) G9.add_edges_from(G9edges) #print "G9edges:",G9.edges() rgof=open("./InterviewAlgorithm/graphmining/RecursiveGlossOverlapGraph.WebSpider-HTML.out.2") G10edges=json.load(rgof) G10.add_edges_from(G10edges)
def graph_mining(dotfiles): dataset=[] for dotf in dotfiles: dotnx=nx.Graph(read_dot(dotf)) dataset.append(dotnx) gsp=GSpan(dataset) gsp.GraphSet_Projection() if __name__=="__main__": spcon=SparkContext() sqlcon=SQLContext(spcon) #input_dot_files=['/media/Krishna_iResearch_/Krishna_iResearch_OpenSource/GitHub/virgo64-linux-github-code/linux-kernel-extensions/drivers/virgo/saturn_program_analysis/saturn_program_analysis_trees/cfg_read_virgo_kernel_analytics_config.dot','/media/Krishna_iResearch_/Krishna_iResearch_OpenSource/GitHub/virgo64-linux-github-code/linux-kernel-extensions/drivers/virgo/saturn_program_analysis/saturn_program_analysis_trees/memory_skbuff_h_skb_header_pointer_cfg.dot','/media/Krishna_iResearch_/Krishna_iResearch_OpenSource/GitHub/asfer-github-code/python-src/software_analytics/kcachegrind_callgraph_DiscreteHyperbolicFactorization_TileSearch_Optimized.dot','/media/Krishna_iResearch_/Krishna_iResearch_OpenSource/GitHub/asfer-github-code/python-src/software_analytics/kcachegrind_callgraph_ls.dot'] ftrace_callgraph_dot("ftrace.DiscreteHyperbolicFactorization_TileSearch_Optimized.log") input_dot_files=['/media/Krishna_iResearch_/Krishna_iResearch_OpenSource/GitHub/asfer-github-code/python-src/software_analytics/kcachegrind_callgraph_DiscreteHyperbolicFactorization_TileSearch_Optimized.dot','/media/Krishna_iResearch_/Krishna_iResearch_OpenSource/GitHub/asfer-github-code/python-src/software_analytics/kcachegrind_callgraph_ls.dot','CyclomaticComplexitySparkMapReducer.ftrace_callgraph.dot'] for dot_file in input_dot_files: nxg=nx.Graph(read_dot(dot_file)) nxgnodes=[[]] nxgedges=[[]] cnt=0 for n in nxg.nodes(): cnt += 1 nxgnodes[0].append((str(cnt),str(n),str(n))) for e in nxg.edges(): nxgedges[0].append((str(e[0]),str(e[1]),"causes")) print "nxgnodes:",nxgnodes print "nxgedges:",nxgedges ndf=sqlcon.createDataFrame(nxgnodes,["id","name","label"]) vdf=sqlcon.createDataFrame(nxgedges,["src","dst","relationship"]) print "ndf:",ndf print "vdf:",vdf gf=GraphFrame(ndf,vdf)
if len(sys.argv) == 3: dotFolderPath = os.path.abspath(sys.argv[1]) jsonFolderPath = os.path.abspath(sys.argv[2]) if not os.path.exists(jsonFolderPath): os.mkdir(jsonFolderPath) dotFiles = filesWithExtensions(dotFolderPath, dotFileExtensions) counter = 0 for dotFile in dotFiles: dotFilePath = dotFolderPath + '/' + dotFile try: dot_graph = pgv.AGraph(dotFilePath) graph_netx = from_agraph(dot_graph) except (ValueError, DotError) as e: try: graph_netx = read_dot(dotFilePath) except (ValueError, DotError) as f: print(dotFile + ' not in graphviz format') continue graph_json = json_graph.node_link_data(graph_netx) #dot_graph) filename = dotFile[:dotFile.rfind('.')] json.dump(graph_json, open(jsonFolderPath + '/' + filename + '.json', 'w'), indent=2) print(filename + '.json converted') counter += 1 with open(dataFileName, 'w') as jsonFile: data = {} jsonFiles = filesWithExtensions(jsonFolderPath, ['.json']) while dataFileName in jsonFiles:
def buildCFGDepedcyInflueceDict(cfgPath, filePathList): '''build cfgdependency and cfg inlunce for cur code :param cfgPath: path to icfg.dot :param filePathList: the source code path :return: :first: {nodeid:set(its control dependent nodeids)} :second: {line:set{its control dependent lines}} :third: {nodeid:set(its control influent nodeids)} :forth: {line:set(its control influent lines)} ''' CFGs = read_dot(cfgPath) funcDict, lineToEntry = getCFGFuncEntryExit(CFGs, filePathList) nodeToLineDict = buildCFGNodeToLineDict(CFGs, filePathList) CFGs = preProcessCFG(CFGs, funcDict, filePathList, lineToEntry, nodeToLineDict) CFGsR = CFGs.reverse(True) CFGDepdcyDict = dict() CFGInfluenceDict = dict() for funName in funcDict: exitID = funcDict[funName]["exit"] entryID = funcDict[funName]["entry"] pdomTree = buildPdomTree(entryID, exitID, funName, CFGsR) PdomPtoChildDict = buildPdomPtoChildDict(pdomTree, CFGsR) CFGs.add_edges_from([(funName, entryID), (funName, exitID)]) CFGDepdcyDict, CFGInfluenceDict = appendCFGDepdcyDict( CFGDepdcyDict, CFGInfluenceDict, CFGs, PdomPtoChildDict, pdomTree) CFGLineDepdcyDict = dict() CFGLineInflueceDict = dict() for key in CFGInfluenceDict: if key in nodeToLineDict: lineKey = nodeToLineDict[key] if lineKey not in CFGLineInflueceDict: CFGLineInflueceDict[lineKey] = set() for cfif in list(CFGInfluenceDict[key]): if cfif in nodeToLineDict: CFGLineInflueceDict[lineKey].add(nodeToLineDict[cfif]) if cfif in funcDict: CFGLineInflueceDict[lineKey].add( funcDict[cfif]["entryLine"]) elif key in funcDict: lineKey = funcDict[key]["entryLine"] if lineKey not in CFGLineInflueceDict: CFGLineInflueceDict[lineKey] = set() for cfif in list(CFGInfluenceDict[key]): if cfif in nodeToLineDict: CFGLineInflueceDict[lineKey].add(nodeToLineDict[cfif]) if cfif in funcDict: CFGLineInflueceDict[lineKey].add( funcDict[cfif]["entryLine"]) for key in CFGDepdcyDict: if key in nodeToLineDict: lineKey = nodeToLineDict[key] if lineKey not in CFGLineDepdcyDict: CFGLineDepdcyDict[lineKey] = set() for cfdp in list(CFGDepdcyDict[key]): if cfdp in nodeToLineDict: CFGLineDepdcyDict[lineKey].add(nodeToLineDict[cfdp]) if cfdp in funcDict: CFGLineDepdcyDict[lineKey].add(funcDict[cfdp]["entryLine"]) elif key in funcDict: lineKey = funcDict[key]["entryLine"] if lineKey not in CFGLineDepdcyDict: CFGLineDepdcyDict[lineKey] = set() for cfdp in list(CFGDepdcyDict[key]): if cfdp in nodeToLineDict: CFGLineDepdcyDict[lineKey].add(nodeToLineDict[cfdp]) if cfdp in funcDict: CFGLineDepdcyDict[lineKey].add(funcDict[cfdp]["entryLine"]) return CFGDepdcyDict, CFGLineDepdcyDict, CFGInfluenceDict, CFGLineInflueceDict
def buildCodeGadgetList(doneLines, svfgPath, cfgPath, sensiAPIPath, filePathList, callgPath): ''' :param doneLines: :param svfgPath: :param cfgPath: :param sensiAPIPath: :param filePathList: :param callgPath: :return: ''' SVFG = read_dot(svfgPath) # SVFG in networkx format CFGs = read_dot(cfgPath) # CFGs in networkx format CallG = read_dot(callgPath) # CallG in networkx format print("start - getting callgraph...") calleeToCallerDict = buildCallGraphDict( CFGs, CallG, filePathList) # {callee line:set(caller line)} print("end - getting callgraph...") CFGsNodeToLineDict = buildCFGNodeToLineDict( CFGs, filePathList) # for cfg:{nodeid:line} entryDict = getCFGsEntryLineToNode( CFGs, filePathList) # for cfg entry nodes:{line:nodeid} for key in entryDict: # add entry nodeid to line info CFGsNodeToLineDict[entryDict[key]] = key print("start - extracting apis...") apiLines = extractAPILines(doneLines, CFGs, sensiAPIPath, CFGsNodeToLineDict) # apiLines = extractOperatorLines(doneLines, operatorLineDir, filePathList) print("end - extracting apis...") if (apiLines == list()): return dict() # =====================================# # build program dependence graph (PDG) # # =====================================# print("start - building PDG...") PDG = nx.DiGraph() control_edges = list() print("start - building CFGLineDepdcy...") CFGDepdcy, CFGLineDepdcy, CFGInfluence, CFGLineInfluence = buildCFGDepedcyInflueceDict( cfgPath, filePathList) # {line:its cfg-dependency-line} for line in CFGLineDepdcy: control_dependent_line_set = CFGLineDepdcy[line] for control_dependent_line in control_dependent_line_set: control_edges.append((control_dependent_line, line, {"c/d": "c"})) PDG.add_edges_from(control_edges) print("end - building CFGLineDepdcy...") print("start - building VFGLineDepdcy...") SVFGnodeToLineDict = buildSVFGNodeToLineDict(SVFG, filePathList) value_edges = list() for n in SVFG._pred: if n not in SVFGnodeToLineDict: continue cur_line = SVFGnodeToLineDict[n] pred_lines = set() visited_nodeid = set() end = n.find(":s") if end != -1: n = n[:end] visited_nodeid.add(n) dfs_pred_lines(True, n, SVFGnodeToLineDict, SVFG._pred, pred_lines, visited_nodeid) for pred_line in pred_lines: if pred_line != cur_line: value_edges.append((pred_line, cur_line, {"c/d": "d"})) PDG.add_edges_from(value_edges) print("end - building VFGLineDepdcy...") print("end - building PDG...") codeGadgetLinesDict = dict() # {api:cdg} for apiLine in apiLines: print("start - processing apiline{}:...".format(apiLine)) sliced_lines = set() # backward traversal bqueue = list() visited = set() bqueue.append(apiLine) visited.add(apiLine) while bqueue: fro = bqueue.pop(0) sliced_lines.add(fro) if fro in PDG._pred: for pred in PDG._pred[fro]: if pred not in visited: visited.add(pred) bqueue.append(pred) # forward traversal fqueue = list() visited = set() fqueue.append(apiLine) visited.add(apiLine) while fqueue: fro = fqueue.pop(0) sliced_lines.add(fro) if fro in PDG._succ: for succ in PDG._succ[fro]: if succ not in visited: visited.add(succ) fqueue.append(succ) print("end - processing apiline{}:...".format(apiLine)) entryToLinesDict = dict() for relatedLine in sliced_lines: entry = locateLineEntry(relatedLine, list(entryDict.keys())) if entry not in entryToLinesDict: entryToLinesDict[entry] = list() entryToLinesDict[entry].append(relatedLine) for entry in entryToLinesDict: entryToLinesDict[entry] = sorted(entryToLinesDict[entry]) callChianList = getCallChainList(calleeToCallerDict, entryToLinesDict.keys()) maxLen = 0 lenToCallChainDict = dict() for callChian in callChianList: if len(callChian) > maxLen: maxLen = len(callChian) if len(callChian) not in lenToCallChainDict: lenToCallChainDict[len(callChian)] = list() lenToCallChainDict[len(callChian)].append(callChian) finalCallChain = list() finalCallChain.extend(lenToCallChainDict[maxLen][0]) for callChain in lenToCallChainDict[maxLen]: for entry in callChain: if entry not in set(finalCallChain): finalCallChain.append(entry) for lenn in lenToCallChainDict: if lenn != maxLen: for callChain in lenToCallChainDict[lenn]: for entry in callChain: if entry not in set(finalCallChain): finalCallChain.append(entry) codeGadgetLines = list() for entry in finalCallChain: codeGadgetLines.extend(entryToLinesDict[entry]) codeGadgetLinesDict[apiLine] = codeGadgetLines return codeGadgetLinesDict
def __call__(self, graph_data: str, number_input_symbols: int = None, graph_data_format: str = 'dot_string') -> FDFA: """ Returns an initialized FDFA instance given the graph_data graph_data and graph_data_format must match :param graph_data: The string containing graph data. Could be a filename or just the raw data :param number_input_symbols: The number of input symbols to the FDFA needed to compute the correct frequency flows in the case of cycles. Only really optional when using a graph_data_format that already has this information. :param graph_data_format: The graph data file format. {'dot_file', 'dot_string', 'learning_interface'} :returns: instance of an initialized FDFA object :raises ValueError: checks if graph_data and graph_data_format have a compatible data loader. :raises ValueError: checks, based on graph_data_format, whether it is legal to not specify the number_input_symbols. """ has_number_input_symbols = number_input_symbols is not None if graph_data_format == 'dot_string': graph = nx_agraph.from_agraph(pygraphviz.AGraph(string=graph_data)) elif graph_data_format == 'dot_file': graph = read_dot(graph_data) elif graph_data_format == 'learning_interface': learning_interface = graph_data graph = read_dot(learning_interface.learned_model_filepath) number_input_symbols = learning_interface.num_training_examples has_number_input_symbols = True else: msg = 'graph_data_format ({}) must be one of: "dot_file", ' + \ '"dot_string"'.format(graph_data_format) raise ValueError(msg) if not has_number_input_symbols: msg = f'must provide the number_input_symbols to load a FDFA' raise ValueError(msg) # these are not things that are a part of flexfringe's automaton # data model, so give them default values final_transition_sym = DEFAULT_FINAL_TRANS_SYMBOL empty_transition_sym = DEFAULT_EMPTY_TRANS_SYMBOL config_data = FDFA.load_flexfringe_data(graph, number_input_symbols, final_transition_sym, empty_transition_sym) config_data['final_transition_sym'] = final_transition_sym config_data['empty_transition_sym'] = empty_transition_sym nodes_have_changed = (self.nodes != config_data['nodes']) edges_have_changed = (self.edges != config_data['edges']) no_instance_loaded_yet = (self._instance is None) if no_instance_loaded_yet or nodes_have_changed or edges_have_changed: # saving these so we can just return initialized instances if the # underlying data has not changed self.nodes = config_data['nodes'] self.edges = config_data['edges'] self._instance = FDFA(**config_data) return self._instance
def compare_schema(file_prev, file_now, file_result): ''' Input - graph file in dot format Output - file in png format with diff ''' logger = logging.getLogger('compare_schema') logger.debug('Try read dot files with network schema') logger.info('Read file {}'.format(file_prev)) try: graph_prev = read_dot(file_prev) except IOError: logger.error('Fail load schema') return logger.info('Read file {}'.format(file_now)) try: graph_now = read_dot(file_now) except IOError: logger.error('Fail to load schema') return fig = plt.subplots(figsize=(14, 9)) if nx.is_isomorphic(graph_prev, graph_now): logger.info('Schema is equal') else: logger.info('Changes were exists') graph_sum = nx.compose(graph_prev, graph_now) pos = nx.spring_layout(graph_sum) for node in graph_sum.nodes(): color = '#00CCCC' if node not in graph_prev.nodes() or node not in graph_now.nodes(): color = 'red' nx.draw_networkx_nodes(graph_sum, pos, nodelist=[node], node_size=1500, node_color=color) for begin, end in graph_sum.edges(): color = '#00CCCC' if (begin, end) not in graph_prev.edges() or ( begin, end) not in graph_now.edges(): color = 'red' el_b = el_e = dict() nx.draw_networkx_edges(graph_sum, pos, edgelist=[([begin, end])], edge_color=color) # Вот здесь прибито гвоздями # ПОчему при импорте есть проблема # Линки мультидиграф должны быть tuple вида # (источник.назначение,номер_линка) # после импорта, номер линка уезжает в аттрибут edge-а el_b[(begin, end)] = graph_sum[begin][end]['0']['interfaces'] el_e[(begin, end)] = graph_sum[begin][end]['0']['interfaces'] nx.draw_networkx_edge_labels(graph_sum, pos, edge_labels=el_b, label_pos=0.5, font_size=8, font_color=color) nodes_lab = dict() for host in graph_sum.nodes(): if not graph_sum.nodes[host]['cap'] == '': nodes_lab[host] = host + '\n' + graph_sum.nodes[host]['cap'] else: nodes_lab[host] = host nx.draw_networkx_labels(graph_sum, pos, font_size=12) logger.info('Save diff schema to {}'.format(file_result)) plt.savefig(file_result)
def initial_computation(reaction, result_dir, tree_size, threshold, reference, methods, mode='enumeration'): smarts_dict = {} G = None prev = "" f = open(os.path.join(result_dir, "pt.txt"), 'r') for i in range(tree_size): l = f.readline() if l == '': break prev = l c = 0 f.close() #l = linecache.getline(result_dir+"pt.txt", n) f = open("./tmp.dot", 'w') f.write(prev) f.close() #os.system("dot -Tpng ./tmp.dot -o nako.png") G = read_dot("./tmp.dot") linecache.clearcache() mol_file = open(os.path.join(result_dir, "finalResult.txt")) id_to_string = {} for l in mol_file: if l.startswith("digraph") or l.startswith("l") or l.startswith( "in") or l.startswith("end") or l.startswith("$.digraph"): continue else: try: l = l.rstrip("\n").split(",") node_id = l[0] string = ''.join(l[1:]) if len(string) == 0: continue id_to_string[node_id] = string except: pass route_manager = RouteManager(G, id_to_string, result_dir, reference, methods) print(route_manager.target) start = time.time() #route_manager.extract_route() if mode == 'enumeration': route_manager.enumeration() elif mode == 'sampling': route_manager.sampling() end = time.time() print("Elapsed time for {}:{} sec.".format(mode, end - start)) #print("the number of route:", len(route_manager.route_list)) #start = time.time() #route_manager.add_name() with open( os.path.join(result_dir, "route_manager_" + str(tree_size) + ".pickle"), "wb") as f1: pickle.dump(route_manager, f1) return route_manager