def route_manager_reader(reaction,
result_dir,
tree_size,
threshold,
reference,
methods,
id_to_string,
mode='enumeration'):
smarts_dict = {}
G = None
prev = ""
f = open(os.path.join(result_dir, "pt.txt"), 'r')
for i in range(tree_size):
l = f.readline()
if l == '':
break
prev = l
c = 0
f.close()
f = open("./tmp.dot", 'w')
f.write(prev)
f.close()
G = read_dot("./tmp.dot")
linecache.clearcache()
route_manager = RouteManager(G, id_to_string, result_dir, reference,
methods)
return route_manager
def graph_mining(dotfiles): dataset=[] for dotf in dotfiles: dotnx=nx.Graph(read_dot(dotf)) dataset.append(dotnx) gsp=GSpan(dataset) gsp.GraphSet_Projection()
def get_rts_from_pot_file(pot_file):
""" return an rts dict and networkx graph """
from networkx.drawing.nx_pydot import from_pydot, read_dot
import networkx as nx
with open(pot_file) as file:
pot = read_dot(file)
graph = nx.DiGraph(pot)
rts = []
for node, data in graph.nodes(data=True):
task = {"id":get_int(node)}
for k, v in data.items():
task[str(k)] = get_int(v)
for successor in graph.successors(node):
task["p"] = []
for k, v in graph.get_edge_data(node, successor).items():
task["p"].append({k:get_int(v),"id":get_int(successor)})
rts.append(task)
# sort by id
rts.sort(key=lambda t: t["id"])
return (rts, graph)
def load_graph(path):
g = read_dot(path)
for v in g.nodes:
g.nodes[v]['weight'] = int(g.nodes[v]['weight'])
for e in g.edges:
g.edges[e]['weight'] = int(g.edges[e]['weight'])
return g
def main():
parser = argparse.ArgumentParser(description="Finds cycles in dot file graphs, such as those from Puppet. "
"By Jason Antman <http://blog.jasonantman.com>")
parser.add_argument('dotfile', metavar='DOTFILE', nargs='?', type=argparse.FileType('r'), default=sys.stdin,
help="the dotfile to process. Uses standard input if argument is '-' or not present")
parser.add_argument("--only-shortest", action='store_true',
help="only show the shortest cycles. Example: if both A->C and A->B->C exist, only show the former. "
"This vastly reduces the amount of output when analysing dependency issues.")
parser.add_argument("--print-labels", action='store_true',
help="print the node labels instead of their ids.")
args = parser.parse_args()
# read in the specified file, create a networkx DiGraph
G = nx.DiGraph(read_dot(args.dotfile))
C = nx.simple_cycles(G)
if args.only_shortest:
C = remove_super_cycles(C)
if args.print_labels:
C = extract_node_labels(C, G)
for i in C:
# append the first node again so that the cycle is complete
i.append(i[0])
print(" -> ".join(i))
def read_write_legend(dot):
with TempTestDir("tst") as dir_name:
path = os.path.join(dir_name, "graph.dot")
with open(path, 'w') as graph_file:
graph_file.write(dot.legend)
graph = read_dot(path)
return graph
def cflow_dir(a):
index = nx.DiGraph()
for c in glob.glob(os.path.join(a, "*.c")):
g = None
dot = str(Path(c).with_suffix(".dot"))
if not os.path.isfile(dot):
g = import_cflow(c, Path(c).with_suffix(".cflow"))
write_dot(g, dot)
print(dot,
popen("ctags -x %s | wc -l" % (c))[0],
len(set(e[0] for e in g.edges())))
else:
print(dot)
try:
# g = nx.drawing.nx_agraph.read_dot(dot)
g = read_dot(dot)
except (TypeError, pygraphviz.agraph.DotError):
print('nx_pydot <- nx_agraph')
g = nx.drawing.nx_pydot.read_dot(dot)
# digraph_print(g, [], Path(c).with_suffix(".tree"))
# index.add_nodes_from(g.nodes())
index.add_edges_from(g.edges())
write_dot(index, str(os.path.join(a, 'index.dot')))
digraph_print(digraph_tree(index), [], os.path.join(a, 'index.tree'))
return index
def load_config(self, config, configtype="json"):
try:
if configtype == "dot":
self.graph = read_dot(config)
elif configtype == "json":
self.graph = json_graph.node_link_graph(json.load(open(config)))
elif configtype == "gml":
self.graph = read_gml(config)
except Exception,e:
print "Config read error: {}".format(str(e))
self.logger.error("Error reading configuration: {}".format(str(e)))
sys.exit(-1)
def load_config(self, config, configtype="json"):
try:
if configtype == "dot":
self.graph = read_dot(config)
elif configtype == "json":
self.graph = json_graph.node_link_graph(json.load(
open(config)))
elif configtype == "gml":
self.graph = read_gml(config)
except Exception, e:
print "Config read error: {}".format(str(e))
self.logger.error("Error reading configuration: {}".format(str(e)))
sys.exit(-1)
def dot_to_graph(d: str) -> networkx.Graph:
"""Convert a DOT formatted string into a Graph
Args:
d:
str, the DOT formatted string. See [1] for more about
the DOT language and formatting.
Returns:
Graph, the NetworkX Graph
"""
with io.StringIO(d) as f:
return read_dot(path=f)
def load_graph(path):
G = nx.DiGraph(read_dot(path))
for n in G.nodes:
G.nodes[n]["name"] = n
G.nodes[n]["out_edges"] = [
edge_operator_operands(G.edges[e]) for e in G.edges if e[0] == n
]
G.nodes[n]["in_edges"] = [
edge_operator_operands(G.edges[e]) for e in G.edges if e[1] == n
]
for e in G.edges:
u, v = e
G.edges[e]["src"] = G.nodes[u]
G.edges[e]["dst"] = G.nodes[v]
return G
def main():
try:
graph = read_dot(sys.argv[1])
root = next(filter(lambda R: graph.in_degree(R) == 0, graph.nodes()))
paths = list(
map(
lambda P: list(all_simple_paths(graph, source=root, target=P)),
list((filter(lambda L: graph.out_degree(L) == 0,
graph.nodes)))))
list(map(lambda R: print("".join(R)),
list(chain.from_iterable(paths))))
except:
print("Unexpected error")
def test_from_shower():
# will need an eventwise with Parents, Children, MCPID
# layer -1 0 1 1 -1 2 2 3 3 3 -1
# idx 0 1 2 3 4 5 6 7 8 9 10
children = [[], [0, 2, 3], [5], [6, 5, 4], [], [], [7, 8, 9], [], [], []]
parents = [[1], [], [1], [1], [3], [2, 3], [3], [6], [6], [6]]
mcpid = [4, 5, 5, 3, 2, 1, -5, -1, 7, 11]
n_nodes = len(children)
shower = FormShower.Shower(list(range(n_nodes)), parents, children, mcpid)
dot = DrawTrees.DotGraph(shower)
with TempTestDir("tst") as dir_name:
path = os.path.join(dir_name, "graph.dot")
with open(path, 'w') as graph_file:
graph_file.write(str(dot))
graph = read_dot(path)
assert len(graph.nodes) == n_nodes
assert len(graph.edges) == 10
def main(args):
# create a random graph
# G=nx.gnp_random_graph(n=10,p=0.6)
G = read_dot(args.dot_file)
# remember the coordinates of the vertices
if args.layout == "shell":
coords = nx.shell_layout(G)
else:
coords = pydot_layout(G)
# remove "len(clique)>2" if you're interested in maxcliques with 2 edges
cliques = [clique for clique in nx.find_cliques(G) if len(clique) > 2]
# #draw the graph
nx.draw(G, pos=coords, node_size=200)
print(coords)
in_more_than_one_clique = set()
seen = set()
for clique in cliques:
for node in clique:
if node not in seen:
seen.add(node)
else:
in_more_than_one_clique.add(node)
# already_processed_nodes = set()
for clique in cliques:
print("Clique to appear: ", clique)
color = draw_circle_around_clique(clique, coords)
node_colors = []
for node in clique:
if node not in in_more_than_one_clique:
node_colors.append(color)
else:
node_colors.append("grey")
nx.draw_networkx_nodes(G,
pos=coords,
nodelist=clique,
node_color=node_colors,
node_size=200)
# already_processed_nodes.update(clique)
nx.draw_networkx_labels(G, coords, font_size=8)
# nx.draw_networkx_nodes(G,pos=coords, nodelist=clique)
plt.savefig(args.outfile)
plt.close()
def test_create_dot(self):
"""
Verify that a dot file is correctly created from pstats data stored in a file field.
"""
with self._stats_file():
try:
# create dot
with tempfile.NamedTemporaryFile(delete=False) as dotfile:
dot = _create_dot(self._profile(), 5)
dotfile.write(dot.encode('utf-8'))
# verify generated dot is valid
G = read_dot(dotfile.name)
self.assertGreater(len(G.nodes()), 0)
finally:
os.unlink(dotfile.name)
def main():
parser = argparse.ArgumentParser(description="Finds cycles in dot file graphs, such as those from Puppet. "
"By Jason Antman <http://blog.jasonantman.com>")
parser.add_argument('dotfile', metavar='DOTFILE', nargs='?', type=argparse.FileType('r'), default=sys.stdin,
help="the dotfile to process. Uses standard input if argument is '-' or not present")
parser.add_argument("--only-shortest", action='store_true',
help="only show the shortest cycles. Example: if both A->C and A->B->C exist, only show the former. "
"This vastly reduces the amount of output when analysing dependency issues.")
args = parser.parse_args()
# read in the specified file, create a networkx DiGraph
G = nx.DiGraph(read_dot(args.dotfile))
C = nx.simple_cycles(G)
if args.only_shortest:
C = remove_super_cycles(C)
for i in C:
print(i)
def test_create_dot(self):
"""
Verify that a dot file is correctly created from pstats data stored in a file field.
"""
with self._stats_file() as filename:
try:
# create dot
with tempfile.NamedTemporaryFile(delete=False) as dotfile:
dot = _create_dot(self._profile(), 5)
dot = dot.encode('utf-8') if PY3 else dot
dotfile.write(dot)
# verify generated dot is valid
G = read_dot(dotfile.name)
self.assertGreater(len(G.nodes()), 0)
finally:
os.unlink(dotfile.name)
def main():
path = ""
if (len(sys.argv) > 1):
path = sys.argv[1]
else:
usage()
sys.exit(1)
try:
fh = open(path)
except IOError as e:
sys.stderr.write("ERROR: could not read file " + path + "\n")
usage()
sys.exit(1)
# read in the specified file, create a networkx DiGraph
G = nx.DiGraph(read_dot(path))
C = nx.simple_cycles(G)
for i in C:
print i
def main(input_file, k, output_file=None):
G = read_dot(input_file)
dic = _gen_vars(G, k)
f1 = _each_v_has_c(dic)
f2 = _each_v_only_one_c(dic)
f3 = _adj_not_same_c(G, dic)
solver = Solver()
solver.add(f1 + f2 + f3)
if solver.check().r > 0:
m = solver.model()
colors = [p for p in chain(*dic.values()) if m[p]]
palette = color_palette('pastel', k).as_hex()
for c in colors:
s = str(c)
cc = s.split('_')
G.nodes[cc[0]]['fillcolor'] = palette[int(cc[1])]
G.nodes[cc[0]]['style'] = 'filled'
if output_file:
write_dot(G, output_file)
else:
print(to_pydot(G).to_string())
else:
print(f"No se pudo encontrar una {k}-coloración para la gráfica",
file=sys.stderr)
])
label.extend([
"betweeness_min", "betweeness_max", "betweeness_median",
"betweeness_mean", "betweeness_std"
])
label.extend([
"shortest_path_min", "shortest_path_max", "shortest_path_median",
"shortest_path_mean", "shortest_path_std"
])
label.append("diameter")
label.append("radius")
csvwriter.writerow(label)
for fname in os.listdir("output"):
print(fname)
try:
G = read_dot(os.path.join("output", fname))
nx.draw(G)
except:
print("cannot load graph")
continue
if G.number_of_nodes() == 0:
print("Cannot read binary file")
continue
data = []
data.append(fname)
data.append(G.number_of_nodes())
data.append(G.number_of_edges())
data.append(density(G))
deg_centrality = degree_centrality(G)
data.extend(properties_of_array(deg_centrality))
cln_centrality = closeness_centrality(G)
import networkx as nx
from networkx.drawing.nx_pydot import read_dot, write_dot
from networkx.algorithms.dag import descendants
G = nx.DiGraph(read_dot("/home/daniel/kernel-fullprop.dot"))
DAG = nx.DiGraph()
# Need to convert the graph to a DAG
components = nx.strongly_connected_components(G)
contraction = dict()
for c in components:
component_iter = iter(c)
contract_to = next(component_iter)
contraction[contract_to] = contract_to
while True:
try:
element = next(component_iter)
assert element not in contraction
contraction[element] = contract_to
except StopIteration:
break
DAG.add_node(contract_to)
for (u, v) in G.edges:
DAG.add_edge(contraction[u], contraction[v])
blacklist = {
"vsnprintf(bottom)",
"sprintf(bottom)",
"snprintf(bottom)",
def convert_from_dot(dot_path, app=None):
"""
Create a DAG object from a graph stored as a dot file.
Parameters
------------------------
dot_path - string
Where the dot file is located.
app - None/string
The application that the graph represents, e.g., "Cholesky".
Returns
------------------------
dag - DAG object
Converted version of the graph described by the dot file.
Notes
------------------------
1. This is very slow so isn't recommended for even medium-sized DAGs. The vast majority of the time
seems to be taken by read_dot (from Networkx) itself, which surely shouldn't be so slow, so I may
investigate this further in the future.
"""
# Use read_dot from Networkx to load the graph.
graph = nx.DiGraph(read_dot(dot_path))
# Check if it's actually a DAG and make the graph directed if it isn't already.
if graph.is_directed():
G = graph
else:
G = nx.DiGraph()
G.name = graph.name
G.add_nodes_from(graph)
done = set()
for u, v in graph.edges():
if (v, u) not in done:
G.add_edge(u, v)
done.add((u, v))
G.graph = deepcopy(graph.graph)
G.node = deepcopy(graph.node)
# Look for cycles.
try:
nx.topological_sort(G)
except nx.NetworkXUnfeasible:
raise ValueError(
'Input graph in convert_from_dot has at least one cycle so is not a DAG!'
)
# Get app name from the filename (if not input).
if not app:
filename = dot_path.split('/')[-1]
app = filename.split('.')[0]
# Create the DAG object.
dag = DAG(app=app)
done = set()
for t in nx.topological_sort(G):
if t not in done:
nd = Task()
nd.ID = int(t)
nd.entry = True
done.add(t)
else:
for n in dag.DAG:
if n.ID == int(t):
nd = n
break
count = 0
for s in G.successors(t):
count += 1
if s not in done:
nd1 = Task()
nd1.ID = int(s)
done.add(s)
else:
for n in dag.DAG:
if n.ID == int(s):
nd1 = n
break
dag.DAG.add_edge(nd, nd1)
if not count:
nd.exit = True
dag.num_tasks = len(dag.DAG)
return dag
import os
from networkx.drawing.nx_pydot import read_dot
from map import utils
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
current_map = read_dot(os.path.join(BASE_DIR, "dot_map.dot"))
tables = [n for n in current_map.nodes if n.lower().startswith('t')]
CHEF = 'chef'
adjacency = utils.get_adjacency_with_direction(current_map)
all_locations = current_map.nodes
import networkx as nx
from networkx.drawing.nx_pydot import write_dot
from networkx.drawing.nx_pydot import read_dot
fileName = sys.argv[1]
outputPath = sys.argv[2]
if outputPath[-1] != "/":
outputPath += "/"
# Slicing path to get the file name
i = sys.argv[1].rfind("/") + 1
outputFile = sys.argv[1][i:]
# Building graph and converting string names like 0_2_1 to integers while keeping the topology
G = read_dot(fileName)
G = nx.convert_node_labels_to_integers(G)
# Full file path - including name ERRADO! CORRIGIR. RSTRIP NAO E SUFIXO OU PREFIXO E SIM COMB DE VAL
fileName = outputPath + outputFile.split(".")[0] + ".txt"
V = G.number_of_nodes()
E = G.number_of_edges()
# Output .dot in .txt as an edge list, with number of vertices and edges on the first line
nx.write_edgelist(G, fileName, data=False)
with open(fileName, 'r') as original:
data = original.read()
with open(fileName, 'w') as modified:
modified.write(str(V) + " " + str(E) + "\n" + data)
def update_figure(n_points):
# This example shows how to draw a NetworkX graph in Plotly.
import plotly.graph_objects as go
import networkx as nx
from networkx.drawing.nx_pydot import read_dot
G = nx.DiGraph(read_dot('generated.gv'))
# pos = nx.nx_pydot.graphviz_layout(G)
'''
pos = nx.nx_pydot.pydot_layout(G, prog='neato')
pos = nx.circular_layout(G)
pos = nx.planar_layout(G) #nice
pos = nx.spiral_layout(G) #nice
'''
pos = nx.nx_pydot.graphviz_layout(G, prog='dot') # nice
# Retrieve the coordinates of the nodes and store them
# in two separate edge lists: one for X coordinates
# and one for Y coordinates.
edge_x = []
edge_y = []
for edge in G.edges():
# Get the X and Y coordinates from pos.
x0 = pos[edge[0]][0]
y0 = pos[edge[0]][1]
x1 = pos[edge[1]][0]
y1 = pos[edge[1]][1]
edge_x.append(x0)
edge_x.append(x1)
# Plotly will connect every node on the edge list in sequence.
# Inserting a "None" node to prevent Plotly from connecting nodes
# that should not be connected.
edge_x.append(None)
edge_y.append(y0)
edge_y.append(y1)
edge_y.append(None)
# Create a line plot to draw all the edges.
edge_trace = go.Scatter(
x=edge_x,
y=edge_y,
mode='lines',
line=dict(width=0.7))
# Create a node list
node_x = []
node_y = []
descriptions = []
for node in G.nodes():
# Saving node coordinates to the node list.
x = pos[node][0]
y = pos[node][1]
node_x.append(x)
node_y.append(y)
descriptions.append(
str(df[df['code'] == node]['name'].values)[2:-2] +
'<br>Departement:' +
str(df[df['code'] == node]['department'].values)[2:-2] +
'<br>Code:' +
str(node) +
'<br>Credit Hour:' +
str(df[df['code'] == node]['Credit Hours'].values)[2:-2] +
'<br>Prerequistite:' +
add_br_to_long_string(str(df[df['code'] == node]['Prerequisites'].values))[2:-2] +
'<br>Description:<br>' +
add_br_to_long_string(str(df[df['code'] == node]['Description'].values))[2:-2]
)
# print(list(G.nodes))
# Create a scatter plot to draw all the nodes.
node_trace = go.Scatter(
x=node_x,
y=node_y,
mode="markers+text", # Show both markers and labels
text=list(G.nodes), # The texts will be the labels of the nodes
textposition="middle left", # Place the text to the left of the node
hovertext=list(descriptions),
hoverinfo='text', # Tooltip will be from the "text" argument
marker=dict(
size=10,
color="white",
line_width=2)
)
fig = go.Figure(data=[edge_trace, node_trace],
layout=go.Layout(
# title="A NetworkX Graph Rendered with Plotly",
titlefont_size=16,
showlegend=False,
xaxis=dict(showgrid=False, zeroline=False, showticklabels=False),
yaxis=dict(showgrid=False, zeroline=False, showticklabels=False))
)
fig.update_layout(
autosize=False,
width=1900,
height=1000
)
return fig
def load_data(path):
network = read_dot(path)
return network
def compose_query_by_dot_path(self, dot_path: str):
g = read_dot(dot_path)
self.compose_query_by_digraph(g)
def __dotfile2json(self, dot_file):
g = nx.Graph(read_dot(dot_file))
json_data = json_graph.node_link_data(g)
return json_data
if __name__=="__main__":
G1=nx.Graph()
G2=nx.Graph()
G3=nx.Graph()
G4=nx.Graph()
G5=nx.Graph()
G6=nx.Graph()
G7=nx.Graph()
G8=nx.Graph()
G9=nx.Graph()
G10=nx.Graph()
G11=nx.Graph()
G12=nx.Graph()
G13=nx.Graph()
G14=nx.Graph(read_dot("../cpp-src/EventNet/EventNet_BoostGL.dot"))
G1.add_edges_from([(1,2),(2,3),(2,4),(1,5),(2,5)])
G2.add_edges_from([(1,3),(4,3),(5,4),(2,5),(1,5)])
G3.add_edges_from([(1,4),(5,3),(5,2),(4,5),(1,2)])
G4.add_edges_from([(1,5),(2,3),(4,2),(2,5),(3,2),(1,6)])
G5.add_edges_from([(1,4),(2,1),(5,4),(4,1),(4,2),(5,6),(7,1)])
G6.add_edges_from([(5,6),(7,1)])
G7.add_edges_from([(5,4),(7,1)])
G8.add_edges_from([(1,4),(5,6),(7,1)])
rgof=open("./InterviewAlgorithm/graphmining/RecursiveGlossOverlapGraph.WebSpider-HTML.out.1")
G9edges=json.load(rgof)
G9.add_edges_from(G9edges)
#print "G9edges:",G9.edges()
rgof=open("./InterviewAlgorithm/graphmining/RecursiveGlossOverlapGraph.WebSpider-HTML.out.2")
G10edges=json.load(rgof)
G10.add_edges_from(G10edges)
def graph_mining(dotfiles):
dataset=[]
for dotf in dotfiles:
dotnx=nx.Graph(read_dot(dotf))
dataset.append(dotnx)
gsp=GSpan(dataset)
gsp.GraphSet_Projection()
if __name__=="__main__":
spcon=SparkContext()
sqlcon=SQLContext(spcon)
#input_dot_files=['/media/Krishna_iResearch_/Krishna_iResearch_OpenSource/GitHub/virgo64-linux-github-code/linux-kernel-extensions/drivers/virgo/saturn_program_analysis/saturn_program_analysis_trees/cfg_read_virgo_kernel_analytics_config.dot','/media/Krishna_iResearch_/Krishna_iResearch_OpenSource/GitHub/virgo64-linux-github-code/linux-kernel-extensions/drivers/virgo/saturn_program_analysis/saturn_program_analysis_trees/memory_skbuff_h_skb_header_pointer_cfg.dot','/media/Krishna_iResearch_/Krishna_iResearch_OpenSource/GitHub/asfer-github-code/python-src/software_analytics/kcachegrind_callgraph_DiscreteHyperbolicFactorization_TileSearch_Optimized.dot','/media/Krishna_iResearch_/Krishna_iResearch_OpenSource/GitHub/asfer-github-code/python-src/software_analytics/kcachegrind_callgraph_ls.dot']
ftrace_callgraph_dot("ftrace.DiscreteHyperbolicFactorization_TileSearch_Optimized.log")
input_dot_files=['/media/Krishna_iResearch_/Krishna_iResearch_OpenSource/GitHub/asfer-github-code/python-src/software_analytics/kcachegrind_callgraph_DiscreteHyperbolicFactorization_TileSearch_Optimized.dot','/media/Krishna_iResearch_/Krishna_iResearch_OpenSource/GitHub/asfer-github-code/python-src/software_analytics/kcachegrind_callgraph_ls.dot','CyclomaticComplexitySparkMapReducer.ftrace_callgraph.dot']
for dot_file in input_dot_files:
nxg=nx.Graph(read_dot(dot_file))
nxgnodes=[[]]
nxgedges=[[]]
cnt=0
for n in nxg.nodes():
cnt += 1
nxgnodes[0].append((str(cnt),str(n),str(n)))
for e in nxg.edges():
nxgedges[0].append((str(e[0]),str(e[1]),"causes"))
print "nxgnodes:",nxgnodes
print "nxgedges:",nxgedges
ndf=sqlcon.createDataFrame(nxgnodes,["id","name","label"])
vdf=sqlcon.createDataFrame(nxgedges,["src","dst","relationship"])
print "ndf:",ndf
print "vdf:",vdf
gf=GraphFrame(ndf,vdf)
if len(sys.argv) == 3:
dotFolderPath = os.path.abspath(sys.argv[1])
jsonFolderPath = os.path.abspath(sys.argv[2])
if not os.path.exists(jsonFolderPath):
os.mkdir(jsonFolderPath)
dotFiles = filesWithExtensions(dotFolderPath, dotFileExtensions)
counter = 0
for dotFile in dotFiles:
dotFilePath = dotFolderPath + '/' + dotFile
try:
dot_graph = pgv.AGraph(dotFilePath)
graph_netx = from_agraph(dot_graph)
except (ValueError, DotError) as e:
try:
graph_netx = read_dot(dotFilePath)
except (ValueError, DotError) as f:
print(dotFile + ' not in graphviz format')
continue
graph_json = json_graph.node_link_data(graph_netx) #dot_graph)
filename = dotFile[:dotFile.rfind('.')]
json.dump(graph_json,
open(jsonFolderPath + '/' + filename + '.json', 'w'),
indent=2)
print(filename + '.json converted')
counter += 1
with open(dataFileName, 'w') as jsonFile:
data = {}
jsonFiles = filesWithExtensions(jsonFolderPath, ['.json'])
while dataFileName in jsonFiles:
def buildCFGDepedcyInflueceDict(cfgPath, filePathList):
'''build cfgdependency and cfg inlunce for cur code
:param cfgPath: path to icfg.dot
:param filePathList: the source code path
:return:
:first: {nodeid:set(its control dependent nodeids)}
:second: {line:set{its control dependent lines}}
:third: {nodeid:set(its control influent nodeids)}
:forth: {line:set(its control influent lines)}
'''
CFGs = read_dot(cfgPath)
funcDict, lineToEntry = getCFGFuncEntryExit(CFGs, filePathList)
nodeToLineDict = buildCFGNodeToLineDict(CFGs, filePathList)
CFGs = preProcessCFG(CFGs, funcDict, filePathList, lineToEntry,
nodeToLineDict)
CFGsR = CFGs.reverse(True)
CFGDepdcyDict = dict()
CFGInfluenceDict = dict()
for funName in funcDict:
exitID = funcDict[funName]["exit"]
entryID = funcDict[funName]["entry"]
pdomTree = buildPdomTree(entryID, exitID, funName, CFGsR)
PdomPtoChildDict = buildPdomPtoChildDict(pdomTree, CFGsR)
CFGs.add_edges_from([(funName, entryID), (funName, exitID)])
CFGDepdcyDict, CFGInfluenceDict = appendCFGDepdcyDict(
CFGDepdcyDict, CFGInfluenceDict, CFGs, PdomPtoChildDict, pdomTree)
CFGLineDepdcyDict = dict()
CFGLineInflueceDict = dict()
for key in CFGInfluenceDict:
if key in nodeToLineDict:
lineKey = nodeToLineDict[key]
if lineKey not in CFGLineInflueceDict:
CFGLineInflueceDict[lineKey] = set()
for cfif in list(CFGInfluenceDict[key]):
if cfif in nodeToLineDict:
CFGLineInflueceDict[lineKey].add(nodeToLineDict[cfif])
if cfif in funcDict:
CFGLineInflueceDict[lineKey].add(
funcDict[cfif]["entryLine"])
elif key in funcDict:
lineKey = funcDict[key]["entryLine"]
if lineKey not in CFGLineInflueceDict:
CFGLineInflueceDict[lineKey] = set()
for cfif in list(CFGInfluenceDict[key]):
if cfif in nodeToLineDict:
CFGLineInflueceDict[lineKey].add(nodeToLineDict[cfif])
if cfif in funcDict:
CFGLineInflueceDict[lineKey].add(
funcDict[cfif]["entryLine"])
for key in CFGDepdcyDict:
if key in nodeToLineDict:
lineKey = nodeToLineDict[key]
if lineKey not in CFGLineDepdcyDict:
CFGLineDepdcyDict[lineKey] = set()
for cfdp in list(CFGDepdcyDict[key]):
if cfdp in nodeToLineDict:
CFGLineDepdcyDict[lineKey].add(nodeToLineDict[cfdp])
if cfdp in funcDict:
CFGLineDepdcyDict[lineKey].add(funcDict[cfdp]["entryLine"])
elif key in funcDict:
lineKey = funcDict[key]["entryLine"]
if lineKey not in CFGLineDepdcyDict:
CFGLineDepdcyDict[lineKey] = set()
for cfdp in list(CFGDepdcyDict[key]):
if cfdp in nodeToLineDict:
CFGLineDepdcyDict[lineKey].add(nodeToLineDict[cfdp])
if cfdp in funcDict:
CFGLineDepdcyDict[lineKey].add(funcDict[cfdp]["entryLine"])
return CFGDepdcyDict, CFGLineDepdcyDict, CFGInfluenceDict, CFGLineInflueceDict
def buildCodeGadgetList(doneLines, svfgPath, cfgPath, sensiAPIPath,
filePathList, callgPath):
'''
:param doneLines:
:param svfgPath:
:param cfgPath:
:param sensiAPIPath:
:param filePathList:
:param callgPath:
:return:
'''
SVFG = read_dot(svfgPath) # SVFG in networkx format
CFGs = read_dot(cfgPath) # CFGs in networkx format
CallG = read_dot(callgPath) # CallG in networkx format
print("start - getting callgraph...")
calleeToCallerDict = buildCallGraphDict(
CFGs, CallG, filePathList) # {callee line:set(caller line)}
print("end - getting callgraph...")
CFGsNodeToLineDict = buildCFGNodeToLineDict(
CFGs, filePathList) # for cfg:{nodeid:line}
entryDict = getCFGsEntryLineToNode(
CFGs, filePathList) # for cfg entry nodes:{line:nodeid}
for key in entryDict: # add entry nodeid to line info
CFGsNodeToLineDict[entryDict[key]] = key
print("start - extracting apis...")
apiLines = extractAPILines(doneLines, CFGs, sensiAPIPath,
CFGsNodeToLineDict)
# apiLines = extractOperatorLines(doneLines, operatorLineDir, filePathList)
print("end - extracting apis...")
if (apiLines == list()):
return dict()
# =====================================#
# build program dependence graph (PDG) #
# =====================================#
print("start - building PDG...")
PDG = nx.DiGraph()
control_edges = list()
print("start - building CFGLineDepdcy...")
CFGDepdcy, CFGLineDepdcy, CFGInfluence, CFGLineInfluence = buildCFGDepedcyInflueceDict(
cfgPath, filePathList) # {line:its cfg-dependency-line}
for line in CFGLineDepdcy:
control_dependent_line_set = CFGLineDepdcy[line]
for control_dependent_line in control_dependent_line_set:
control_edges.append((control_dependent_line, line, {"c/d": "c"}))
PDG.add_edges_from(control_edges)
print("end - building CFGLineDepdcy...")
print("start - building VFGLineDepdcy...")
SVFGnodeToLineDict = buildSVFGNodeToLineDict(SVFG, filePathList)
value_edges = list()
for n in SVFG._pred:
if n not in SVFGnodeToLineDict:
continue
cur_line = SVFGnodeToLineDict[n]
pred_lines = set()
visited_nodeid = set()
end = n.find(":s")
if end != -1:
n = n[:end]
visited_nodeid.add(n)
dfs_pred_lines(True, n, SVFGnodeToLineDict, SVFG._pred, pred_lines,
visited_nodeid)
for pred_line in pred_lines:
if pred_line != cur_line:
value_edges.append((pred_line, cur_line, {"c/d": "d"}))
PDG.add_edges_from(value_edges)
print("end - building VFGLineDepdcy...")
print("end - building PDG...")
codeGadgetLinesDict = dict() # {api:cdg}
for apiLine in apiLines:
print("start - processing apiline{}:...".format(apiLine))
sliced_lines = set()
# backward traversal
bqueue = list()
visited = set()
bqueue.append(apiLine)
visited.add(apiLine)
while bqueue:
fro = bqueue.pop(0)
sliced_lines.add(fro)
if fro in PDG._pred:
for pred in PDG._pred[fro]:
if pred not in visited:
visited.add(pred)
bqueue.append(pred)
# forward traversal
fqueue = list()
visited = set()
fqueue.append(apiLine)
visited.add(apiLine)
while fqueue:
fro = fqueue.pop(0)
sliced_lines.add(fro)
if fro in PDG._succ:
for succ in PDG._succ[fro]:
if succ not in visited:
visited.add(succ)
fqueue.append(succ)
print("end - processing apiline{}:...".format(apiLine))
entryToLinesDict = dict()
for relatedLine in sliced_lines:
entry = locateLineEntry(relatedLine, list(entryDict.keys()))
if entry not in entryToLinesDict:
entryToLinesDict[entry] = list()
entryToLinesDict[entry].append(relatedLine)
for entry in entryToLinesDict:
entryToLinesDict[entry] = sorted(entryToLinesDict[entry])
callChianList = getCallChainList(calleeToCallerDict,
entryToLinesDict.keys())
maxLen = 0
lenToCallChainDict = dict()
for callChian in callChianList:
if len(callChian) > maxLen:
maxLen = len(callChian)
if len(callChian) not in lenToCallChainDict:
lenToCallChainDict[len(callChian)] = list()
lenToCallChainDict[len(callChian)].append(callChian)
finalCallChain = list()
finalCallChain.extend(lenToCallChainDict[maxLen][0])
for callChain in lenToCallChainDict[maxLen]:
for entry in callChain:
if entry not in set(finalCallChain):
finalCallChain.append(entry)
for lenn in lenToCallChainDict:
if lenn != maxLen:
for callChain in lenToCallChainDict[lenn]:
for entry in callChain:
if entry not in set(finalCallChain):
finalCallChain.append(entry)
codeGadgetLines = list()
for entry in finalCallChain:
codeGadgetLines.extend(entryToLinesDict[entry])
codeGadgetLinesDict[apiLine] = codeGadgetLines
return codeGadgetLinesDict
def __call__(self, graph_data: str,
number_input_symbols: int = None,
graph_data_format: str = 'dot_string') -> FDFA:
"""
Returns an initialized FDFA instance given the graph_data
graph_data and graph_data_format must match
:param graph_data: The string containing graph data.
Could be a filename or just the raw
data
:param number_input_symbols: The number of input symbols to the
FDFA needed to compute the correct
frequency flows in the case of
cycles.
Only really optional when using a
graph_data_format
that already has this information.
:param graph_data_format: The graph data file format.
{'dot_file', 'dot_string',
'learning_interface'}
:returns: instance of an initialized FDFA object
:raises ValueError: checks if graph_data and
graph_data_format have a compatible
data loader.
:raises ValueError: checks, based on graph_data_format,
whether it is legal to not specify
the number_input_symbols.
"""
has_number_input_symbols = number_input_symbols is not None
if graph_data_format == 'dot_string':
graph = nx_agraph.from_agraph(pygraphviz.AGraph(string=graph_data))
elif graph_data_format == 'dot_file':
graph = read_dot(graph_data)
elif graph_data_format == 'learning_interface':
learning_interface = graph_data
graph = read_dot(learning_interface.learned_model_filepath)
number_input_symbols = learning_interface.num_training_examples
has_number_input_symbols = True
else:
msg = 'graph_data_format ({}) must be one of: "dot_file", ' + \
'"dot_string"'.format(graph_data_format)
raise ValueError(msg)
if not has_number_input_symbols:
msg = f'must provide the number_input_symbols to load a FDFA'
raise ValueError(msg)
# these are not things that are a part of flexfringe's automaton
# data model, so give them default values
final_transition_sym = DEFAULT_FINAL_TRANS_SYMBOL
empty_transition_sym = DEFAULT_EMPTY_TRANS_SYMBOL
config_data = FDFA.load_flexfringe_data(graph,
number_input_symbols,
final_transition_sym,
empty_transition_sym)
config_data['final_transition_sym'] = final_transition_sym
config_data['empty_transition_sym'] = empty_transition_sym
nodes_have_changed = (self.nodes != config_data['nodes'])
edges_have_changed = (self.edges != config_data['edges'])
no_instance_loaded_yet = (self._instance is None)
if no_instance_loaded_yet or nodes_have_changed or edges_have_changed:
# saving these so we can just return initialized instances if the
# underlying data has not changed
self.nodes = config_data['nodes']
self.edges = config_data['edges']
self._instance = FDFA(**config_data)
return self._instance
def compare_schema(file_prev, file_now, file_result):
'''
Input - graph file in dot format
Output - file in png format with diff
'''
logger = logging.getLogger('compare_schema')
logger.debug('Try read dot files with network schema')
logger.info('Read file {}'.format(file_prev))
try:
graph_prev = read_dot(file_prev)
except IOError:
logger.error('Fail load schema')
return
logger.info('Read file {}'.format(file_now))
try:
graph_now = read_dot(file_now)
except IOError:
logger.error('Fail to load schema')
return
fig = plt.subplots(figsize=(14, 9))
if nx.is_isomorphic(graph_prev, graph_now):
logger.info('Schema is equal')
else:
logger.info('Changes were exists')
graph_sum = nx.compose(graph_prev, graph_now)
pos = nx.spring_layout(graph_sum)
for node in graph_sum.nodes():
color = '#00CCCC'
if node not in graph_prev.nodes() or node not in graph_now.nodes():
color = 'red'
nx.draw_networkx_nodes(graph_sum,
pos,
nodelist=[node],
node_size=1500,
node_color=color)
for begin, end in graph_sum.edges():
color = '#00CCCC'
if (begin, end) not in graph_prev.edges() or (
begin, end) not in graph_now.edges():
color = 'red'
el_b = el_e = dict()
nx.draw_networkx_edges(graph_sum,
pos,
edgelist=[([begin, end])],
edge_color=color)
# Вот здесь прибито гвоздями
# ПОчему при импорте есть проблема
# Линки мультидиграф должны быть tuple вида
# (источник.назначение,номер_линка)
# после импорта, номер линка уезжает в аттрибут edge-а
el_b[(begin, end)] = graph_sum[begin][end]['0']['interfaces']
el_e[(begin, end)] = graph_sum[begin][end]['0']['interfaces']
nx.draw_networkx_edge_labels(graph_sum,
pos,
edge_labels=el_b,
label_pos=0.5,
font_size=8,
font_color=color)
nodes_lab = dict()
for host in graph_sum.nodes():
if not graph_sum.nodes[host]['cap'] == '':
nodes_lab[host] = host + '\n' + graph_sum.nodes[host]['cap']
else:
nodes_lab[host] = host
nx.draw_networkx_labels(graph_sum, pos, font_size=12)
logger.info('Save diff schema to {}'.format(file_result))
plt.savefig(file_result)
def initial_computation(reaction,
result_dir,
tree_size,
threshold,
reference,
methods,
mode='enumeration'):
smarts_dict = {}
G = None
prev = ""
f = open(os.path.join(result_dir, "pt.txt"), 'r')
for i in range(tree_size):
l = f.readline()
if l == '':
break
prev = l
c = 0
f.close()
#l = linecache.getline(result_dir+"pt.txt", n)
f = open("./tmp.dot", 'w')
f.write(prev)
f.close()
#os.system("dot -Tpng ./tmp.dot -o nako.png")
G = read_dot("./tmp.dot")
linecache.clearcache()
mol_file = open(os.path.join(result_dir, "finalResult.txt"))
id_to_string = {}
for l in mol_file:
if l.startswith("digraph") or l.startswith("l") or l.startswith(
"in") or l.startswith("end") or l.startswith("$.digraph"):
continue
else:
try:
l = l.rstrip("\n").split(",")
node_id = l[0]
string = ''.join(l[1:])
if len(string) == 0:
continue
id_to_string[node_id] = string
except:
pass
route_manager = RouteManager(G, id_to_string, result_dir, reference,
methods)
print(route_manager.target)
start = time.time()
#route_manager.extract_route()
if mode == 'enumeration':
route_manager.enumeration()
elif mode == 'sampling':
route_manager.sampling()
end = time.time()
print("Elapsed time for {}:{} sec.".format(mode, end - start))
#print("the number of route:", len(route_manager.route_list))
#start = time.time()
#route_manager.add_name()
with open(
os.path.join(result_dir,
"route_manager_" + str(tree_size) + ".pickle"),
"wb") as f1:
pickle.dump(route_manager, f1)
return route_manager
