def prepare_data(date):
#print('analysing graph')
g = read_dot(os.path.join(FOLDER + date, date + '.dot'))
#g = read_dot(date+'.dot')
#pickle.dump(g, open('g.pickle', 'wb'))
#g = pickle.load(open('g.pickle', 'rb'))
subs = sub_graphs(g)
ids_by_number_of_matched_files = rank_ids_amount(subs)
dirsdict = getDirsDict()
jsons = loadJson(date)
#jsons = pickle.load(open('jsons.pickle', 'rb'))
#pickle.dump(jsons, open('jsons.pickle', 'wb'))
#json.dump(jsons, open('jsons.json', 'w'))
#jsons = json.load(open('jsons.json'))
lengths = {}
for i in get_all_ids(g):
lengths[i] = get_lengths(jsons, i, dirsdict)
#json.dump(lengths, open('lengths.json', 'w'))
#lengths = json.load(open('lengths.json'))
ids_by_length = rank_ids_length(lengths)
# add unmatched to subgraph:
for n in nx.isolates(g):
subs.append({n: []})
return subs, ids_by_length, ids_by_number_of_matched_files, lengths, jsons
def prepare_data(filename):
N = parse(filename)
# G - initial graph
G = read_dot("data/graph.dot")
# read productions to list of tuples [(Left-side, Right-side)]
productions = get_prod_func(N)
# read rules to dict
rules = get_rules_func(N)
return N, G, productions, rules
def get_prod_func(N):
prod = [[0 for _ in range(2)] for _ in range(N)]
i = 0
with open("data/left_sides.txt", "r") as left:
while i < N:
for label in left:
if label not in "\n":
prod[i][0] = label.strip('\n')
filename = "data/P%s.dot" % str(i + 1)
P = read_dot(filename)
prod[i][1] = P
i += 1
return prod
def draw_graph_from_dot(dot_fpath):
G = read_dot(dot_fpath)
plt.figure(figsize=(10, 10), facecolor="w", frameon=False)
pos = nx.spring_layout(G)
offset = .05
pos_labels = {}
keys = pos.keys()
for key in keys:
x, y = pos[key]
pos_labels[key] = (x, y + offset)
node_colors = [G.node[key]['fillcolor'] for key in G.node]
nx.draw_networkx_nodes(G,
pos,
node_size=1200,
node_shape='o',
node_color=node_colors)
nx.draw_networkx_edges(G, pos, width=2, edge_color='b')
nx.draw_networkx_labels(G, pos_labels, fontsize=2)
return
def ACTest():
G = nxdr.read_dot(f'dag3/random8.dot')
TC = nx.transitive_closure(G) # Complexity of TC
nxdr.write_dot(TC, f'dump.dot')
antichains_stacks = [([], list(reversed(list(nx.topological_sort(G)))))]
pprint.pprint(antichains_stacks)
i = 0
while antichains_stacks:
(antichain, stack) = antichains_stacks.pop()
print(f'i({i}):AC:{antichain},stack:{stack}')
#yield antichain
j = 0
while stack:
x = stack.pop()
print(f'j({j}),x:{x},stack:{stack},TC[{x}]:{TC[x]}')
new_antichain = antichain + [x]
new_stack = [
t for t in stack if not ((t in TC[x]) or (x in TC[t]))
]
antichains_stacks.append((new_antichain, new_stack))
j = j + 1
i = i + 1
def __init__(self, dotFile):
self.G = nxdr.read_dot(
dotFile
) # Read a dot file, it is assumed nodes are numbered as str(integers)
if not nx.is_directed_acyclic_graph(self.G):
raise IOError(f'The graph is not acyclic')
# self.originalSavedAlready = False
N = self.G.nodes(data=True)
for n in N:
n[1].update(children=[])
n[1].update(rank=-1)
n[1].update(start=-1)
n[1].update(finish=-1)
n[1].update(stack=1)
n[1].update(alloc=-1)
self.origG = nxdag.transitive_closure_dag(
copy.deepcopy(self.G)
) # This stores the original ATG, augments the DAG with its transitive closure.
self.valid = True
self.isScheduled = False
def gv_dot_iterator_colorize(infile, outfile):
if os.path.exists(outfile):
raise ValueError("Output file already exists")
dot_graph = read_dot(infile)
for n in dot_graph.nodes():
if dot_graph.node[n]['label'].startswith('{it_'):
dot_graph.node[n]['style'] = 'filled'
dot_graph.node[n]['fillcolor'] = 'aquamarine'
elif dot_graph.node[n]['label'].startswith(
'{pd_mix_') or dot_graph.node[n]['label'].startswith(
'{it_mix_'):
dot_graph.node[n]['style'] = 'filled'
dot_graph.node[n]['fillcolor'] = 'burlywood'
elif dot_graph.node[n]['label'].startswith('{pd_'):
dot_graph.node[n]['style'] = 'filled'
dot_graph.node[n]['fillcolor'] = 'coral'
else:
dot_graph.node[n]['style'] = 'filled'
dot_graph.node[n]['fillcolor'] = 'gray'
write_dot(dot_graph, outfile)
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
import re
import sys
import networkx as nx
from networkx.drawing.nx_agraph import read_dot
regex = r'^{([a-zA-Z0-9_]+).*}'
G = read_dot(sys.argv[1])
for u, v in G.edges():
source_se = re.search(regex, G._node[u]['label'])
target_se = re.search(regex, G._node[v]['label'])
print(source_se.group(1) + ":" + target_se.group(1))
f = "eda-script-67.py"
f = "practical-model-evaluation-day-1.py" # AttributeError: 'FindDependencies' object has no attribute '_bag'
f = "how-to-attack-a-machine-learning-model.py" # AttributeError: 'Attribute' object has no attribute 'id'
f = "very-simple-pytorch-training-0-59.py" # CLASS support - still to finish
#f="keras-u-net-starter-lb-0-277.py" # TODO AttributeError: 'Call' object has no attribute 'id'
src = open(indir + f, "r").read()
# print(src)
collector = DJ.FindDependencies(f[:-3])
collector.collect(src)
s = collector.getStringCollected()
# Save
o = open(outdir + f[:-2] + "digraph", "w")
o.write(s)
# # Save
# #
g = ag.read_dot(outdir + f[:-2] + "digraph")
######
n = f[:-3]
# def toRDF(name, digraph):
# n = name
# g = digraph
# DJ = Namespace("http://purl.org/dj/")
# K = Namespace("http://purl.org/dj/kaggle/")
# L = Namespace("http://purl.org/dj/python/lib/")
# notebook = URIRef(str(K) + n)
# Loc = Namespace(str(K) + str(n) + "#")
# #print(notebook)
# rdfg = Graph()
# rdfg.bind("rdf", RDF)
from collections import Counter
from json import load
import networkx as nx
from networkx.drawing.nx_agraph import read_dot
from networkx.relabel import convert_node_labels_to_integers
from sys import argv
GRAPH = argv[1]
OUT = argv[2]
G = read_dot(GRAPH)
with open(OUT+'.labels', 'w') as out:
for n, label in sorted(G.nodes.data('label'), key=lambda x: int(x[0])):
out.write(f'{label}\n')
with open(OUT+'.mtx', 'w') as out:
out.write('%%MatrixMarket matrix coordinate pattern symmetric\n')
num_nodes = G.number_of_nodes()
num_edges = G.number_of_edges()
out.write(f'{num_nodes} {num_nodes} {num_edges}\n')
for source, target, weight in sorted(G.edges.data('weight'), key=lambda x: int(x[0])):
out.write(f'{source} {target} {weight}\n')
#!/usr/bin/python
import json
import networkx as nx
from networkx.readwrite import json_graph
from networkx.drawing.nx_agraph import read_dot
#G=nx.grid_graph(dim=[6,6,8])
#G=nx.hypercube_graph(6)
#G=nx.grid_2d_graph(12,14)
#G=nx.barabasi_albert_graph(1000,1)
G=read_dot("inkscape3.dot")
# this d3 example uses the name attribute for the mouse-hover value,
# so add a name to each node
# unidirectional graphs only have G.neighbors(n)
for n in G:
G.node[n]['name'] = n
G.node[n]['deps'] = len(G.successors(n))
G.node[n]['rdeps'] = len(G.predecessors(n))
# write json formatted data
d = json_graph.node_link_data(G) # node-link format to serialize
# write json
f = open('js/apt.js','w')
f.write('dep_json = `{"pkg":')
json.dump(d, f)
f.write('}`;')
GRAPH = 'datasets/lastfm/lastfmw_original.dot'
OUT = 'datasets/lastfm/network.dot'
TOPN = 30
def get_nodes(graph):
for (n, degree) in graph.degree():
data = {
'label': graph.nodes[n]['label'].replace('\\n', ' '),
'weight': graph.nodes[n]['weight']
}
yield n, data
def get_edges(graph):
for n1, n2, weight in graph.edges.data('weights'):
data = {'weight': float(weight)}
yield n1, n2, data
source_graph = read_dot(GRAPH)
G = nx.Graph()
for n, data in get_nodes(source_graph):
G.add_node(n, **data)
for n1, n2, data in get_edges(source_graph):
G.add_edge(n1, n2, **data)
print('Writing Network... ', G.number_of_nodes(), G.number_of_edges())
write_dot(G, OUT)
def build(name):
path = f'{name}-new.dot'
G = read_dot(path)
d = {}
for n in G.nodes:
if not (n in d):
d[n] = {
'num_units': min(10, get_num_units(G, n)),
'succ_str': list(G.successors(n))[:10]
}
g = Digraph('g', filename=f'{name}-build.gv')
g.graph_attr.update(splines="false", nodesep='1', ranksep='2')
g.attr(arrowShape="none")
for e in d:
with g.subgraph(name=f'cluster_{e}') as c:
c.attr(color="white")
for i in range(d[e]['num_units']):
name_d = f'{e}_{i}'
if 'Input' in e:
color = "#33cc33"
c.node(name_d,
shape="circle",
style="filled",
color=color,
fontcolor=color)
elif 'Dropout' in e:
color = "#ffcc00"
c.node(name_d,
shape="rect",
style="filled",
color=color,
fontcolor=color)
elif 'Concatenate' in e:
color = "#993300"
c.node(name_d,
shape="rect",
style="filled",
color=color,
fontcolor=color)
elif len(d[e]['succ_str']) == 0: # OUTPUT
color = "#ff0000"
c.node(name_d,
shape="circle",
style="filled",
color=color,
fontcolor=color)
else:
color = "#3498db"
c.node(name_d,
shape="circle",
style="filled",
color=color,
fontcolor=color)
for s in d[e]['succ_str']:
if 'Dense' in s:
for j in range(d[s]['num_units']):
name_a = f'{s}_{j}'
g.edge(name_d, name_a)
else:
for j in range(d[s]['num_units']):
name_a = f'{s}_{j}'
g.edge(name_d, name_a)
g.view()
#!/usr/bin/python
import networkx as nx
from networkx.drawing.nx_agraph import read_dot
from networkx.drawing.nx_agraph import write_dot
from networkx.readwrite import json_graph
#G = read_dot("all_nobreak.dot")
g = read_dot("inkscape-mergecycl.dot")
cycles = list(nx.simple_cycles(g))
print cycles
for n1 in g.nodes_iter():
if g.has_edge(n1, n1):
g.remove_edge(n1, n1)
for n2 in g.successors(n1):
for n3 in g.successors(n2):
for n4 in nx.dfs_preorder_nodes(g, n3):
if g.has_edge(n1, n4):
g.remove_edge(n1, n4)
#nx.write_graphml(g, "inkscape-reduced.graphml")
write_dot(g,"inkscape-reduced.dot")
nx.write_gml(g,"inkscape-reduced.gml")
g = nx.convert_node_labels_to_integers(g)
data = json_graph.node_link_data(g)
print data
#import json
#serial = json.dumps(data)
#print serial
def parseGraph(self, dotFile):
topologyGraph = nx_dot.read_dot(dotFile)
return self.parseTopologyGraph(topologyGraph)
import json as J
from os import listdir
from os.path import isfile, join
from graphviz import Source
import networkx.drawing, networkx.drawing.nx_agraph as ag
indir = "graphs/"
# srcdir = "sources/"
outdir = "rdf/"
files = [f for f in listdir(indir) if isfile(join(indir, f))]
for f in files:
with open(indir + f) as notebook:
print("Processing: {0}".format(f))
try:
g = ag.read_dot(indir + f)
n = f[:-7]
rdfg = DJ.toRDF(n[:-1], g)
print("Writing to: {0}".format(outdir + f[:-7] + "ttl"))
rdfg.serialize(destination=outdir + f[:-7] + "ttl", format="ttl")
except Exception as err:
print("ERROR Exception: {0} [{1}]".format(err, f))
#print("ERROR Some error occurred with: {0}".format(f))
# try:
# # Save
#
# rdfg.write(outdir + f[:-5] + "ttl", format="nt")
#
# except AttributeError as err:
# print("AttributeError: {0} [{1}]".format(err, f))
def nx_graph_from_dotfile(filename: str) -> nx.DiGraph:
""" Get a networkx graph from a DOT file, and reverse the edges. """
return read_dot(filename).reverse()
import numpy
import powerlaw
import networkx as nx
from networkx.drawing.nx_agraph import read_dot
from sys import argv
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
if __name__ == '__main__':
if len(argv) < 2 or not os.path.isfile(argv[1]):
print('Error: first argument is no file')
exit()
g = nx.Graph(read_dot(argv[1]))
degrees = [len(nbrs) for n, nbrs in g.adj.items()]
print('nodes ', len(g))
print('edges ', len(g.edges()))
print('max degree ', max(degrees))
# print('diameter ', nx.diameter(g))
# print('average clustering ', nx.average_clustering(g))
numpy.seterr(divide='ignore', invalid='ignore')
fit = powerlaw.Fit(degrees, discrete=True, verbose=False)
print('power law exponent ', fit.alpha)
print('xmin ', fit.xmin)
print('%nodes under xmin ',
100 * sum(i < fit.xmin for i in degrees) / float(len(g)))
print('KS distance ', fit.power_law.KS())
print('compare with exponential distribution ... ')
comp = fit.distribution_compare('power_law', 'exponential')
