def test_quoting(self):
g = pydot.Dot()
g.add_node(pydot.Node("test", label=string.printable))
#print g.to_string()
data = g.create(format='jpe')
self.assertEqual(len(data) > 0, True)
def to_png_with_pydot(self, filename):
import pydot
graph = pydot.Dot(graph_type='digraph', rankdir="LR")
self._to_pydot(graph)
graph.write_png(filename)
def layout(self, type_):
if pygraphviz:
G = pygraphviz.AGraph(strict=False, directed=True)
if type_ == 'dot LR':
G.graph_attr['rankdir'] = 'LR'
type_ = type_.split()[0]
G.graph_attr['ranksep'] = '0.125'
elif pydot:
G = pydot.Dot('graphname', graph_type='digraph')
if type_ == 'dot LR':
G.set_layout('dot')
G.set('rankdir', 'LR')
else:
G.set_layout(type_)
G.set('ranksep', '0.125')
for from_, to in self.link.values():
if pygraphviz:
G.add_edge(
(from_, from_.gnx),
(to, to.gnx),
)
elif pydot:
G.add_edge(pydot.Edge(from_.gnx, to.gnx))
for i in self.nodeItem:
if pygraphviz:
G.add_node((i, i.gnx))
elif pydot:
gnode = pydot.Node(i.gnx)
# rect = self.nodeItem[i].boundingRect()
G.add_node(gnode)
for child in i.children:
key = (i, child)
if key not in self.hierarchyLinkItem or child not in self.nodeItem:
continue
G.add_edge(pydot.Edge(i.gnx, child.gnx))
if pygraphviz:
G.layout(prog=type_)
elif pydot:
tempName = tempfile.NamedTemporaryFile(dir=tempfile.gettempdir(),
delete=False)
G.write_dot(tempName.name)
G = pydot.graph_from_dot_file(tempName.name)
for i in self.nodeItem:
if pygraphviz:
gn = G.get_node((i, i.gnx))
x, y = map(float, gn.attr['pos'].split(','))
i.u['_bklnk']['x'] = x
i.u['_bklnk']['y'] = -y
self.nodeItem[i].setPos(x, -y)
self.nodeItem[i].do_update()
elif pydot:
lst = G.get_node(''.join(['"', i.gnx, '"']))
if lst:
x, y = map(float, lst[0].get_pos().strip('"').split(','))
i.u['_bklnk']['x'] = x
i.u['_bklnk']['y'] = -y
self.nodeItem[i].setPos(x, -y)
self.nodeItem[i].do_update()
if pydot:
x, y, width, height = map(float, G.get_bb().strip('"').split(','))
self.ui.canvasView.setSceneRect(
self.ui.canvas.sceneRect().adjusted(x, y, width, height))
self.do_update(adjust=False)
self.center_graph()
names = [
'1388714212', '1388717228', '1388725781', '1388732850', '1388735678',
'1388807221', '1389315179'
]
vtot = {}
for name in names:
print "Running for megadump", name
f = open('../../.galileo/53A0ED39B4DB/dump-' + name + '.txt', 'rb')
v = ngrams_vector(list(megadump_parser(f)), 2)
vtot = vector_add(v, vtot)
graph = pydot.Dot(graph_type='digraph')
for a, b in v:
if a == "00" or b == "00":
continue
if v[a, b] > 2:
e = pydot.Edge(a, b, label=str(v[a, b]))
graph.add_edge(e)
# ok, we are set, let's save our graph into a file
graph.write_png('2gram-vis-' + name + '.png')
print "Running for all."
graph = pydot.Dot(graph_type='digraph')
def supported_plot_formats():
"""return automatically all `pydot` extensions"""
import pydot
return [".%s" % f for f in pydot.Dot().formats]
def dfs(initial_state):
graph = pydot.Dot(graph_type='digraph',
label="Missionaries and Cannibals State Space",
fontsize="20",
color="red",
fontcolor="blue",
style="filled",
fillcolor="black")
start_node = Node(initial_state, None, None, 0) #Starting depth=0
if start_node.goal_test():
return start_node.find_solution()
stack = LifoQueue()
stack.put(start_node)
explored = []
killed = []
print("\n\nDepth First Search Implementation\n")
print("The starting node is " + str(start_node.state) +
" with depth=%d" % start_node.depth)
while not (stack.empty()):
node = stack.get()
print("\nThe node to be expanded is " +
str(node.state)) #print(" with depth="+str(node.depth))
explored.append(node.state)
graph.add_node(node.graph_node)
if node.parent:
diff = np.subtract(node.parent.state, node.state)
if node.parent.state[2] == 1: # 1
diff[0], diff[1] = -diff[0], -diff[1] #- removed
graph.add_edge(
pydot.Edge(node.parent.graph_node,
node.graph_node,
label=str(diff)))
children = node.generate_child()
if not node.is_killed():
print("Its children nodes are\n", end="")
for child in children:
if child.state not in explored:
print(str(child.state) + "\tdepth=%d" % child.depth)
if child.goal_test():
print("is the goal state\n")
graph.add_node(child.graph_node)
diff = np.subtract(node.parent.state, node.state)
if node.parent.state[2] == 1: # 1
diff[0], diff[1] = -diff[0], -diff[1] #- removed
graph.add_edge(
pydot.Edge(child.parent.graph_node,
child.graph_node,
label=str(diff)))
# colour all leaves blue
leafs = {n.get_name(): True for n in graph.get_nodes()}
for e in graph.get_edge_list():
leafs[e.get_source()] = False
for leaf in leafs:
if leafs[leaf] and str(leaf) not in killed and str(
leaf) != "\"[0, 0, 1]\"": #[0,0,1]
node = pydot.Node(leaf,
style="filled",
fillcolor="blue")
graph.add_node(node)
draw_legend(graph)
graph.write_png('solution_dfs.png')
return child.find_solution()
if child.is_valid():
stack.put(child)
explored.append(child.state)
else:
print("This node is killed")
killed.append("\"" + str(node.state) + "\"")
return
def main():
df = pd.read_csv('data/interim/situ_topics_kmeans_tfidf.csv',
converters={
'sop': eval,
'situ_lst': eval
})
df['sop_graph'] = df['sop'].apply(lambda x: '\n'.join(x))
df = df[[
'type', 'juri', 'role', 'filename', 'situation', 'sop_graph',
'cluster', 'situ_lst', 'sop'
]]
df2 = df.drop(columns=['role', 'filename', 'juri'])
for i, df in df2.groupby('cluster'):
df3 = df.copy().reset_index(drop=True)
cluster = int(df3.iloc[0]['cluster'])
# convert situation texts to tf-idf vectors
situ_tfidf = TfidfVectorizer().fit(df3['situ_lst'].apply(
situ.preprocess))
situ_tfidf_mtx = situ_tfidf.transform(df3['situation'])
# convert SOP texts to tf-idf vectors
sop_tfidf = TfidfVectorizer().fit(df3['sop'].apply(situ.preprocess))
sop_tfidf_mtx = sop_tfidf.transform(df3['sop_graph'])
# obtain similarity matrix
situ_similarity = cosine_similarity(situ_tfidf_mtx)
sop_similarity = cosine_similarity(sop_tfidf_mtx)
# add a place holder column for similar nodes
df3['situ_group'] = df3['situation']
df3['sop_group'] = df3['sop_graph']
# The threshold for cosine similarity
similarity_threshold = 0.45
situ_grouped = set()
for i in range(situ_similarity.shape[0]):
if i in situ_grouped:
continue
same_group = np.where(
situ_similarity[i] >= similarity_threshold)[0]
situ_grouped = situ_grouped | set(same_group)
for j in same_group:
df3.at[j, 'situ_group'] = df3.iloc[i]['situation']
sop_grouped = set()
for i in range(sop_similarity.shape[0]):
if i in sop_grouped:
continue
same_group = np.where(sop_similarity[i] >= similarity_threshold)[0]
sop_grouped = sop_grouped | set(same_group)
for j in same_group:
df3.at[j, 'sop_group'] = df3.iloc[i]['sop_graph']
graph = pydot.Dot(graph_type='graph', concentrate=True, rankdir='LR')
df3 = df3[['type', 'situ_group', 'sop_group']]
nrows, _ = df3.shape
for i in range(nrows):
row = df3.iloc[i]
for p, c in zip(row[:-1], row[1:]):
edge = pydot.Edge(p, c)
graph.add_edge(edge)
graph.write_png(f"img/flowcharts/situ_cluster_{cluster}.png")
subprocess.call(['chmod', '+x', 'down.sh'])
# Dump the mapping from nodes to SSH ports
fout = open('demo.map', 'w')
for vmname in sorted(vms):
fout.write('%s %d\n' % (vmname, args.base_port + vms[vmname]['id']))
fout.close()
if args.graphviz:
try:
import pydot
colors = ['red', 'green', 'blue', 'orange', 'yellow']
fcolors = ['black', 'black', 'white', 'black', 'black']
gvizg = pydot.Dot(graph_type='graph')
i = 0
for dif in difs:
for vmname in dif_graphs[dif]:
node = pydot.Node(dif + vmname,
label="%s(%s)" % (vmname, dif),
style="filled",
fillcolor=colors[i],
fontcolor=fcolors[i])
gvizg.add_node(node)
for vmname in dif_graphs[dif]:
for (neigh, lower_dif) in dif_graphs[dif][vmname]:
if vmname > neigh:
# Use lexicographical filter to avoid duplicate edges
continue
def create(self, cr, uid, ids, datas, context={}):
pool = pooler.get_pool(cr.dbname)
wiz = pool.get("wiz.print.wkf").browse(cr, uid, ids[0])
graph = pydot.Dot(graph_type="digraph",
rankdir="TB",
graph_name="\"Workflow of %s\"" % wiz.model_id.name)
st_ids = pool.get("wkf.state").search(
cr, uid, [('model_id', '=', wiz.model_id.id)])
for st in pool.get("wkf.state").browse(cr, uid, st_ids):
n = pydot.Node(st.name)
graph.add_node(n)
dec_ids = pool.get("wkf.decision").search(
cr, uid, [('model_id', '=', wiz.model_id.id)])
for dec in pool.get("wkf.decision").browse(cr, uid, dec_ids):
if len(dec.choices) > 1:
n = pydot.Node(dec.name, shape="diamond", label="")
graph.add_node(n)
l = dec.name
l += "\\n[" + ",".join([d.role_id.name
for d in dec.deciders]) + "]"
e = pydot.Edge(dec.name,
dec.name,
color="transparent",
label='"%s"' % l)
graph.add_edge(e)
if not (dec.interface_type == "dialog"
and not dec.choice_button):
for cond in dec.conditions:
e = pydot.Edge(cond.state_from.name,
dec.name,
label="")
graph.add_edge(e)
i = 0
n = len(dec.choices)
for ch in dec.choices:
i += 1
if n == 2:
p = i == 1 and "sw" or "se"
elif n == 3:
p = i == 1 and "sw" or i == 2 and "s" or "se"
else:
p = "s"
if ch.state_to:
e = pydot.Edge(dec.name,
ch.state_to.name,
label=ch.name or "",
tailport=p)
else:
e = pydot.Edge(dec.name,
ch.next_decision_id.name,
label=ch.name or "",
tailport=p)
graph.add_edge(e)
elif len(dec.choices) == 1:
ch = dec.choices[0]
l = ch.name or ""
l += "\\n[" + ",".join([d.role_id.name
for d in dec.deciders]) + "]"
for cond in dec.conditions:
e = pydot.Edge(cond.state_from.name,
ch.state_to.name,
label=l)
graph.add_edge(e)
graph.write_ps("wkf.ps")
os.system("epstopdf wkf.ps > wkf.pdf")
pdf = file("wkf.pdf").read()
return (pdf, "pdf")
def pydotprint(fct, outfile=None,
compact=True, format='png', with_ids=False,
high_contrast=True, cond_highlight=None, colorCodes=None,
max_label_size=70, scan_graphs=False,
var_with_name_simple=False,
print_output_file=True,
assert_nb_all_strings=-1
):
"""
Print to a file (png format) the graph of a compiled theano function's ops.
:param fct: the theano fct returned by theano.function.
:param outfile: the output file where to put the graph.
:param compact: if True, will remove intermediate var that don't have name.
:param format: the file format of the output.
:param with_ids: Print the toposort index of the node in the node name.
and an index number in the variable ellipse.
:param high_contrast: if true, the color that describes the respective
node is filled with its corresponding color, instead of coloring
the border
:param colorCodes: dictionary with names of ops as keys and colors as
values
:param cond_highlight: Highlights a lazy if by sorrounding each of the 3
possible categories of ops with a border. The categories
are: ops that are on the left branch, ops that are on the
right branch, ops that are on both branches
As an alternative you can provide the node that represents
the lazy if
:param scan_graphs: if true it will plot the inner graph of each scan op
in files with the same name as the name given for the main
file to which the name of the scan op is concatenated and
the index in the toposort of the scan.
This index can be printed with the option with_ids.
:param var_with_name_simple: If true and a variable have a name,
we will print only the variable name.
Otherwise, we concatenate the type to the var name.
:param assert_nb_all_strings: Used for tests. If non-negative, assert that
the number of unique string nodes in the dot graph is equal to
this number. This is used in tests to verify that dot won't
merge Theano nodes.
In the graph, ellipses are Apply Nodes (the execution of an op)
and boxes are variables. If variables have names they are used as
text (if multiple vars have the same name, they will be merged in
the graph). Otherwise, if the variable is constant, we print its
value and finally we print the type + a unique number to prevent
multiple vars from being merged. We print the op of the apply in
the Apply box with a number that represents the toposort order of
application of those Apply. If an Apply has more than 1 input, we
label each edge between an input and the Apply node with the
input's index.
Green boxes are inputs variables to the graph,
blue boxes are outputs variables of the graph,
grey boxes are variables that are not outputs and are not used,
red ellipses are transfers from/to the gpu (ops with names GpuFromHost,
HostFromGpu).
"""
if colorCodes is None:
colorCodes = default_colorCodes
if outfile is None:
outfile = os.path.join(config.compiledir, 'theano.pydotprint.' +
config.device + '.' + format)
if isinstance(fct, Function):
mode = fct.maker.mode
profile = getattr(fct, "profile", None)
if (not isinstance(mode, ProfileMode)
or not fct in mode.profile_stats):
mode = None
fct_fgraph = fct.maker.fgraph
elif isinstance(fct, gof.FunctionGraph):
mode = None
profile = None
fct_fgraph = fct
else:
raise ValueError(('pydotprint expects as input a theano.function or '
'the FunctionGraph of a function!'), fct)
if not pydot_imported:
raise RuntimeError("Failed to import pydot. You must install pydot"
" for `pydotprint` to work.")
return
g = pd.Dot()
if cond_highlight is not None:
c1 = pd.Cluster('Left')
c2 = pd.Cluster('Right')
c3 = pd.Cluster('Middle')
cond = None
for node in fct_fgraph.toposort():
if (node.op.__class__.__name__ == 'IfElse'
and node.op.name == cond_highlight):
cond = node
if cond is None:
_logger.warn("pydotprint: cond_highlight is set but there is no"
" IfElse node in the graph")
cond_highlight = None
if cond_highlight is not None:
def recursive_pass(x, ls):
if not x.owner:
return ls
else:
ls += [x.owner]
for inp in x.inputs:
ls += recursive_pass(inp, ls)
return ls
left = set(recursive_pass(cond.inputs[1], []))
right = set(recursive_pass(cond.inputs[2], []))
middle = left.intersection(right)
left = left.difference(middle)
right = right.difference(middle)
middle = list(middle)
left = list(left)
right = list(right)
var_str = {}
all_strings = set()
def var_name(var):
if var in var_str:
return var_str[var]
if var.name is not None:
if var_with_name_simple:
varstr = var.name
else:
varstr = 'name=' + var.name + " " + str(var.type)
elif isinstance(var, gof.Constant):
dstr = 'val=' + str(numpy.asarray(var.data))
if '\n' in dstr:
dstr = dstr[:dstr.index('\n')]
varstr = '%s %s' % (dstr, str(var.type))
elif (var in input_update
and input_update[var].variable.name is not None):
if var_with_name_simple:
varstr = input_update[var].variable.name + " UPDATE"
else:
varstr = (input_update[var].variable.name + " UPDATE "
+ str(var.type))
else:
#a var id is needed as otherwise var with the same type will be
#merged in the graph.
varstr = str(var.type)
if (varstr in all_strings) or with_ids:
idx = ' id=' + str(len(var_str))
if len(varstr) + len(idx) > max_label_size:
varstr = varstr[:max_label_size - 3 - len(idx)] + idx + '...'
else:
varstr = varstr + idx
elif len(varstr) > max_label_size:
varstr = varstr[:max_label_size - 3] + '...'
idx = 1
while varstr in all_strings:
idx += 1
suffix = ' id=' + str(idx)
varstr = (varstr[:max_label_size - 3 - len(suffix)] +
'...' +
suffix)
var_str[var] = varstr
all_strings.add(varstr)
return varstr
topo = fct_fgraph.toposort()
apply_name_cache = {}
def apply_name(node):
if node in apply_name_cache:
return apply_name_cache[node]
prof_str = ''
if mode:
time = mode.profile_stats[fct].apply_time.get(node, 0)
#second, % total time in profiler, %fct time in profiler
if mode.local_time == 0:
pt = 0
else:
pt = time * 100 / mode.local_time
if mode.profile_stats[fct].fct_callcount == 0:
pf = 0
else:
pf = time * 100 / mode.profile_stats[fct].fct_call_time
prof_str = ' (%.3fs,%.3f%%,%.3f%%)' % (time, pt, pf)
elif profile:
time = profile.apply_time.get(node, 0)
#second, %fct time in profiler
if profile.fct_callcount == 0:
pf = 0
else:
pf = time * 100 / profile.fct_call_time
prof_str = ' (%.3fs,%.3f%%)' % (time, pf)
applystr = str(node.op).replace(':', '_')
applystr += prof_str
if (applystr in all_strings) or with_ids:
idx = ' id=' + str(topo.index(node))
if len(applystr) + len(idx) > max_label_size:
applystr = (applystr[:max_label_size - 3 - len(idx)] + idx
+ '...')
else:
applystr = applystr + idx
elif len(applystr) > max_label_size:
applystr = applystr[:max_label_size - 3] + '...'
idx = 1
while applystr in all_strings:
idx += 1
suffix = ' id=' + str(idx)
applystr = (applystr[:max_label_size - 3 - len(suffix)] +
'...' +
suffix)
all_strings.add(applystr)
apply_name_cache[node] = applystr
return applystr
# Update the inputs that have an update function
input_update = {}
outputs = list(fct_fgraph.outputs)
if isinstance(fct, Function):
for i in reversed(fct.maker.expanded_inputs):
if i.update is not None:
input_update[outputs.pop()] = i
apply_shape = 'ellipse'
var_shape = 'box'
for node_idx, node in enumerate(topo):
astr = apply_name(node)
use_color = None
for opName, color in colorCodes.items():
if opName in node.op.__class__.__name__:
use_color = color
if use_color is None:
nw_node = pd.Node(astr, shape=apply_shape)
elif high_contrast:
nw_node = pd.Node(astr, style='filled', fillcolor=use_color,
shape=apply_shape)
else:
nw_node = pd.Node(astr, color=use_color, shape=apply_shape)
g.add_node(nw_node)
if cond_highlight:
if node in middle:
c3.add_node(nw_node)
elif node in left:
c1.add_node(nw_node)
elif node in right:
c2.add_node(nw_node)
for id, var in enumerate(node.inputs):
varstr = var_name(var)
label = str(var.type)
if len(node.inputs) > 1:
label = str(id) + ' ' + label
if len(label) > max_label_size:
label = label[:max_label_size - 3] + '...'
if var.owner is None:
if high_contrast:
g.add_node(pd.Node(varstr,
style='filled',
fillcolor='green',
shape=var_shape))
else:
g.add_node(pd.Node(varstr, color='green', shape=var_shape))
g.add_edge(pd.Edge(varstr, astr, label=label))
elif var.name or not compact:
g.add_edge(pd.Edge(varstr, astr, label=label))
else:
#no name, so we don't make a var ellipse
g.add_edge(pd.Edge(apply_name(var.owner), astr, label=label))
for id, var in enumerate(node.outputs):
varstr = var_name(var)
out = any([x[0] == 'output' for x in var.clients])
label = str(var.type)
if len(node.outputs) > 1:
label = str(id) + ' ' + label
if len(label) > max_label_size:
label = label[:max_label_size - 3] + '...'
if out:
g.add_edge(pd.Edge(astr, varstr, label=label))
if high_contrast:
g.add_node(pd.Node(varstr, style='filled',
fillcolor='blue', shape=var_shape))
else:
g.add_node(pd.Node(varstr, color='blue', shape=var_shape))
elif len(var.clients) == 0:
g.add_edge(pd.Edge(astr, varstr, label=label))
if high_contrast:
g.add_node(pd.Node(varstr, style='filled',
fillcolor='grey', shape=var_shape))
else:
g.add_node(pd.Node(varstr, color='grey', shape=var_shape))
elif var.name or not compact:
g.add_edge(pd.Edge(astr, varstr, label=label))
# else:
#don't add egde here as it is already added from the inputs.
if cond_highlight:
g.add_subgraph(c1)
g.add_subgraph(c2)
g.add_subgraph(c3)
if not outfile.endswith('.' + format):
outfile += '.' + format
g.write(outfile, prog='dot', format=format)
if print_output_file:
print 'The output file is available at', outfile
if assert_nb_all_strings != -1:
assert len(all_strings) == assert_nb_all_strings
if scan_graphs:
scan_ops = [(idx, x) for idx, x in enumerate(fct_fgraph.toposort())
if isinstance(x.op, theano.scan_module.scan_op.Scan)]
path, fn = os.path.split(outfile)
basename = '.'.join(fn.split('.')[:-1])
# Safe way of doing things .. a file name may contain multiple .
ext = fn[len(basename):]
for idx, scan_op in scan_ops:
# is there a chance that name is not defined?
if hasattr(scan_op.op, 'name'):
new_name = basename + '_' + scan_op.op.name + '_' + str(idx)
else:
new_name = basename + '_' + str(idx)
new_name = os.path.join(path, new_name + ext)
pydotprint(scan_op.op.fn, new_name, compact, format, with_ids,
high_contrast, cond_highlight, colorCodes,
max_label_size, scan_graphs)
def pydotprint_variables(vars,
outfile=None,
format='png',
depth=-1,
high_contrast=True, colorCodes=None,
max_label_size=50,
var_with_name_simple=False):
''' Identical to pydotprint just that it starts from a variable instead
of a compiled function. Could be useful ? '''
if colorCodes is None:
colorCodes = default_colorCodes
if outfile is None:
outfile = os.path.join(config.compiledir, 'theano.pydotprint.' +
config.device + '.' + format)
try:
import pydot as pd
except ImportError:
print ("Failed to import pydot. You must install pydot for "
"`pydotprint_variables` to work.")
return
g = pd.Dot()
my_list = {}
orphanes = []
if type(vars) not in (list, tuple):
vars = [vars]
var_str = {}
def var_name(var):
if var in var_str:
return var_str[var]
if var.name is not None:
if var_with_name_simple:
varstr = var.name
else:
varstr = 'name=' + var.name + " " + str(var.type)
elif isinstance(var, gof.Constant):
dstr = 'val=' + str(var.data)
if '\n' in dstr:
dstr = dstr[:dstr.index('\n')]
varstr = '%s %s' % (dstr, str(var.type))
else:
#a var id is needed as otherwise var with the same type will be
#merged in the graph.
varstr = str(var.type)
varstr += ' ' + str(len(var_str))
if len(varstr) > max_label_size:
varstr = varstr[:max_label_size - 3] + '...'
var_str[var] = varstr
return varstr
def apply_name(node):
name = str(node.op).replace(':', '_')
if len(name) > max_label_size:
name = name[:max_label_size - 3] + '...'
return name
def plot_apply(app, d):
if d == 0:
return
if app in my_list:
return
astr = apply_name(app) + '_' + str(len(my_list.keys()))
if len(astr) > max_label_size:
astr = astr[:max_label_size - 3] + '...'
my_list[app] = astr
use_color = None
for opName, color in colorCodes.items():
if opName in app.op.__class__.__name__:
use_color = color
if use_color is None:
g.add_node(pd.Node(astr, shape='box'))
elif high_contrast:
g.add_node(pd.Node(astr, style='filled', fillcolor=use_color,
shape='box'))
else:
g.add_node(pd.Nonde(astr, color=use_color, shape='box'))
for i, nd in enumerate(app.inputs):
if nd not in my_list:
varastr = var_name(nd) + '_' + str(len(my_list.keys()))
if len(varastr) > max_label_size:
varastr = varastr[:max_label_size - 3] + '...'
my_list[nd] = varastr
if nd.owner is not None:
g.add_node(pd.Node(varastr))
elif high_contrast:
g.add_node(pd.Node(varastr, style='filled',
fillcolor='green'))
else:
g.add_node(pd.Node(varastr, color='green'))
else:
varastr = my_list[nd]
label = ''
if len(app.inputs) > 1:
label = str(i)
g.add_edge(pd.Edge(varastr, astr, label=label))
for i, nd in enumerate(app.outputs):
if nd not in my_list:
varastr = var_name(nd) + '_' + str(len(my_list.keys()))
if len(varastr) > max_label_size:
varastr = varastr[:max_label_size - 3] + '...'
my_list[nd] = varastr
color = None
if nd in vars:
color = 'blue'
elif nd in orphanes:
color = 'gray'
if color is None:
g.add_node(pd.Node(varastr))
elif high_contrast:
g.add_node(pd.Node(varastr, style='filled',
fillcolor=color))
else:
g.add_node(pd.Node(varastr, color=color))
else:
varastr = my_list[nd]
label = ''
if len(app.outputs) > 1:
label = str(i)
g.add_edge(pd.Edge(astr, varastr, label=label))
for nd in app.inputs:
if nd.owner:
plot_apply(nd.owner, d - 1)
for nd in vars:
if nd.owner:
for k in nd.owner.outputs:
if k not in vars:
orphanes.append(k)
for nd in vars:
if nd.owner:
plot_apply(nd.owner, depth)
try:
g.write_png(outfile, prog='dot')
except pd.InvocationException, e:
# Some version of pydot are bugged/don't work correctly with
# empty label. Provide a better user error message.
if pd.__version__ == "1.0.28" and "label=]" in e.message:
raise Exception("pydot 1.0.28 is know to be bugged. Use another "
"working version of pydot")
elif "label=]" in e.message:
raise Exception("Your version of pydot " + pd.__version__ +
" returned an error. Version 1.0.28 is known"
" to be bugged and 1.0.25 to be working with"
" Theano. Using another version of pydot could"
" fix this problem. The pydot error is: " +
e.message)
def get_pydot_graph(layers, output_shape=True, verbose=False):
"""
Creates a PyDot graph of the network defined by the given layers.
:parameters:
- layers : list
List of the layers, as obtained from lasange.layers.get_all_layers
- output_shape: (default `True`)
If `True`, the output shape of each layer will be displayed.
- verbose: (default `False`)
If `True`, layer attributes like filter shape, stride, etc.
will be displayed.
- verbose:
:returns:
- pydot_graph : PyDot object containing the graph
"""
pydot_graph = pydot.Dot('Network', graph_type='digraph')
pydot_nodes = {}
pydot_edges = []
for i, layer in enumerate(layers):
layer_type = '{0}: {1}'.format(layer.__class__.__name__, layer.name)
key = repr(layer)
label = layer_type
color = get_hex_color(layer_type)
if verbose:
for attr in [
'num_filters', 'num_units', 'ds', 'filter_shape', 'stride',
'strides', 'p'
]:
if hasattr(layer, attr):
label += '\n' + \
'{0}: {1}'.format(attr, getattr(layer, attr))
if hasattr(layer, 'nonlinearity'):
try:
nonlinearity = layer.nonlinearity.__name__
except AttributeError:
nonlinearity = layer.nonlinearity.__class__.__name__
label += '\n' + 'nonlinearity: {0}'.format(nonlinearity)
if output_shape:
output_shape = lasagne.layers.get_output_shape(layer)
if len(output_shape) is 3:
output_shape_str = '(Batch Size, Seq Len, {})'.format(
output_shape[-1])
if len(output_shape) is 2:
output_shape_str = '(Batch Size x Seq Len, {})'.format(
output_shape[-1])
if layer.name == 'mask':
output_shape_str = '(Batch Size, Seq Len)'
label += '\n' + \
'Output shape: {0}'.format(output_shape_str)
pydot_nodes[key] = pydot.Node(
key,
label=label,
shape='record',
fillcolor=color,
style='filled',
)
if hasattr(layer, 'input_layers'):
for input_layer in layer.input_layers:
pydot_edges.append([repr(input_layer), key])
if hasattr(layer, 'input_layer'):
pydot_edges.append([repr(layer.input_layer), key])
for node in pydot_nodes.values():
pydot_graph.add_node(node)
for edge in pydot_edges:
pydot_graph.add_edge(
pydot.Edge(pydot_nodes[edge[0]], pydot_nodes[edge[1]]))
return pydot_graph
def visualise(tree):
graph = pydot.Dot(graph_type='graph')
open = [(tree, 0)]
closed = []
nodeCounter = 0
filterLabel = lambda x: x.value if x.id == 'Const' or x.id == 'Name' else x.id
def nodeColour(id):
if id is 'Const':
return 'green'
elif id is 'Name':
return 'yellow'
elif id is "Module":
return "grey"
else:
return 'cyan'
while len(open) is not 0:
parent = open[0]
children_ = list(
filter(lambda x: x is not None,
[parent[0].first, parent[0].second, parent[0].third]))
children = []
for i in range(len(children_)):
if isinstance(children_[i], (list, )):
children = children_[:i] + children_[i]
else:
children.append(children_[i])
parentNode = pydot.Node(parent[1],
label=filterLabel(parent[0]),
style="filled",
fillcolor=nodeColour(parent[0].id))
nodeCounter += 1
children = list(
zip(children, range(nodeCounter, nodeCounter + len(children))))
nodeCounter += len(children)
childrenNodes = [
pydot.Node(c[1],
label=filterLabel(c[0]),
style="filled",
fillcolor=nodeColour(c[0].id)) for c in children
]
graph.add_node(parentNode)
[graph.add_node(c) for c in childrenNodes]
[
graph.add_edge(pydot.Edge(parentNode, c))
for c in map(lambda x: x[1], children)
]
closed.append(open[0])
open = children + open[1:]
formato = "svg"
filename = 'draft_tree.%s' % formato
graph.write(filename, format=formato)
formato = "png"
filename = 'draft_tree.%s' % formato
graph.write(filename, format=formato)
img = mpimg.imread(filename)
imgplot = plt.imshow(img)
plt.axis("off")
plt.show()
def test_dot_args(self):
g = pydot.Dot()
u = pydot.Node('a')
g.add_node(u)
g.write_svg('test.svg', prog=['twopi', '-Goverlap=scale'])
def get_beam_dot(self, dictionary=None, n_best=None):
"""
Create pydot graph representation of the beam.
:param outputs:
self.outputs from the beam
:param dictionary:
tok 2 word dict to save words in the tree nodes
:returns:
pydot graph
"""
try:
import pydot
except ImportError:
print("Please install pydot package to dump beam visualization")
graph = pydot.Dot(graph_type='digraph')
outputs = [i.tolist() for i in self.outputs]
bookkeep = [i.tolist() for i in self.bookkeep]
all_scores = [i.tolist() for i in self.all_scores]
if n_best is None:
n_best = int(self.beam_size / 2)
# get top nbest hyp
top_hyp_idx_n_best = []
n_best_colors = [
'aquamarine', 'chocolate1', 'deepskyblue', 'green2', 'tan'
]
sorted_finished = self.get_rescored_finished(n_best=n_best)
for hyptail in sorted_finished:
# do not include EOS since it has rescored score not from original
# self.all_scores, we color EOS with black
top_hyp_idx_n_best.append(self.get_hyp_from_finished(hyptail))
# create nodes
for tstep, lis in enumerate(outputs):
for hypid, token in enumerate(lis):
if tstep == 0:
hypid = 0 # collapse all __NULL__ nodes
node_tail = self.HypothesisTail(
timestep=tstep,
hypid=hypid,
score=all_scores[tstep][hypid],
tokenid=token,
)
color = 'white'
rank = None
for i, hypseq in enumerate(top_hyp_idx_n_best):
if node_tail in hypseq:
if n_best <= 5: # color nodes only if <=5
color = n_best_colors[i]
rank = i
break
label = ("<{}".format(
dictionary.
vec2txt([token]) if dictionary is not None else token) +
" : " + "{:.{prec}f}>".format(
all_scores[tstep][hypid], prec=3))
graph.add_node(
pydot.Node(
node_tail.__repr__(),
label=label,
fillcolor=color,
style='filled',
xlabel='{}'.format(rank) if rank is not None else '',
))
# create edges
for revtstep, lis in reversed(list(enumerate(bookkeep))):
for i, prev_id in enumerate(lis):
from_node = graph.get_node('"{}"'.format(
self.HypothesisTail(
timestep=revtstep,
hypid=prev_id,
score=all_scores[revtstep][prev_id],
tokenid=outputs[revtstep][prev_id],
).__repr__()))[0]
to_node = graph.get_node('"{}"'.format(
self.HypothesisTail(
timestep=revtstep + 1,
hypid=i,
score=all_scores[revtstep + 1][i],
tokenid=outputs[revtstep + 1][i],
).__repr__()))[0]
newedge = pydot.Edge(from_node.get_name(), to_node.get_name())
graph.add_edge(newedge)
return graph
def __init__(self, name):
self.name = name
self.G = pydot.Dot(graph_type="digraph")
def lerGrafoDoArquivoMatrizAdjacente(self, caminhoMatriz, caminhoLista):
print "Lendo matriz adjacente: " + caminhoMatriz
arquivoLista = open(caminhoLista, 'r')
listaNome = []
i = 1
while True:
nome = arquivoLista.readline()
if nome == '':
break
listaNome.append(nome.strip())
i += 1
print("listaNome.len: " + str(len(listaNome)))
arquivoMatriz = open(caminhoMatriz, 'r')
self.grafo = pydot.Dot('Grafo', graph_type='graph')
#obter numero de no em primeira linha
lista = arquivoMatriz.readline().split('\n')[0].split('\t')
no_count = 0
aresta_count = 0
for coluna in range(0, len(lista)):
self.grafo.add_node(pydot.Node(listaNome[coluna]))
no_count += 1
if lista[coluna] == "1":
self.grafo.add_edge(pydot.Edge(listaNome[coluna],
listaNome[0]))
#print ' add edge:'+listaNome[coluna]+" - "+listaNome[0]
aresta_count += 1
print "numero no: " + str(no_count)
#loop de toda linha, 2 ate no_count inclusive
for i in range(1, no_count):
linha = arquivoMatriz.readline()
if linha != "":
lista = linha.split('\n')[0].split('\t')
j = 0
for cell in lista:
if cell == '1' and j > i:
self.grafo.add_edge(
pydot.Edge(listaNome[j], listaNome[i]))
#print 'add Edge: ' + str(j+i) + ' - ' + str(i)
aresta_count += 1
j += 1
else:
print "Erro de leitura na Matriz de adjacencia"
break
print("numero aresta: " + str(aresta_count))
subX = pydot.Subgraph('', rank='same')
subX.set_name('X')
subY = pydot.Subgraph('', rank='same')
subY.set_name('Y')
for nome in listaNome:
if nome.isdigit():
subY.add_node(pydot.Node(nome))
else:
subX.add_node(pydot.Node(nome))
self.grafo.add_subgraph(subX)
self.grafo.add_subgraph(subY)
subM = pydot.Subgraph()
subM.set_name('M')
if len(self.grafo.get_edge_list()) > 0:
subM.add_edge(self.grafo.get_edge_list()[0])
self.grafo.add_subgraph(subM)
print("Matriz lida")
#print self.grafo.to_string()
f = open('DOT_original.dot', 'w')
f.write(self.grafo.to_string())
f.close()
return
def __init__(self, graph):
self._graph = graph
self._roots = self._graph.root
self.image = pydot.Dot(graph_type="digraph")
self.edges = OrderedSet()
self.nodes = OrderedSet()
from caps.util import utilities as ut
import pydot
simulation = "L500_NR_0"
mt = db.Simulation(simulation + "", db_dir="/Users/Kaylea/Data/L500_NR_0")
clusters = mt.get_halo_ids(main_halos_only=True)
halos = mt.get_halo_properties(clusters, ["M_hc", "r_hc"], "1.0005")
for cluster in clusters:
if cluster != 17:
continue
dot_object = pydot.Dot(graph_type='graph')
dot_object.set_node_defaults(shape='circle', fontsize="14")
dot_object.set_edge_defaults(arrowhead="diamond")
print cluster
z0_mass = halos[halos["id"] == cluster]["M_hc"]
z0_radius = halos[halos["id"] == cluster]["r_hc"]
nodes = {}
edges = []
node_name = "1.0005_%d" % cluster
nodes[node_name] = pydot.Node(name=node_name,
style="bold",
xlabel=str(cluster),
label="",
def draw(tree, exp): graph = pydot.Dot(graph_type='graph') tree.to_graph(graph) name = "Tree_of_" + exp + ".png" graph.write_png(name)
def pass_manager_drawer(pass_manager, filename, style=None, raw=False):
"""
Draws the pass manager.
This function needs `pydot <https://github.com/erocarrera/pydot>`, which in turn needs
Graphviz <https://www.graphviz.org/>` to be installed.
Args:
pass_manager (PassManager): the pass manager to be drawn
filename (str): file path to save image to
style (dict or OrderedDict): keys are the pass classes and the values are
the colors to make them. An example can be seen in the DEFAULT_STYLE. An ordered
dict can be used to ensure a priority coloring when pass falls into multiple
categories. Any values not included in the provided dict will be filled in from
the default dict
raw (Bool) : True if you want to save the raw Dot output not an image. The
default is False.
Returns:
PIL.Image or None: an in-memory representation of the pass manager. Or None if
no image was generated or PIL is not installed.
Raises:
ImportError: when nxpd or pydot not installed.
VisualizationError: If raw=True and filename=None.
"""
try:
import subprocess
_PROC = subprocess.Popen(['dot', '-V'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
_PROC.communicate()
if _PROC.returncode != 0:
HAS_GRAPHVIZ = False
else:
HAS_GRAPHVIZ = True
except Exception: # pylint: disable=broad-except
# this is raised when the dot command cannot be found, which means GraphViz
# isn't installed
HAS_GRAPHVIZ = False
try:
import pydot
if not HAS_GRAPHVIZ:
raise ImportError
except ImportError:
raise ImportError(
"pass_manager_drawer requires pydot and graphviz. "
"Run 'pip install pydot'. "
"Graphviz can be installed using 'brew install graphviz' on Mac"
" or by downloading it from the website.")
passes = pass_manager.passes()
if not style:
style = DEFAULT_STYLE
# create the overall graph
graph = pydot.Dot()
# identifiers for nodes need to be unique, so assign an id
# can't just use python's id in case the exact same pass was
# appended more than once
component_id = 0
prev_node = None
for controller_group in passes:
# label is the name of the flow controller (without the word controller)
label = controller_group['type'].__name__.replace('Controller', '')
# create the subgraph for this controller
subgraph = pydot.Cluster(str(component_id),
label=label,
fontname='helvetica')
component_id += 1
for pass_ in controller_group['passes']:
# label is the name of the pass
node = pydot.Node(str(component_id),
label=str(type(pass_).__name__),
color=_get_node_color(pass_, style),
shape="rectangle",
fontname='helvetica')
subgraph.add_node(node)
component_id += 1
# the arguments that were provided to the pass when it was created
arg_spec = inspect.getfullargspec(pass_.__init__)
# 0 is the args, 1: to remove the self arg
args = arg_spec[0][1:]
num_optional = len(arg_spec[3]) if arg_spec[3] else 0
# add in the inputs to the pass
for arg_index, arg in enumerate(args):
nd_style = 'solid'
# any optional args are dashed
# the num of optional counts from the end towards the start of the list
if arg_index >= (len(args) - num_optional):
nd_style = 'dashed'
input_node = pydot.Node(component_id,
label=arg,
color="black",
shape="ellipse",
fontsize=10,
style=nd_style,
fontname='helvetica')
subgraph.add_node(input_node)
component_id += 1
subgraph.add_edge(pydot.Edge(input_node, node))
# if there is a previous node, add an edge between them
if prev_node:
subgraph.add_edge(pydot.Edge(prev_node, node))
prev_node = node
graph.add_subgraph(subgraph)
if raw:
if filename:
graph.write(filename, format='raw')
return None
else:
raise VisualizationError(
"if format=raw, then a filename is required.")
if not HAS_PIL and filename:
# linter says this isn't a method - it is
graph.write_png(filename) # pylint: disable=no-member
return None
with tempfile.TemporaryDirectory() as tmpdirname:
tmppath = os.path.join(tmpdirname, 'pass_manager.png')
# linter says this isn't a method - it is
graph.write_png(tmppath) # pylint: disable=no-member
image = Image.open(tmppath)
image = utils._trim(image)
os.remove(tmppath)
if filename:
image.save(filename, 'PNG')
return image
def create_schema_graph(tables=None, metadata=None, show_indexes=True, show_datatypes=True, font="Bitstream-Vera Sans",
concentrate=True, relation_options={}, rankdir='TB', show_column_keys=False, restrict_tables=None):
"""
Args:
show_column_keys (boolean, default=False): If true then add a PK/FK suffix to columns names that are primary and foreign keys
restrict_tables (None or list of strings): Restrict the graph to only consider tables whose name are defined restrict_tables
"""
relation_kwargs = {
'fontsize':"7.0"
}
relation_kwargs.update(relation_options)
if metadata is None and tables is not None and len(tables):
metadata = tables[0].metadata
elif tables is None and metadata is not None:
if not len(metadata.tables):
metadata.reflect()
tables = metadata.tables.values()
else:
raise ValueError("You need to specify at least tables or metadata")
graph = pydot.Dot(prog="dot",mode="ipsep",overlap="ipsep",sep="0.01",concentrate=str(concentrate), rankdir=rankdir)
if restrict_tables is None:
restrict_tables = set([t.name.lower() for t in tables])
else:
restrict_tables = set([t.lower() for t in restrict_tables])
tables = [t for t in tables if t.name.lower() in restrict_tables]
for table in tables:
graph.add_node(pydot.Node(str(table.name),
shape="plaintext",
label=_render_table_html(table, metadata, show_indexes, show_datatypes, show_column_keys),
fontname=font, fontsize="7.0"
))
for table in tables:
for fk in table.foreign_keys:
if fk.column.table not in tables:
continue
edge = [table.name, fk.column.table.name]
is_inheritance = fk.parent.primary_key and fk.column.primary_key
if is_inheritance:
edge = edge[::-1]
graph_edge = pydot.Edge(
dir='both',
headlabel="+ %s"%fk.column.name, taillabel='+ %s'%fk.parent.name,
arrowhead=is_inheritance and 'none' or 'odot' ,
arrowtail=(fk.parent.primary_key or fk.parent.unique) and 'empty' or 'crow' ,
fontname=font,
#samehead=fk.column.name, sametail=fk.parent.name,
*edge, **relation_kwargs
)
graph.add_edge(graph_edge)
# not sure what this part is for, doesn't work with pydot 1.0.2
# graph_edge.parent_graph = graph.parent_graph
# if table.name not in [e.get_source() for e in graph.get_edge_list()]:
# graph.edge_src_list.append(table.name)
# if fk.column.table.name not in graph.edge_dst_list:
# graph.edge_dst_list.append(fk.column.table.name)
# graph.sorted_graph_elements.append(graph_edge)
return graph
def build_pydot(
graph,
steps=None,
inputs=None,
outputs=None,
solution=None,
executed=None,
title=None,
node_props=None,
edge_props=None,
clusters=None,
):
"""
Build a *Graphviz* out of a Network graph/steps/inputs/outputs and return it.
See :meth:`Plotter.plot()` for the arguments, sample code, and
the legend of the plots.
"""
import pydot
from .base import NetworkOperation, Operation
from .modifiers import optional
from .network import _EvictInstruction, _PinInstruction
_monkey_patch_for_jupyter(pydot)
assert graph is not None
steps_thickness = 3
fill_color = "wheat"
steps_color = "#009999"
new_clusters = {}
def append_or_cluster_node(dot, nx_node, node):
if not clusters or not nx_node in clusters:
dot.add_node(node)
else:
cluster_name = clusters[nx_node]
node_cluster = new_clusters.get(cluster_name)
if not node_cluster:
node_cluster = new_clusters[cluster_name] = pydot.Cluster(
cluster_name, label=cluster_name)
node_cluster.add_node(node)
def append_any_clusters(dot):
for cluster in new_clusters.values():
dot.add_subgraph(cluster)
def quote_dot_kws(word):
return "'%s'" % word if word in pydot.dot_keywords else word
def get_node_name(a):
if isinstance(a, Operation):
a = a.name
return quote_dot_kws(a)
dot = pydot.Dot(graph_type="digraph",
label=quote_dot_kws(title),
fontname="italic")
# draw nodes
for nx_node in graph.nodes:
if isinstance(nx_node, str):
kw = {}
# FrameColor change by step type
if steps and nx_node in steps:
choice = _merge_conditions(
_is_class_value_in_list(steps, _EvictInstruction, nx_node),
_is_class_value_in_list(steps, _PinInstruction, nx_node),
)
# 0 is singled out because `nx_node` exists in `steps`.
color = "NOPE #990000 blue purple".split()[choice]
kw = {"color": color, "penwidth": steps_thickness}
# SHAPE change if with inputs/outputs.
# tip: https://graphviz.gitlab.io/_pages/doc/info/shapes.html
choice = _merge_conditions(inputs and nx_node in inputs, outputs
and nx_node in outputs)
shape = "rect invhouse house hexagon".split()[choice]
# LABEL change with solution.
if solution and nx_node in solution:
kw["style"] = "filled"
kw["fillcolor"] = fill_color
# kw["tooltip"] = str(solution.get(nx_node)) # not working :-()
node = pydot.Node(name=quote_dot_kws(nx_node), shape=shape, **kw)
else: # Operation
kw = {"fontname": "italic"}
if steps and nx_node in steps:
kw["penwdth"] = steps_thickness
shape = "egg" if isinstance(nx_node, NetworkOperation) else "oval"
if executed and nx_node in executed:
kw["style"] = "filled"
kw["fillcolor"] = fill_color
node = pydot.Node(name=quote_dot_kws(nx_node.name),
shape=shape,
**kw)
_apply_user_props(node, node_props, key=node.get_name())
append_or_cluster_node(dot, nx_node, node)
_report_unmatched_user_props(node_props, "node")
append_any_clusters(dot)
# draw edges
for src, dst, data in graph.edges(data=True):
src_name = get_node_name(src)
dst_name = get_node_name(dst)
kw = {}
if data.get("optional"):
kw["style"] = "dashed"
if data.get("sideffect"):
kw["color"] = "blue"
# `splines=ortho` not working :-()
edge = pydot.Edge(src=src_name, dst=dst_name, splines="ortho", **kw)
_apply_user_props(edge, edge_props, key=(src, dst))
dot.add_edge(edge)
_report_unmatched_user_props(edge_props, "edge")
# draw steps sequence
if steps and len(steps) > 1:
it1 = iter(steps)
it2 = iter(steps)
next(it2)
for i, (src, dst) in enumerate(zip(it1, it2), 1):
src_name = get_node_name(src)
dst_name = get_node_name(dst)
edge = pydot.Edge(
src=src_name,
dst=dst_name,
label=str(i),
style="dotted",
color=steps_color,
fontcolor=steps_color,
fontname="bold",
fontsize=18,
penwidth=steps_thickness,
arrowhead="vee",
splines=True,
)
dot.add_edge(edge)
return dot
def create_uml_graph(mappers, show_operations=True, show_attributes=True, show_inherited=True, show_multiplicity_one=False, show_datatypes=True, linewidth=1.0, font="Bitstream-Vera Sans"):
graph = pydot.Dot(prog='neato',mode="major",overlap="0", sep="0.01",dim="3", pack="True", ratio=".75")
relations = set()
for mapper in mappers:
graph.add_node(pydot.Node(escape(mapper.class_.__name__),
shape="plaintext", label=_mk_label(mapper, show_operations, show_attributes, show_datatypes, show_inherited, linewidth),
fontname=font, fontsize="8.0",
))
if mapper.inherits:
graph.add_edge(pydot.Edge(escape(mapper.inherits.class_.__name__),escape(mapper.class_.__name__),
arrowhead='none',arrowtail='empty', style="setlinewidth(%s)" % linewidth, arrowsize=str(linewidth)))
for loader in mapper.iterate_properties:
if isinstance(loader, RelationshipProperty) and loader.mapper in mappers:
if hasattr(loader, 'reverse_property'):
relations.add(frozenset([loader, loader.reverse_property]))
else:
relations.add(frozenset([loader]))
for relation in relations:
#if len(loaders) > 2:
# raise Exception("Warning: too many loaders for join %s" % join)
args = {}
def multiplicity_indicator(prop):
if prop.uselist:
return ' *'
if hasattr(prop, 'local_side'):
cols = prop.local_side
else:
cols = prop.local_columns
if any(col.nullable for col in cols):
return ' 0..1'
if show_multiplicity_one:
return ' 1'
return ''
if len(relation) == 2:
src, dest = relation
from_name = escape(src.parent.class_.__name__)
to_name = escape(dest.parent.class_.__name__)
def calc_label(src,dest):
return '+' + src.key + multiplicity_indicator(src)
args['headlabel'] = calc_label(src,dest)
args['taillabel'] = calc_label(dest,src)
args['arrowtail'] = 'none'
args['arrowhead'] = 'none'
args['constraint'] = False
else:
prop, = relation
from_name = escape(prop.parent.class_.__name__)
to_name = escape(prop.mapper.class_.__name__)
args['headlabel'] = '+%s%s' % (prop.key, multiplicity_indicator(prop))
args['arrowtail'] = 'none'
args['arrowhead'] = 'vee'
graph.add_edge(pydot.Edge(from_name,to_name,
fontname=font, fontsize="7.0", style="setlinewidth(%s)"%linewidth, arrowsize=str(linewidth),
**args)
)
return graph
def make_graph(db_name: str, room_id: str, file_prefix: str,
limit: int) -> None:
"""
Generate a dot and SVG file for a graph of events in the room based on the
topological ordering by reading from a Synapse SQLite database.
"""
conn = sqlite3.connect(db_name)
sql = "SELECT room_version FROM rooms WHERE room_id = ?"
c = conn.execute(sql, (room_id, ))
room_version = KNOWN_ROOM_VERSIONS[c.fetchone()[0]]
sql = ("SELECT json, internal_metadata FROM event_json as j "
"INNER JOIN events as e ON e.event_id = j.event_id "
"WHERE j.room_id = ?")
args = [room_id]
if limit:
sql += " ORDER BY topological_ordering DESC, stream_ordering DESC LIMIT ?"
args.append(limit)
c = conn.execute(sql, args)
events = [
make_event_from_dict(json.loads(e[0]), room_version, json.loads(e[1]))
for e in c.fetchall()
]
events.sort(key=lambda e: e.depth)
node_map = {}
state_groups = {}
graph = pydot.Dot(graph_name="Test")
for event in events:
c = conn.execute(
"SELECT state_group FROM event_to_state_groups WHERE event_id = ?",
(event.event_id, ),
)
res = c.fetchone()
state_group = res[0] if res else None
if state_group is not None:
state_groups.setdefault(state_group, []).append(event.event_id)
t = datetime.datetime.fromtimestamp(
float(event.origin_server_ts) /
1000).strftime("%Y-%m-%d %H:%M:%S,%f")
content = json.dumps(unfreeze(event.get_dict()["content"]))
label = ("<"
"<b>%(name)s </b><br/>"
"Type: <b>%(type)s </b><br/>"
"State key: <b>%(state_key)s </b><br/>"
"Content: <b>%(content)s </b><br/>"
"Time: <b>%(time)s </b><br/>"
"Depth: <b>%(depth)s </b><br/>"
"State group: %(state_group)s<br/>"
">") % {
"name": event.event_id,
"type": event.type,
"state_key": event.get("state_key", None),
"content": html.escape(content, quote=True),
"time": t,
"depth": event.depth,
"state_group": state_group,
}
node = pydot.Node(name=event.event_id, label=label)
node_map[event.event_id] = node
graph.add_node(node)
for event in events:
for prev_id in event.prev_event_ids():
try:
end_node = node_map[prev_id]
except Exception:
end_node = pydot.Node(name=prev_id,
label=f"<<b>{prev_id}</b>>")
node_map[prev_id] = end_node
graph.add_node(end_node)
edge = pydot.Edge(node_map[event.event_id], end_node)
graph.add_edge(edge)
for group, event_ids in state_groups.items():
if len(event_ids) <= 1:
continue
cluster = pydot.Cluster(str(group),
label=f"<State Group: {str(group)}>")
for event_id in event_ids:
cluster.add_node(node_map[event_id])
graph.add_subgraph(cluster)
graph.write("%s.dot" % file_prefix, format="raw", prog="dot")
graph.write_svg("%s.svg" % file_prefix, prog="dot")
def prov_to_dot(bundle,
show_nary=True,
use_labels=False,
direction='BT',
show_element_attributes=True,
show_relation_attributes=True):
"""
Convert a provenance bundle/document into a DOT graphical representation.
:param bundle: The provenance bundle/document to be converted.
:type bundle: :class:`ProvBundle`
:param show_nary: shows all elements in n-ary relations.
:type show_nary: bool
:param use_labels: uses the prov:label property of an element as its name (instead of its identifier).
:type use_labels: bool
:param direction: specifies the direction of the graph. Valid values are "BT" (default), "TB", "LR", "RL".
:param show_element_attributes: shows attributes of elements.
:type show_element_attributes: bool
:param show_relation_attributes: shows attributes of relations.
:type show_relation_attributes: bool
:returns: :class:`pydot.Dot` -- the Dot object.
"""
if direction not in {'BT', 'TB', 'LR', 'RL'}:
# Invalid direction is provided
direction = 'BT' # reset it to the default value
maindot = pydot.Dot(graph_type='digraph',
rankdir=direction,
charset='utf-8')
node_map = {}
count = [0, 0, 0, 0] # counters for node ids
def _bundle_to_dot(dot, bundle):
def _attach_attribute_annotation(node, record):
# Adding a node to show all attributes
attributes = list((attr_name, value)
for attr_name, value in record.attributes
if attr_name not in PROV_ATTRIBUTE_QNAMES)
if not attributes:
return # No attribute to display
# Sort the attributes.
attributes = sorted_attributes(record.get_type(), attributes)
ann_rows = [ANNOTATION_START_ROW]
ann_rows.extend(
ANNOTATION_ROW_TEMPLATE %
(attr.uri, escape(six.text_type(attr)), ' href=\"%s\"' %
value.uri if isinstance(value, Identifier) else '',
escape(
six.text_type(value) if not isinstance(value, datetime)
else six.text_type(value.isoformat())))
for attr, value in attributes)
ann_rows.append(ANNOTATION_END_ROW)
count[3] += 1
annotations = pydot.Node('ann%d' % count[3],
label='\n'.join(ann_rows),
**ANNOTATION_STYLE)
dot.add_node(annotations)
dot.add_edge(pydot.Edge(annotations, node,
**ANNOTATION_LINK_STYLE))
def _add_bundle(bundle):
count[2] += 1
subdot = pydot.Cluster(graph_name='c%d' % count[2],
URL='"%s"' % bundle.identifier.uri)
if use_labels:
if bundle.label == bundle.identifier:
bundle_label = '"%s"' % six.text_type(bundle.label)
else:
# Fancier label if both are different. The label will be
# the main node text, whereas the identifier will be a
# kind of subtitle.
bundle_label = ('<%s<br />'
'<font color="#333333" point-size="10">'
'%s</font>>')
bundle_label = bundle_label % (six.text_type(
bundle.label), six.text_type(bundle.identifier))
subdot.set_label('"%s"' % six.text_type(bundle_label))
else:
subdot.set_label('"%s"' % six.text_type(bundle.identifier))
_bundle_to_dot(subdot, bundle)
dot.add_subgraph(subdot)
return subdot
def _add_node(record):
count[0] += 1
node_id = 'n%d' % count[0]
if use_labels:
if record.label == record.identifier:
node_label = '"%s"' % six.text_type(record.label)
else:
# Fancier label if both are different. The label will be
# the main node text, whereas the identifier will be a
# kind of subtitle.
node_label = ('<%s<br />'
'<font color="#333333" point-size="10">'
'%s</font>>')
node_label = node_label % (six.text_type(
record.label), six.text_type(record.identifier))
else:
node_label = '"%s"' % six.text_type(record.identifier)
uri = record.identifier.uri
style = DOT_PROV_STYLE[record.get_type()]
node = pydot.Node(node_id,
label=node_label,
URL='"%s"' % uri,
**style)
node_map[uri] = node
dot.add_node(node)
if show_element_attributes:
_attach_attribute_annotation(node, rec)
return node
def _add_generic_node(qname, prov_type=None):
count[0] += 1
node_id = 'n%d' % count[0]
node_label = '"%s"' % six.text_type(qname)
uri = qname.uri
style = GENERIC_NODE_STYLE[
prov_type] if prov_type else DOT_PROV_STYLE[0]
node = pydot.Node(node_id,
label=node_label,
URL='"%s"' % uri,
**style)
node_map[uri] = node
dot.add_node(node)
return node
def _get_bnode():
count[1] += 1
bnode_id = 'b%d' % count[1]
bnode = pydot.Node(bnode_id,
label='""',
shape='point',
color='gray')
dot.add_node(bnode)
return bnode
def _get_node(qname, prov_type=None):
if qname is None:
return _get_bnode()
uri = qname.uri
if uri not in node_map:
_add_generic_node(qname, prov_type)
return node_map[uri]
records = bundle.get_records()
relations = []
for rec in records:
if rec.is_element():
_add_node(rec)
else:
# Saving the relations for later processing
relations.append(rec)
if not bundle.is_bundle():
for bundle in bundle.bundles:
_add_bundle(bundle)
for rec in relations:
args = rec.args
# skipping empty records
if not args:
continue
# picking element nodes
attr_names, nodes = zip(
*((attr_name, value)
for attr_name, value in rec.formal_attributes
if attr_name in PROV_ATTRIBUTE_QNAMES))
inferred_types = list(map(INFERRED_ELEMENT_CLASS.get, attr_names))
other_attributes = [(attr_name, value)
for attr_name, value in rec.attributes
if attr_name not in PROV_ATTRIBUTE_QNAMES]
add_attribute_annotation = (show_relation_attributes
and other_attributes)
add_nary_elements = len(nodes) > 2 and show_nary
style = DOT_PROV_STYLE[rec.get_type()]
if len(nodes) < 2: # too few elements for a relation?
continue # cannot draw this
if add_nary_elements or add_attribute_annotation:
# a blank node for n-ary relations or the attribute annotation
bnode = _get_bnode()
# the first segment
dot.add_edge(
pydot.Edge(_get_node(nodes[0], inferred_types[0]),
bnode,
arrowhead='none',
**style))
style = dict(style) # copy the style
del style['label'] # not showing label in the second segment
# the second segment
dot.add_edge(
pydot.Edge(bnode, _get_node(nodes[1], inferred_types[1]),
**style))
if add_nary_elements:
style['color'] = 'gray' # all remaining segment to be gray
style['fontcolor'] = 'dimgray' # text in darker gray
for attr_name, node, inferred_type in zip(
attr_names[2:], nodes[2:], inferred_types[2:]):
if node is not None:
style['label'] = attr_name.localpart
dot.add_edge(
pydot.Edge(bnode,
_get_node(node,
inferred_type), **style))
if add_attribute_annotation:
_attach_attribute_annotation(bnode, rec)
else:
# show a simple binary relations with no annotation
dot.add_edge(
pydot.Edge(_get_node(nodes[0], inferred_types[0]),
_get_node(nodes[1], inferred_types[1]),
**style))
try:
unified = bundle.unified()
except ProvException:
# Could not unify this bundle
# try the original document anyway
unified = bundle
_bundle_to_dot(maindot, unified)
return maindot
def make_graph(*,
nodule_out_edge_map: par.NoduleOutEdgeMap,
edge_lookup_map: par.EdgeLookupMap,
unique_id_conv: UniqueIdConverter = None,
token_conv: TokenConverter = None) -> pydot.Graph:
# If ID converter is not specified, use the default.
if unique_id_conv is None:
logger.info(
'Unique ID converter not specified, using default converter')
def unique_id_conv(x):
return str(x)
# If token converter is not specified, use the default.
if token_conv is None:
logger.info('Token converter not specified, using default converter')
def token_conv(t: ctpt.Token) -> str:
token_data: ctpt.TokenData = t.data
return str(token_data)
# Storage for Graphviz nodes and edges.
gv_node_map: typ.MutableMapping[ctpc.UniqueId, pydot.Node] = {}
gv_edges: typ.MutableSequence[pydot.Edge] = []
# Create Graphviz nodes from nodules.
nodule_count = 0
for nodule in nodule_out_edge_map:
nodule_gv_node = pydot.Node(name=unique_id_conv(nodule),
shape='point',
width=0.125,
height=0.125)
gv_node_map[nodule] = nodule_gv_node
nodule_count += 1
logger.info(f'Created {nodule_count} Graphviz node(s) from nodules')
# Create Graphviz nodes from tokens.
token_count = 0
for token in itertools.chain.from_iterable(
edge_def.token_seq for edge_def in edge_lookup_map.values()):
token_id: ctpt.TokenId = token.id
token_gv_node = pydot.Node(name=unique_id_conv(token_id),
shape='circle',
label=token_conv(token))
gv_node_map[token_id] = token_gv_node
token_count += 1
logger.info(f'Created {token_count} Graphviz node(s) from tokens')
# Creating Graphviz edges are more complicated.
# An edge needs to be drawn between nodules and tokens, not just from nodule to nodule.
# In addition, only edges directly entering a nodule should have arrowheads.
# Start with each edge definition.
virtual_edge_count = 0
for edge_id, edge_def in edge_lookup_map.items():
src_nodule: par.Nodule = edge_def.src_nodule
dst_nodule: par.Nodule = edge_def.dst_nodule
token_seq: ctpt.TokenSequence = edge_def.token_seq
# For this edge definition, this stores the newest node ID to connect a Graphviz edge from.
curr_anchor_node_id: ctpc.UniqueId = src_nodule
tail_label: str = par.stack_cmd_label_str(edge_def.start_cmd)
head_label: str = par.stack_cmd_label_str(edge_def.close_cmd)
for token in token_seq:
token_id: ctpt.TokenId = token.id
# Draw a Graphviz edge (without arrowhead) from the current anchor node to this token.
gv_edge = pydot.Edge(gv_node_map[curr_anchor_node_id],
gv_node_map[token_id],
arrowhead='none',
headlabel='',
taillabel=tail_label)
tail_label = ''
gv_edges.append(gv_edge)
# Update the current anchor node ID.
curr_anchor_node_id = token_id
# Draw a Graphviz edge from the current anchor node to the destination nodule.
# This edge will have an arrowhead.
gv_edge = pydot.Edge(gv_node_map[curr_anchor_node_id],
gv_node_map[dst_nodule],
arrowhead='vee',
headlabel=head_label,
taillabel=tail_label)
gv_edges.append(gv_edge)
virtual_edge_count += 1
actual_edge_count = len(gv_edges)
logger.info(
f'Created {actual_edge_count} Graphviz edge(s) from {virtual_edge_count} virtual edge(s)'
)
# Create Graphviz graph, and add nodes and edges to it.
graph: pydot.Graph = pydot.Dot(graph_type='digraph',
strict=False,
overlap=False,
splines=True)
for gv_node in gv_node_map.values():
graph.add_node(gv_node)
for gv_edge in gv_edges:
graph.add_edge(gv_edge)
return graph
def show_bgpmap():
"""return a bgp map in a png file, from the json tree in q argument"""
data = get_query()
if not data:
abort(400)
data = base64.b64decode(data)
data = json.loads(data)
graph = pydot.Dot('BGPMAP', graph_type='digraph')
nodes = {}
edges = {}
prepend_as = {}
def escape(label):
label = label.replace("&", "&")
label = label.replace(">", ">")
label = label.replace("<", "<")
return label
def add_node(_as, **kwargs):
if _as not in nodes:
kwargs[
"label"] = '<<TABLE CELLBORDER="0" BORDER="0" CELLPADDING="0" CELLSPACING="0"><TR><TD ALIGN="CENTER">' + escape(
kwargs.get("label", get_as_name(_as))).replace(
"\r", "<BR/>") + "</TD></TR></TABLE>>"
nodes[_as] = pydot.Node(_as,
style="filled",
fontsize="10",
**kwargs)
graph.add_node(nodes[_as])
return nodes[_as]
def add_edge(_previous_as, _as, **kwargs):
kwargs["splines"] = "true"
force = kwargs.get("force", False)
edge_tuple = (_previous_as, _as)
if force or edge_tuple not in edges:
edge = pydot.Edge(*edge_tuple, **kwargs)
graph.add_edge(edge)
edges[edge_tuple] = edge
elif "label" in kwargs and kwargs["label"]:
e = edges[edge_tuple]
label_without_star = kwargs["label"].replace("*", "")
if e.get_label() is not None:
labels = e.get_label().split("\r")
else:
return edges[edge_tuple]
if "%s*" % label_without_star not in labels:
labels = [kwargs["label"]] + [
l for l in labels if not l.startswith(label_without_star)
]
labels = sorted(labels,
cmp=lambda x, y: x.endswith("*") and -1 or 1)
label = escape("\r".join(labels))
e.set_label(label)
return edges[edge_tuple]
for host, asmaps in data.iteritems():
add_node(host,
label="%s\r%s" % (host.upper(), app.config["DOMAIN"].upper()),
shape="box",
fillcolor="#F5A9A9")
as_number = app.config["AS_NUMBER"].get(host, None)
if as_number:
node = add_node(as_number, fillcolor="#F5A9A9")
edge = add_edge(as_number, nodes[host])
edge.set_color("red")
edge.set_style("bold")
#colors = [ "#009e23", "#1a6ec1" , "#d05701", "#6f879f", "#939a0e", "#0e9a93", "#9a0e85", "#56d8e1" ]
previous_as = None
hosts = data.keys()
for host, asmaps in data.iteritems():
first = True
for asmap in asmaps:
previous_as = host
color = "#%x" % random.randint(0, 16777215)
hop = False
hop_label = ""
for _as in asmap:
if _as == previous_as:
if not prepend_as.get(_as, None):
prepend_as[_as] = {}
if not prepend_as[_as].get(host, None):
prepend_as[_as][host] = {}
if not prepend_as[_as][host].get(asmap[0], None):
prepend_as[_as][host][asmap[0]] = 1
prepend_as[_as][host][asmap[0]] += 1
continue
if not hop:
hop = True
if _as not in hosts:
hop_label = _as
if first:
hop_label = hop_label + "*"
continue
else:
hop_label = ""
if _as == asmap[-1]:
add_node(
_as,
fillcolor="#F5A9A9",
shape="box",
)
else:
add_node(
_as,
fillcolor=(first and "#F5A9A9" or "white"),
)
if hop_label:
edge = add_edge(nodes[previous_as],
nodes[_as],
label=hop_label,
fontsize="7")
else:
edge = add_edge(nodes[previous_as],
nodes[_as],
fontsize="7")
hop_label = ""
if first or _as == asmap[-1]:
edge.set_style("bold")
edge.set_color("red")
elif edge.get_style() != "bold":
edge.set_style("dashed")
edge.set_color(color)
previous_as = _as
first = False
for _as in prepend_as:
for n in set([
n for h, d in prepend_as[_as].iteritems()
for p, n in d.iteritems()
]):
graph.add_edge(
pydot.Edge(*(_as, _as),
label=" %dx" % n,
color="grey",
fontcolor="grey"))
fmt = request.args.get('fmt', 'png')
#response = Response("<pre>" + graph.create_dot() + "</pre>")
if fmt == "png":
response = Response(graph.create_png(), mimetype='image/png')
elif fmt == "svg":
response = Response(graph.create_svg(), mimetype='image/svg+xml')
else:
abort(400, "Incorrect format")
response.headers['Last-Modified'] = datetime.now()
response.headers[
'Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
response.headers['Pragma'] = 'no-cache'
response.headers['Expires'] = '-1'
return response
def model_to_dot(
model,
show_shapes=False,
show_layer_names=True,
rankdir='TB',
dpi=96,
transparent_bg=False,
):
"""Convert a Keras model to dot format.
# Arguments
model: A Keras model instance.
show_shapes: whether to display shape information.
show_layer_names: whether to display layer names.
rankdir: `rankdir` argument passed to PyDot,
a string specifying the format of the plot:
'TB' creates a vertical plot;
'LR' creates a horizontal plot.
# Returns
A `pydot.Dot` instance representing the Keras model.
"""
_check_pydot()
dot = pydot.Dot()
dot.set('rankdir', rankdir)
if transparent_bg:
dot.set('bgcolor', "#ffffff00")
dot.set('concentrate', True)
dot.set('dpi', dpi)
dot.set_node_defaults(shape='record')
if isinstance(model, Sequential):
if not model.built:
model.build()
layers = model.layers
# Create graph nodes.
for layer in layers:
layer_id = str(id(layer))
# Append a wrapped layer's label to node's label, if it exists.
layer_name = layer.name
class_name = layer.__class__.__name__
if isinstance(layer, Wrapper):
layer_name = '{}({})'.format(layer_name, layer.layer.name)
child_class_name = layer.layer.__class__.__name__
class_name = '{}({})'.format(class_name, child_class_name)
# Create node's label.
if show_layer_names:
label = '{}: {}'.format(layer_name, class_name)
else:
label = class_name
# Rebuild the label as a table including input/output shapes.
if show_shapes:
try:
outputlabels = str(layer.output_shape)
except AttributeError:
outputlabels = 'multiple'
if hasattr(layer, 'input_shape'):
inputlabels = str(layer.input_shape)
elif hasattr(layer, 'input_shapes'):
inputlabels = ', '.join(
[str(ishape) for ishape in layer.input_shapes])
else:
inputlabels = 'multiple'
label = '%s\n|{input:|output:}|{{%s}|{%s}}' % (label, inputlabels,
outputlabels)
if transparent_bg:
node_kwargs = {'style': 'filled', 'fillcolor': '#ffffffff'}
else:
node_kwargs = {}
node = pydot.Node(layer_id, label=label, **node_kwargs)
dot.add_node(node)
# Connect nodes with edges.
for layer in layers:
layer_id = str(id(layer))
for i, node in enumerate(layer._inbound_nodes):
node_key = layer.name + '_ib-' + str(i)
if node_key in model._network_nodes:
for inbound_layer in node.inbound_layers:
inbound_layer_id = str(id(inbound_layer))
# add node if inbound_layer_id node is not present
if not dot.get_node(inbound_layer_id):
dot.add_node(
pydot.Node(inbound_layer_id,
label='Input',
**node_kwargs))
dot.add_edge(pydot.Edge(inbound_layer_id, layer_id))
return dot
def test_executable_not_found_exception(self):
graph = pydot.Dot('graphname', graph_type='digraph')
self.assertRaises(Exception, graph.create, prog='dothehe')
