def get_top_layers(layer_name, pydot_graph):
# type: (str, pydot.Dot) -> List[XLayer]
"""
Retrieve the top layers for the given layer and pydot graph
! We have to handle graphs with and without blob layers in between
"""
node = pydot_graph.get_node(pydot.quote_if_necessary(layer_name))[0]
P = node.get('LayerParameter')
top_layers = []
if len(P.tops) == 1 and P.name in P.tops:
try:
node = pydot_graph.get_node(
pydot.quote_if_necessary(layer_name + '_blob'))[0]
P = node.get('LayerParameter')
except Exception as e:
print("Tops", P.tops)
print("P", P.name)
raise e
for top_name in P.tops:
try:
top_node = pydot_graph.get_node(
pydot.quote_if_necessary(top_name))[0]
top_layers.append(top_node.get('LayerParameter'))
except Exception as e:
print("Tops", P.tops)
print("P", P.name)
raise e
return top_layers
def visualize(self, outputfile):
# type: () -> None
""" Visualize this xgraph using pydot """
try:
from . import pydot_tools
import pydot
except ImportError:
raise ImportError("XGraph functionality depends on the 'pydot'"
" package. Please make sure that Pydot is"
" installed before trying to visualize XGraphs")
pdg = pydot.Dot(self.get_name(), graph_type='digraph', rankdir='BT')
cm_idx = 1
target_to_cm = {}
for X in self.get_layers():
pydot_attrs = copy.copy(pydot_tools.LAYER_STYLE_DEFAULT)
if 'Input' in X.type:
pydot_attrs["shape"] = "oval"
pydot_attrs["fillcolor"] = self.cm[0]
if X.target != 'cpu':
if X.target not in target_to_cm:
target_to_cm[X.target] = cm_idx
if cm_idx < (len(self.cm) - 1):
cm_idx += 1
pydot_attrs["fillcolor"] = self.cm[target_to_cm[X.target]]
# Add '-pdg' to fix issues of pydot with names with format
# '[...]:0' where ':0' gets removed
node = pydot.Node(pydot.quote_if_necessary(X.name + '-pdg'),
**pydot_attrs)
pdg.add_node(node)
for b in X.bottoms:
src_nodes = pdg.get_node(pydot.quote_if_necessary(b + '-pdg'))
if len(src_nodes) == 0:
raise ValueError(
"Pydot could not find layer with name: {}".format(b))
assert len(src_nodes) == 1
src_node = src_nodes[0]
edge_label = b + "->" + X.name
# logger.debug("--Add bottom edge: {}".format(edge_label))
pdg.add_edge(pydot.Edge(src_node, node, label=edge_label))
pydot_tools.visualize(pdg, outputfile)
def __init__(self, src, dst, attr):
obj_dict = dict()
obj_dict[ 'attributes' ] = attr
obj_dict[ 'type' ] = 'edge'
obj_dict[ 'parent_graph' ] = None
obj_dict[ 'parent_edge_list' ] = None
obj_dict[ 'sequence' ] = None
if isinstance(src, Node):
src = src.get_name()
if isinstance(dst, Node):
dst = dst.get_name()
points = ( quote_if_necessary( src) , quote_if_necessary( dst) )
obj_dict['points'] = points
Edge.__init__(self, src, dst, obj_dict)
def dot_string(self):
edge = []
edge.append(self.tail.ID)
if(self.tail_pos):
edge.append(':'+self.tail_pos)
edge.append('->')
edge.append(self.head.ID)
if(self.head_pos):
edge.append(':'+self.head_pos)
# the order in which attributes are appended is important for pacopt
# therefore the pydot library could not be used, excuse the following hacks
attr_list = ['label', 'debug']
edge_attr = []
for attr in attr_list:
if attr in self.obj_dict['attributes']:
value = self.obj_dict['attributes'][attr]
if value == '':
value = '""'
if value is not None:
edge_attr.append('%s=%s' % (attr, pydot.quote_if_necessary(value)))
edge_attr = ', '.join(edge_attr)
if edge_attr:
edge.append('[' + edge_attr + ']')
return ''.join(edge) + ';'
def remove_node(pydot_graph, node_name, bottom_Xs, top_Xs):
# type: (pydot.Dot, str, List[XLayer], List[XLayer]) -> pydot.Dot
"""
Remove a node from the provided pydot graph
TODO: test
"""
if not (len(bottom_Xs) == 1 or len(top_Xs) == 1):
raise ValueError("Undefined behaviour: can't remove a node if"
" there are multiple bottom nodes and multiple"
" top nodes")
node_name_q = pydot.quote_if_necessary(node_name)
pydot_graph.del_node(node_name_q)
for bX in bottom_Xs:
b_q = pydot.quote_if_necessary(bX.name)
pydot_graph.del_edge(b_q, node_name_q)
for tX in top_Xs:
t_q = pydot.quote_if_necessary(tX.name)
edge = pydot.Edge(b_q,
t_q,
label="{}->ID->{}".format(bX.name, tX.name))
pydot_graph.add_edge(edge)
new_tops = [([bXt] if bXt != node_name else [tX.name for tX in top_Xs])
for bXt in bX.tops]
# flatten
new_tops = [e for sl in new_tops for e in sl]
bX.tops = new_tops
for tX in top_Xs:
t_q = pydot.quote_if_necessary(tX.name)
pydot_graph.del_edge(node_name_q, t_q)
new_bottoms = [
([tXb] if tXb != node_name else [bX.name for bX in bottom_Xs])
for tXb in tX.bottoms
]
# flatten
new_bottoms = [e for sl in new_bottoms for e in sl]
tX.bottoms = new_bottoms
return pydot_graph
def get_bottom_layers(layer_name, pydot_graph):
# type: (str, pydot.Dot) -> List[XLayer]
"""
Retrieve the bottom layers for the given layer and pydot graph
"""
node = pydot_graph.get_node(pydot.quote_if_necessary(layer_name))[0]
P = node.get('LayerParameter')
bottom_layers = []
for bottom_name in P.bottoms:
try:
bottom_node = pydot_graph.get_node(
pydot.quote_if_necessary(bottom_name))[0]
bottom_layers.append(bottom_node.get('LayerParameter'))
except Exception as e:
print(P.bottoms)
print(bottom_name)
raise e
return bottom_layers
def insert_node(pydot_graph, node_name, node, bottom_Xs, top_Xs):
# type: (pydot.Dot, str, pydot.Node, List[XLayer], List[XLayer])
# -> pydot.Dot
"""
Insert a node in the provided pydot graph
TODO: test
"""
if not (len(bottom_Xs) == 1 and len(top_Xs) == 1):
raise ValueError("Undefined behaviour: can't insert a node if"
" there are multiple bottom nodes or multiple"
" top nodes")
n_q = pydot.quote_if_necessary(node_name)
pydot_graph.add_node(node)
# for bX in bottom_Xs:
bX = bottom_Xs[0]
b_q = pydot.quote_if_necessary(bX.name)
edge = pydot.Edge(b_q, n_q, label="{}->{}".format(bX.name, node_name))
pydot_graph.add_edge(edge)
# for tX in top_Xs:
tX = top_Xs[0]
t_q = pydot.quote_if_necessary(tX.name)
pydot_graph.del_edge(b_q, t_q)
new_tops = [(bXt if bXt != tX.name else node_name) for bXt in bX.tops]
bX.tops = new_tops
# for tX in top_Xs:
edge = pydot.Edge(n_q, t_q, label="{}->{}".format(node_name, tX.name))
pydot_graph.add_edge(edge)
new_bottoms = [(tXb if tXb != bX.name else node_name)
for tXb in tX.bottoms]
tX.bottoms = new_bottoms
return pydot_graph
def compute_ignore_nodes(self, nodes, graph):
ignore = set()
stk = nodes[:]
while len(stk) > 0:
node_name = stk.pop()
ignore.add(node_name)
g_node = graph.get_node(pydot.quote_if_necessary(node_name))
if len(g_node) > 0:
g_node = g_node[0]
params = g_node.get('LayerParameter')
if params.bottoms is not None:
stk += [inp for inp in params.bottoms if inp not in ignore]
return ignore
def compute_ignore_nodes(self, nodes, graph) :
ignore = set()
stk = nodes[:]
while len(stk) > 0 :
node_name = stk.pop()
ignore.add(node_name)
g_node = graph.get_node(pydot.quote_if_necessary(node_name))
if len(g_node) == 0 : continue
g_node = g_node[0]
params = g_node.get('LayerParameter')
if params.bottoms == None : continue
for inp in params.bottoms :
if inp not in ignore :
stk.append(inp)
return ignore
def draw_tree(self,
highlighted_concepts=None,
tree=None,
viz=None,
colors=None,
title=None):
"""
:param highlighted_concepts:
:param tree:
:return: A Pydot object
"""
if tree is None:
tree = self.tree
if colors is None:
colors = default_colors
p = nx.drawing.nx_pydot.to_pydot(tree)
if highlighted_concepts:
if not isinstance(highlighted_concepts[0], (list, tuple)):
highlighted_concepts = [highlighted_concepts]
# Give a color to each group of nodes
color_dict = {}
for i, concept_list in enumerate(highlighted_concepts):
highlighted_nodes = [c.descriptor for c in concept_list]
for node in highlighted_nodes:
# quote_if_necessary is required to handle names in the same
# way as pydot
color_dict[quote_if_necessary(node)] = colors[i %
len(colors)]
for node in p.get_nodes():
if node.obj_dict['name'] in color_dict:
node.set_style('filled')
node.set_fillcolor(color_dict[node.obj_dict['name']])
width = 18
p.set_size('{}x1'.format(width))
if viz:
svg_bytes = p.create_svg()
soup = bs4.BeautifulSoup(svg_bytes, 'xml')
width = float(soup.svg.attrs['width'][:-2]) * 1.35
height = float(soup.svg.attrs['height'][:-2]) * 1.4
viz.svg(svgstr=str(svg_bytes),
opts=dict(title=title, width=width, height=height))
return p
def node_id(self, p):
"""
Construct unique node-id for the given particle
"""
if not p: return ''
## construct some unique name
nid = '%s:%s:#%d' % (p.name(), p.hex_id(), p.key())
## container
cnt = p.parent()
if cnt:
r = cnt.registry()
if r: nid = '%s:%s:#%d' % (p.name(), r.identifier(), p.key())
## massage the name:
return pydot.quote_if_necessary(nid)
def add_tree(self, p):
nid = self.node_id(p)
if nid in self._nodes: return nid
# create new node
label = pydot.quote_if_necessary(p.name())
node = pydot.Node(name=nid, label=label, **node_attributes)
self._nodes.add(nid)
self._graph.add_node(node)
for c in p.children():
nidc = self.add_tree(c) ## NB: recursion here
edge = nid, nidc
if not edge in self._edges:
self._graph.add_edge(pydot.Edge(
*edge, **edge_attributes)) ## create edge
self._edges.add(edge)
return nid
def node_id ( self , p ) :
if not p : return ''
nid = '%s:%s:#%d/%d' % ( p.name () , p.hex_id() , p.barcode() , p.status() )
return pydot.quote_if_necessary( nid )
def doLayout(self):
dc = wx.ClientDC(self)
self.PrepareDC(dc)
# intermediate fix: we need the new pydot version for this to
# work, but this will not work with python 2.5.x, so
# we keep the old version (including the need for pygraphviz)
# side-by-side to the new version. Note that we do only check
# for the python version, and not for the platform or else
# as python 2.5.x on a windows platform will most likely fail
# to do the layout in any case
if sys.version_info[0] >= 2 and sys.version_info[1] >= 6:
# create a graph on the level of pydot
G = pydot.Graph("G", graph_type='digraph', strict=False)
for nd in self.shapes.keys(): #iterate over all shapes
shp = self.shapes[nd]
# dot does not accept all names, so we escape the
# name using pydots routines for that
nd = pydot.quote_if_necessary(nd)
# add a node, specify outer rectangle as bounding box to respect
# during the layout, use fixedsize to state this cleanly
# we convert from pixels to inches, as dotty seems to do
# all layouts in inches, for whatever reason
node = pydot.Node(str(nd), shape='box', fixedsize='true', width=str(shp.GetWidth()/72), height=str(shp.GetHeight()/72))
# add this node to the set of nodes on the pydot graph
G.add_node(node)
# now create some edges in this graph
for ed in self.graph.edges[:]:
# note that we changed the name of the nodes during the
# upper loop, and this is what we have to respect here:
# convert/escape any name correctly
# we try to recover this node from the pydot graph using its
# escaped name
nd_from = G.get_node( pydot.quote_if_necessary(str(ed.getFrom()[0].name)) )
nd_to = G.get_node( pydot.quote_if_necessary(str(ed.getTo()[0].name)) )
# can be a list-type in case we have more than one node
# having the same name (how can that happen?
# we ignore this case, as here this is probably an error
# so just progress with the node when we have a non
# list type for either the source or the sink of this edge
if type(nd_from) is not ListType and type(nd_to) is not ListType:
Aedge = pydot.Edge(nd_from, nd_to)
G.add_edge(Aedge)
# ok, done, do the layout now
try:
# create a temp file, but do not delete it (we need
# its file name), sidenode: the delete=False option is new
# to python 2.6, it will not work with python 2.5.x
fd = tempfile.NamedTemporaryFile(delete=False)
fd.write(G.to_string())
fd.close() # does not delete it, so the file name is still ok
fd_name = fd.name
# same here for the output file
fd_out = tempfile.NamedTemporaryFile(delete=False)
fd_out.close()
fd_out_name = fd_out.name
# get the dot executable from the cached set of
# dotty executables
dot_exe = self.dot["dot"]
if dot_exe <> None:
# construct command line, using dot as output format
# again (we need the position and not much more)
dot_exe = dot_exe + " -Tdot -o "+fd_out_name+" "+fd_name
# execute
print(dot_exe)
os.system(dot_exe)
# re-parse the file again, overwrite old graph
G = pydot.graph_from_dot_file(fd_out_name)
# cleanup
os.unlink(fd_name)
# cleanup
os.unlink(fd_out_name)
except IOError:
print "I/O error on layout. Aborting."
return
# if the above failed, we might not be here (have to test that)
# if it did not fail, we have a bounding box
# G.get_bb() will deliver "x,y,w,h" as a string, including the
# quotes, that is why we strip them first (tail and head)
bb = string.split(string.strip(G.get_bb(),'\"'), ',', 4)
# w,h are in bb[2] and bb[3] respectively
self.ResizePane(int(bb[2]), int(bb[3]))
# now set the positions for all the nodes we have
for nd in G.get_node_list()[:]:
# get pos attrib
# for some reason, we can get a Node object as nd, which
# is not a regular node of our graph. Is this internal to
# pydot or a parsing error? However, we just check whether
# we got a "pos" attrib as a result of the dot exec
pos = nd.get('pos')
if pos <> None:
# yes, this is the string "x,y" (including quotes)
a = string.split( string.strip(pos, '\"'), ',', 2)
# move from (0,0) to their layout position
# note the shift in the y-component related to bb[3] (ymax)
# move seems to be an absolute API, so not translate
try:
# we now need the name in the graph. here, we
# rely on the quote escaping done by pydot...
# maybe there is a method to "unescape" a name again?
# could be the source of an error
nd_name = string.strip(nd.get_name(), '\"')
# can fail when we did not hit the right name
# note that we have to "flip" the y coordinate,
# as dotty doe not only calc in inch, but also from
# bottom to top
self.shapes[nd_name].Move(dc, int(a[0]), int(bb[3])-int(a[1]), display=False)
except KeyError:
print "key error: node ["+str(nd)+"] not found in shapes?"
else: # python 2.5.x
import pygraphviz as pgv
# create an empty graph first
G = pgv.AGraph(directed=True,strict=True)
for nd in self.shapes.keys():
shp = self.shapes[nd]
G.add_node(str(nd))
ANd = G.get_node(str(nd))
ANd.attr['shape'] = 'box'
ANd.attr['fixedsize'] = 'true'
ANd.attr['width'] = str(shp.GetWidth()/72)
ANd.attr['height'] = str(shp.GetHeight()/72)
for ed in self.graph.edges[:]:
Aedge = G.add_edge( str(ed.getFrom()[0].name), str(ed.getTo()[0].name) )
try:
G.layout(prog='dot',args='-q1')
except IOError:
print "I/O error on layout. Aborting."
return
# dotty seems to layout as American, as can get:
# y points up, node sizes are in inches and so on
# assumed is a 72dot per inch scaling
# so what we need it to
# - 'flip' the nodes (we layout from top to bottom)
# - recenter the graph (we need the bb for that)
# unfortunately, pygraphviz does not seem to offer the
# bb property of the graph (it IS written to the dot!)
# so we iterate over all nodes and use the max of the
# x and y vars (related to border, not to center) as
# an approximate of the bounding box we will have for
# the graph
xmax, ymax = 0,0
for nd in G.nodes():
pos = nd.attr['pos']
w = (float(nd.attr['width']) * 72) / 2 # in inch, calc 72dots per inch, calculated from center
h = (float(nd.attr['height']) * 72) / 2 # in inch, calc 72dots per inch, calculated from center
a = string.split(pos,',',2)
xmax = max(int(a[0])+w, xmax)
ymax = max(int(a[1])+h, ymax)
# resize canvas
self.ResizePane(xmax, ymax)
for nd in G.nodes():
pos = nd.attr['pos']
a = string.split(pos,',',2)
# move from (0,0) to their layout position
# note the shift in the y-component related to ymax
# move seems to be an absolute API, so not translate
try:
self.shapes[nd].Move(dc, int(a[0]), ymax-int(a[1]))
except KeyError:
print "key error: node ["+str(nd)+"] not found in shapes?"
pass
def escape(s):
"""Escape a string so it can be use in a dot label."""
return pydot.quote_if_necessary(s).replace('<','\\<').replace('>', '\\>').replace("'", "\\'")
def EG_to_string(self, indent=0, root_graph=None):
"""Returns a string representation of the graph in dot language.
This version try to make string looking better than to_string().
"""
idt = ' ' * (4 + indent)
graph = list()
if root_graph is None:
root_graph = self
if root_graph != root_graph.get_parent_graph():
graph.append(' ' * indent)
if root_graph.obj_dict.get('strict', None) is not None:
if root_graph == root_graph.get_parent_graph(
) and root_graph.obj_dict['strict']:
graph.append('strict ')
if root_graph.obj_dict['name'] == '':
graph.append('{\n')
else:
graph.append(
'%s %s {\n' %
(root_graph.obj_dict['type'], root_graph.obj_dict['name']))
for attr in root_graph.obj_dict['attributes'].keys():
if root_graph.obj_dict['attributes'].get(attr, None) is not None:
graph.append(idt + '%s=' % attr)
val = root_graph.obj_dict['attributes'].get(attr)
graph.append(pydot.quote_if_necessary(val))
graph.append(';\n')
edges_done = set()
edge_obj_dicts = list()
for e in root_graph.obj_dict['edges'].values():
edge_obj_dicts.extend(e)
if edge_obj_dicts:
edge_src_set, edge_dst_set = zip(
*[obj['points'] for obj in edge_obj_dicts])
edge_src_set, edge_dst_set = set(edge_src_set), set(edge_dst_set)
else:
edge_src_set, edge_dst_set = set(), set()
node_obj_dicts = list()
for e in root_graph.obj_dict['nodes'].values():
node_obj_dicts.extend(e)
sgraph_obj_dicts = list()
for sg in root_graph.obj_dict['subgraphs'].values():
sgraph_obj_dicts.extend(sg)
obj_list = [
(obj['sequence'], obj)
for obj in (edge_obj_dicts + node_obj_dicts + sgraph_obj_dicts)
]
obj_list.sort()
for _, obj in obj_list:
if obj['type'] == 'node':
node = pydot.Node(obj_dict=obj)
if root_graph.obj_dict.get('suppress_disconnected', False):
if (node.get_name() not in edge_src_set
and node.get_name() not in edge_dst_set):
continue
graph.append(
DEUtils.smart_indent(node.to_string(), idt) + '\n')
elif obj['type'] == 'edge':
edge = pydot.Edge(obj_dict=obj)
if root_graph.obj_dict.get('simplify',
False) and edge in edges_done:
continue
graph.append(
DEUtils.smart_indent(edge.to_string(), idt) + '\n')
edges_done.add(edge)
else:
sgraph = pydot.Subgraph(obj_dict=obj)
sg_str = self.EG_to_string(indent + 4, sgraph)
graph.append(sg_str + '\n')
if root_graph != root_graph.get_parent_graph():
graph.append(' ' * indent)
graph.append('}\n')
return ''.join(graph)
def EG_to_string(self, indent=0, root_graph=None):
"""Returns a string representation of the graph in dot language.
This version try to make string looking better than to_string().
"""
idt = " " * (4 + indent)
graph = list()
if root_graph is None:
root_graph = self
if root_graph != root_graph.get_parent_graph():
graph.append(" " * indent)
if root_graph.obj_dict.get("strict", None) is not None:
if root_graph == root_graph.get_parent_graph() and root_graph.obj_dict["strict"]:
graph.append("strict ")
if root_graph.obj_dict["name"] == "":
graph.append("{\n")
else:
graph.append("%s %s {\n" % (root_graph.obj_dict["type"], root_graph.obj_dict["name"]))
for attr in root_graph.obj_dict["attributes"].keys():
if root_graph.obj_dict["attributes"].get(attr, None) is not None:
graph.append(idt + "%s=" % attr)
val = root_graph.obj_dict["attributes"].get(attr)
graph.append(pydot.quote_if_necessary(val))
graph.append(";\n")
edges_done = set()
edge_obj_dicts = list()
for e in root_graph.obj_dict["edges"].values():
edge_obj_dicts.extend(e)
if edge_obj_dicts:
edge_src_set, edge_dst_set = zip(*[obj["points"] for obj in edge_obj_dicts])
edge_src_set, edge_dst_set = set(edge_src_set), set(edge_dst_set)
else:
edge_src_set, edge_dst_set = set(), set()
node_obj_dicts = list()
for e in root_graph.obj_dict["nodes"].values():
node_obj_dicts.extend(e)
sgraph_obj_dicts = list()
for sg in root_graph.obj_dict["subgraphs"].values():
sgraph_obj_dicts.extend(sg)
obj_list = [(obj["sequence"], obj) for obj in (edge_obj_dicts + node_obj_dicts + sgraph_obj_dicts)]
obj_list.sort()
for _, obj in obj_list:
if obj["type"] == "node":
node = pydot.Node(obj_dict=obj)
if root_graph.obj_dict.get("suppress_disconnected", False):
if node.get_name() not in edge_src_set and node.get_name() not in edge_dst_set:
continue
graph.append(DEUtils.smart_indent(node.to_string(), idt) + "\n")
elif obj["type"] == "edge":
edge = pydot.Edge(obj_dict=obj)
if root_graph.obj_dict.get("simplify", False) and edge in edges_done:
continue
graph.append(DEUtils.smart_indent(edge.to_string(), idt) + "\n")
edges_done.add(edge)
else:
sgraph = pydot.Subgraph(obj_dict=obj)
sg_str = self.EG_to_string(indent + 4, sgraph)
graph.append(sg_str + "\n")
if root_graph != root_graph.get_parent_graph():
graph.append(" " * indent)
graph.append("}\n")
return "".join(graph)