def simplify_graph(infr, graph=None, copy=True):
if graph is None:
graph = infr.graph
simple = graph.copy() if copy else graph
ut.nx_delete_edge_attr(simple, infr.visual_edge_attrs)
ut.nx_delete_node_attr(simple, infr.visual_node_attrs + ['pin'])
return simple
def apply_match_scores(infr):
"""
Applies precomputed matching scores to edges that already exist in the
graph. Typically you should run infr.apply_match_edges() before running
this.
CommandLine:
python -m wbia.algo.graph.core apply_match_scores --show
Example:
>>> # xdoctest: +REQUIRES(--slow)
>>> # ENABLE_DOCTEST
>>> from wbia.algo.graph.core import * # NOQA
>>> infr = testdata_infr('PZ_MTEST')
>>> infr.exec_matching()
>>> infr.apply_match_edges()
>>> infr.apply_match_scores()
>>> infr.get_edge_attrs('score')
"""
if infr.cm_list is None:
infr.print('apply_match_scores - no scores to apply!')
return
infr.print('apply_match_scores', 1)
edges = list(infr.graph.edges())
edge_to_data = infr._get_cm_edge_data(edges)
# Remove existing attrs
ut.nx_delete_edge_attr(infr.graph, 'score')
ut.nx_delete_edge_attr(infr.graph, 'rank')
ut.nx_delete_edge_attr(infr.graph, 'normscore')
edges = list(edge_to_data.keys())
edge_scores = list(ut.take_column(edge_to_data.values(), 'score'))
edge_scores = ut.replace_nones(edge_scores, np.nan)
edge_scores = np.array(edge_scores)
edge_ranks = np.array(ut.take_column(edge_to_data.values(), 'rank'))
# take the inf-norm
normscores = edge_scores / vt.safe_max(edge_scores, nans=False)
# Add new attrs
infr.set_edge_attrs('score', ut.dzip(edges, edge_scores))
infr.set_edge_attrs('rank', ut.dzip(edges, edge_ranks))
# Hack away zero probabilites
# probs = np.vstack([p_nomatch, p_match, p_notcomp]).T + 1e-9
# probs = vt.normalize(probs, axis=1, ord=1, out=probs)
# entropy = -(np.log2(probs) * probs).sum(axis=1)
infr.set_edge_attrs('normscore', dict(zip(edges, normscores)))
def draw_em_graph(P, Pn, PL, gam, num_labels):
"""
python -m ibeis.algo.hots.testem test_em --show --no-cnn
"""
num_labels = PL.shape[1]
name_nodes = ['N%d' % x for x in list(range(1, num_labels + 1))]
annot_nodes = ['X%d' % x for x in list(range(1, len(Pn) + 1))]
nodes = name_nodes + annot_nodes
PL2 = gam[:, num_labels:].T
PL2 += .01
PL2 = PL2 / PL2.sum(axis=1)[:, None]
# PL2 = PL2 / np.linalg.norm(PL2, axis=0)
zero_part = np.zeros((num_labels, len(Pn) + num_labels))
prob_part = np.hstack([PL2, Pn])
print(ut.hz_str(' PL2 = ', ut.repr2(PL2, precision=2)))
# Redo p with posteriors
if ut.get_argflag('--postem'):
P = np.vstack([zero_part, prob_part])
weight_matrix = P # NOQA
graph = ut.nx_from_matrix(P, nodes=nodes)
graph = graph.to_directed()
# delete graph
dup_edges = []
seen_ = set([])
for u, v in graph.edges():
if u < v:
u, v = v, u
if (u, v) not in seen_:
seen_.add((u, v))
else:
dup_edges.append((u, v))
graph.remove_edges_from(dup_edges)
import plottool_ibeis as pt
import networkx as nx
if len(name_nodes) == 3 and len(annot_nodes) == 4:
graph.nodes[annot_nodes[0]]['pos'] = (20., 200.)
graph.nodes[annot_nodes[1]]['pos'] = (220., 200.)
graph.nodes[annot_nodes[2]]['pos'] = (20., 100.)
graph.nodes[annot_nodes[3]]['pos'] = (220., 100.)
graph.nodes[name_nodes[0]]['pos'] = (10., 300.)
graph.nodes[name_nodes[1]]['pos'] = (120., 300.)
graph.nodes[name_nodes[2]]['pos'] = (230., 300.)
nx.set_node_attributes(graph, name='pin', values='true')
print('annot_nodes = %r' % (annot_nodes, ))
print('name_nodes = %r' % (name_nodes, ))
for u in annot_nodes:
for v in name_nodes:
if graph.has_edge(u, v):
print('1) u, v = %r' % ((u, v), ))
graph.edge[u][v]['taillabel'] = graph.edge[u][v]['label']
graph.edge[u][v]['color'] = pt.ORANGE
graph.edge[u][v]['labelcolor'] = pt.BLUE
del graph.edge[u][v]['label']
elif graph.has_edge(v, u):
print('2) u, v = %r' % ((u, v), ))
graph.edge[v][u]['headlabel'] = graph.edge[v][u]['label']
graph.edge[v][u]['color'] = pt.ORANGE
graph.edge[v][u]['labelcolor'] = pt.BLUE
del graph.edge[v][u]['label']
else:
print((u, v))
print('!!')
# import itertools
# name_const_edges = [(u, v, {'style': 'invis'}) for u, v in itertools.combinations(name_nodes, 2)]
# graph.add_edges_from(name_const_edges)
# nx.set_edge_attributes(graph, name='constraint', values={edge: False for edge in graph.edges() if edge[0] == 'b' or edge[1] == 'b'})
# nx.set_edge_attributes(graph, name='constraint', values={edge: False for edge in graph.edges() if edge[0] in annot_nodes and edge[1] in annot_nodes})
# nx.set_edge_attributes(graph, name='constraint', values={edge: True for edge in graph.edges() if edge[0] in name_nodes or edge[1] in name_nodes})
# nx.set_edge_attributes(graph, name='constraint', values={edge: True for edge in graph.edges() if (edge[0] in ['a', 'b'] and edge[1] in ['a', 'b']) and edge[0] in annot_nodes and edge[1] in annot_nodes})
# nx.set_edge_attributes(graph, name='constraint', values={edge: True for edge in graph.edges() if (edge[0] in ['c'] or edge[1] in ['c']) and edge[0] in annot_nodes and edge[1] in annot_nodes})
# nx.set_edge_attributes(graph, name='constraint', values={edge: True for edge in graph.edges() if (edge[0] in ['a'] or edge[1] in ['a']) and edge[0] in annot_nodes and edge[1] in annot_nodes})
# nx.set_edge_attributes(graph, name='constraint', values={edge: True for edge in graph.edges() if (edge[0] in ['b'] or edge[1] in ['b']) and edge[0] in annot_nodes and edge[1] in annot_nodes})
# graph.add_edges_from([('root', n) for n in nodes])
# {node: 'names' for node in name_nodes})
nx.set_node_attributes(graph,
name='color',
values={node: pt.RED
for node in name_nodes})
# nx.set_node_attributes(graph, name='width', values={node: 20 for node in nodes})
# nx.set_node_attributes(graph, name='height', values={node: 20 for node in nodes})
#nx.set_node_attributes(graph, name='group', values={node: 'names' for node in name_nodes})
#nx.set_node_attributes(graph, name='group', values={node: 'annots' for node in annot_nodes})
nx.set_node_attributes(graph,
name='groupid',
values={node: 'names'
for node in name_nodes})
nx.set_node_attributes(graph,
name='groupid',
values={node: 'annots'
for node in annot_nodes})
graph.graph['clusterrank'] = 'local'
# graph.graph['groupattrs'] = {
# 'names': {'rankdir': 'LR', 'rank': 'source'},
# 'annots': {'rankdir': 'TB', 'rank': 'source'},
# }
ut.nx_delete_edge_attr(graph, 'weight')
# pt.show_nx(graph, fontsize=10, layoutkw={'splines': 'spline', 'prog': 'dot', 'sep': 2.0}, verbose=1)
layoutkw = {
# 'rankdir': 'LR',
'splines': 'spline',
# 'splines': 'ortho',
# 'splines': 'curved',
# 'compound': 'True',
# 'prog': 'dot',
'prog': 'neato',
# 'packMode': 'clust',
# 'sep': 4,
# 'nodesep': 1,
# 'ranksep': 1,
}
#pt.show_nx(graph, fontsize=12, layoutkw=layoutkw, verbose=0, as_directed=False)
pt.show_nx(graph,
fontsize=6,
fontname='Ubuntu',
layoutkw=layoutkw,
verbose=0,
as_directed=False)
pt.interactions.zoom_factory()
def update_visual_attrs(infr,
graph=None,
show_reviewed_edges=True,
show_unreviewed_edges=False,
show_inferred_diff=True,
show_inferred_same=True,
show_recent_review=False,
highlight_reviews=True,
show_inconsistency=True,
wavy=False,
simple_labels=False,
show_labels=True,
reposition=True,
use_image=False,
edge_overrides=None,
node_overrides=None,
colorby='name_label',
**kwargs
# hide_unreviewed_inferred=True
):
import wbia.plottool as pt
infr.print('update_visual_attrs', 3)
if graph is None:
graph = infr.graph
# if hide_cuts is not None:
# # show_unreviewed_cuts = not hide_cuts
# show_reviewed_cuts = not hide_cuts
if not getattr(infr, '_viz_init_nodes', False):
infr._viz_init_nodes = True
nx.set_node_attributes(graph, name='shape', values='circle')
# infr.set_node_attrs('shape', 'circle')
if getattr(infr, '_viz_image_config_dirty', True):
infr.update_node_image_attribute(graph=graph, use_image=use_image)
def get_any(dict_, keys, default=None):
for key in keys:
if key in dict_:
return dict_[key]
return default
show_cand = get_any(
kwargs, ['show_candidate_edges', 'show_candidates', 'show_cand'])
if show_cand is not None:
show_cand = True
show_reviewed_edges = True
show_unreviewed_edges = True
show_inferred_diff = True
show_inferred_same = True
if kwargs.get('show_all'):
show_cand = True
# alpha_low = .5
alpha_med = 0.9
alpha_high = 1.0
dark_background = graph.graph.get('dark_background', None)
# Ensure we are starting from a clean slate
# if reposition:
ut.nx_delete_edge_attr(graph, infr.visual_edge_attrs_appearance)
# Set annotation node labels
node_to_nid = None
if not show_labels:
nx.set_node_attributes(graph,
name='label',
values=ut.dzip(graph.nodes(), ['']))
else:
if simple_labels:
nx.set_node_attributes(
graph,
name='label',
values={n: str(n)
for n in graph.nodes()})
else:
if node_to_nid is None:
node_to_nid = nx.get_node_attributes(graph, 'name_label')
node_to_view = nx.get_node_attributes(graph, 'viewpoint')
if node_to_view:
annotnode_to_label = {
aid: 'aid=%r%s\nnid=%r' %
(aid, node_to_view[aid], node_to_nid[aid])
for aid in graph.nodes()
}
else:
annotnode_to_label = {
aid: 'aid=%r\nnid=%r' % (aid, node_to_nid[aid])
for aid in graph.nodes()
}
nx.set_node_attributes(graph,
name='label',
values=annotnode_to_label)
# NODE_COLOR: based on name_label
ut.color_nodes(graph,
labelattr=colorby,
outof=kwargs.get('outof', None),
sat_adjust=-0.4)
# EDGES:
# Grab different types of edges
edges, edge_colors = infr.get_colored_edge_weights(
graph, highlight_reviews)
# reviewed_states = nx.get_edge_attributes(graph, 'evidence_decision')
reviewed_states = {
e: infr.edge_decision(e)
for e in infr.graph.edges()
}
edge_to_inferred_state = nx.get_edge_attributes(
graph, 'inferred_state')
# dummy_edges = [edge for edge, flag in
# nx.get_edge_attributes(graph, '_dummy_edge').items()
# if flag]
edge_to_reviewid = nx.get_edge_attributes(graph, 'review_id')
recheck_edges = [
edge for edge, split in nx.get_edge_attributes(
graph, 'maybe_error').items() if split
]
decision_to_edge = ut.group_pairs(reviewed_states.items())
neg_edges = decision_to_edge[NEGTV]
pos_edges = decision_to_edge[POSTV]
incomp_edges = decision_to_edge[INCMP]
unreviewed_edges = decision_to_edge[UNREV]
inferred_same = [
edge for edge, state in edge_to_inferred_state.items()
if state == 'same'
]
inferred_diff = [
edge for edge, state in edge_to_inferred_state.items()
if state == 'diff'
]
inconsistent_external = [
edge for edge, state in edge_to_inferred_state.items()
if state == 'inconsistent_external'
]
inferred_notcomp = [
edge for edge, state in edge_to_inferred_state.items()
if state == 'notcomp'
]
reviewed_edges = incomp_edges + pos_edges + neg_edges
compared_edges = pos_edges + neg_edges
uncompared_edges = ut.setdiff(edges, compared_edges)
nontrivial_inferred_same = ut.setdiff(
inferred_same, pos_edges + neg_edges + incomp_edges)
nontrivial_inferred_diff = ut.setdiff(
inferred_diff, pos_edges + neg_edges + incomp_edges)
nontrivial_inferred_edges = nontrivial_inferred_same + nontrivial_inferred_diff
# EDGE_COLOR: based on edge_weight
nx.set_edge_attributes(graph,
name='color',
values=ut.dzip(edges, edge_colors))
# LINE_WIDTH: based on review_state
# unreviewed_width = 2.0
# reviewed_width = 5.0
unreviewed_width = 1.0
reviewed_width = 2.0
if highlight_reviews:
nx.set_edge_attributes(
graph,
name='linewidth',
values=ut.dzip(reviewed_edges, [reviewed_width]),
)
nx.set_edge_attributes(
graph,
name='linewidth',
values=ut.dzip(unreviewed_edges, [unreviewed_width]),
)
else:
nx.set_edge_attributes(graph,
name='linewidth',
values=ut.dzip(edges, [unreviewed_width]))
# EDGE_STROKE: based on decision and maybe_error
# fg = pt.WHITE if dark_background else pt.BLACK
# nx.set_edge_attributes(graph, name='stroke', values=ut.dzip(reviewed_edges, [{'linewidth': 3, 'foreground': fg}]))
if show_inconsistency:
nx.set_edge_attributes(
graph,
name='stroke',
values=ut.dzip(recheck_edges, [{
'linewidth': 5,
'foreground': infr._error_color
}]),
)
# Set linestyles to emphasize PCCs
# Dash lines between PCCs inferred to be different
nx.set_edge_attributes(graph,
name='linestyle',
values=ut.dzip(inferred_diff, ['dashed']))
# Treat incomparable/incon-external inference as different
nx.set_edge_attributes(graph,
name='linestyle',
values=ut.dzip(inferred_notcomp, ['dashed']))
nx.set_edge_attributes(graph,
name='linestyle',
values=ut.dzip(inconsistent_external,
['dashed']))
# Dot lines that we are unsure of
nx.set_edge_attributes(graph,
name='linestyle',
values=ut.dzip(unreviewed_edges, ['dotted']))
# Cut edges are implicit and dashed
# nx.set_edge_attributes(graph, name='implicit', values=ut.dzip(cut_edges, [True]))
# nx.set_edge_attributes(graph, name='linestyle', values=ut.dzip(cut_edges, ['dashed']))
# nx.set_edge_attributes(graph, name='alpha', values=ut.dzip(cut_edges, [alpha_med]))
nx.set_edge_attributes(graph,
name='implicit',
values=ut.dzip(uncompared_edges, [True]))
# Only matching edges should impose constraints on the graph layout
nx.set_edge_attributes(graph,
name='implicit',
values=ut.dzip(neg_edges, [True]))
nx.set_edge_attributes(graph,
name='alpha',
values=ut.dzip(neg_edges, [alpha_med]))
nx.set_edge_attributes(graph,
name='implicit',
values=ut.dzip(incomp_edges, [True]))
nx.set_edge_attributes(graph,
name='alpha',
values=ut.dzip(incomp_edges, [alpha_med]))
# Ensure reviewed edges are visible
nx.set_edge_attributes(graph,
name='implicit',
values=ut.dzip(reviewed_edges, [False]))
nx.set_edge_attributes(graph,
name='alpha',
values=ut.dzip(reviewed_edges, [alpha_high]))
if True:
# Infered same edges can be allowed to constrain in order
# to make things look nice sometimes
nx.set_edge_attributes(graph,
name='implicit',
values=ut.dzip(inferred_same, [False]))
nx.set_edge_attributes(graph,
name='alpha',
values=ut.dzip(inferred_same, [alpha_high]))
if not kwargs.get('show_same', True):
nx.set_edge_attributes(graph,
name='alpha',
values=ut.dzip(inferred_same, [0]))
if not kwargs.get('show_diff', True):
nx.set_edge_attributes(graph,
name='alpha',
values=ut.dzip(inferred_diff, [0]))
if not kwargs.get('show_positive_edges', True):
nx.set_edge_attributes(graph,
name='alpha',
values=ut.dzip(pos_edges, [0]))
if not kwargs.get('show_negative_edges', True):
nx.set_edge_attributes(graph,
name='alpha',
values=ut.dzip(neg_edges, [0]))
if not kwargs.get('show_incomparable_edges', True):
nx.set_edge_attributes(graph,
name='alpha',
values=ut.dzip(incomp_edges, [0]))
if not kwargs.get('show_between', True):
if node_to_nid is None:
node_to_nid = nx.get_node_attributes(graph, 'name_label')
between_edges = [(u, v) for u, v in edges
if node_to_nid[u] != node_to_nid[v]]
nx.set_edge_attributes(graph,
name='alpha',
values=ut.dzip(between_edges, [0]))
# SKETCH: based on inferred_edges
# Make inferred edges wavy
if wavy:
# dict(scale=3.0, length=18.0, randomness=None)]
nx.set_edge_attributes(
graph,
name='sketch',
values=ut.dzip(
nontrivial_inferred_edges,
[dict(scale=10.0, length=64.0, randomness=None)],
),
)
# Make dummy edges more transparent
# nx.set_edge_attributes(graph, name='alpha', values=ut.dzip(dummy_edges, [alpha_low]))
selected_edges = kwargs.pop('selected_edges', None)
# SHADOW: based on most recent
# Increase visibility of nodes with the most recently changed timestamp
if show_recent_review and edge_to_reviewid and selected_edges is None:
review_ids = list(edge_to_reviewid.values())
recent_idxs = ut.argmax(review_ids, multi=True)
recent_edges = ut.take(list(edge_to_reviewid.keys()), recent_idxs)
selected_edges = recent_edges
if selected_edges is not None:
# TODO: add photoshop-like parameters like
# spread and size. offset is the same as angle and distance.
nx.set_edge_attributes(
graph,
name='shadow',
values=ut.dzip(
selected_edges,
[{
'rho': 0.3,
'alpha': 0.6,
'shadow_color': 'w' if dark_background else 'k',
'offset': (0, 0),
'scale': 3.0,
}],
),
)
# Z_ORDER: make sure nodes are on top
nodes = list(graph.nodes())
nx.set_node_attributes(graph,
name='zorder',
values=ut.dzip(nodes, [10]))
nx.set_edge_attributes(graph,
name='zorder',
values=ut.dzip(edges, [0]))
nx.set_edge_attributes(graph,
name='picker',
values=ut.dzip(edges, [10]))
# VISIBILITY: Set visibility of edges based on arguments
if not show_reviewed_edges:
infr.print('Making reviewed edges invisible', 10)
nx.set_edge_attributes(graph,
name='style',
values=ut.dzip(reviewed_edges, ['invis']))
if not show_unreviewed_edges:
infr.print('Making un-reviewed edges invisible', 10)
nx.set_edge_attributes(graph,
name='style',
values=ut.dzip(unreviewed_edges, ['invis']))
if not show_inferred_same:
infr.print('Making nontrivial_same edges invisible', 10)
nx.set_edge_attributes(graph,
name='style',
values=ut.dzip(nontrivial_inferred_same,
['invis']))
if not show_inferred_diff:
infr.print('Making nontrivial_diff edges invisible', 10)
nx.set_edge_attributes(graph,
name='style',
values=ut.dzip(nontrivial_inferred_diff,
['invis']))
if selected_edges is not None:
# Always show the most recent review (remove setting of invis)
# infr.print('recent_edges = %r' % (recent_edges,))
nx.set_edge_attributes(graph,
name='style',
values=ut.dzip(selected_edges, ['']))
if reposition:
# LAYOUT: update the positioning layout
def get_layoutkw(key, default):
return kwargs.get(key, graph.graph.get(key, default))
layoutkw = dict(
prog='neato',
splines=get_layoutkw('splines', 'line'),
fontsize=get_layoutkw('fontsize', None),
fontname=get_layoutkw('fontname', None),
sep=10 / 72,
esep=1 / 72,
nodesep=0.1,
)
layoutkw.update(kwargs)
# logger.info(ut.repr3(graph.edges))
pt.nx_agraph_layout(graph, inplace=True, **layoutkw)
if edge_overrides:
for key, edge_to_attr in edge_overrides.items():
nx.set_edge_attributes(graph, name=key, values=edge_to_attr)
if node_overrides:
for key, node_to_attr in node_overrides.items():
nx.set_node_attributes(graph, name=key, values=node_to_attr)
def make_expanded_input_graph(graph, target):
"""
Starting from the `target` property we trace all possible paths in the
`graph` back to all sources.
Args:
graph (nx.DiMultiGraph): the dependency graph with a single source.
target (str): a single target node in graph
Notes:
Each edge in the graph must have a `local_input_id` that defines the
type of edge it is: (eg one-to-many, one-to-one, nwise/multi).
# Step 1: Extracting the Relevant Subgraph
We start by searching for all sources of the graph (we assume there is
only one). Then we extract the subgraph defined by all edges between
the sources and the target. We augment this graph with a dummy super
source `s` and super sink `t`. This allows us to associate an edge with
the real source and sink.
# Step 2: Trace all paths from `s` to `t`.
Create a set of all paths from the source to the sink and accumulate
the `local_input_id` of each edge along the path. This will uniquely
identify each path. We use a hack to condense the accumualated ids in
order to display them nicely.
# Step 3: Create the new `exi_graph`
Using the traced paths with ids we construct a new graph representing
expanded inputs. The nodes in the original graph will be copied for each
unique path that passes through the node. We identify these nodes using
the accumulated ids built along the edges in our path set. For each
path starting from the target we add each node augmented with the
accumulated ids on its output(?) edge. We also add the edges along
these paths which results in the final `exi_graph`.
# Step 4: Identify valid inputs candidates
The purpose of this graph is to identify which inputs are needed
to compute dependant properties. One valid set of inputs is all
sources of the graph. However, sometimes it is preferable to specify
a model that may have been trained from many inputs. Therefore any
node with a one-to-many input edge may also be specified as an input.
# Step 5: Identify root-most inputs
The user will only specify one possible set of the inputs. We refer to
this set as the "root-most" inputs. This is a set of candiate nodes
such that all paths from the sink to the super source are blocked. We
default to the set of inputs which results in the fewest dependency
computations. However this is arbitary.
The last step that is not represented here is to compute the order that
the branches must be specified in when given to the depcache for a
computation.
Returns:
nx.DiGraph: exi_graph: the expanded input graph
Notes:
All * nodes are defined to be distinct.
TODO: To make a * node non-distinct it must be suffixed with an
identifier.
CommandLine:
python -m dtool.input_helpers make_expanded_input_graph --show
Example:
>>> # ENABLE_DOCTEST
>>> from dtool.input_helpers import * # NOQA
>>> from dtool.example_depcache2 import * # NOQA
>>> depc = testdata_depc3()
>>> table = depc['smk_match']
>>> table = depc['vsone']
>>> graph = table.depc.explicit_graph.copy()
>>> target = table.tablename
>>> exi_graph = make_expanded_input_graph(graph, target)
>>> x = list(exi_graph.nodes())[0]
>>> print('x = %r' % (x,))
>>> ut.quit_if_noshow()
>>> import plottool as pt
>>> pt.show_nx(graph, fnum=1, pnum=(1, 2, 1))
>>> pt.show_nx(exi_graph, fnum=1, pnum=(1, 2, 2))
>>> ut.show_if_requested()
"""
# FIXME: this does not work correctly when
# The nesting of non-1-to-1 dependencies is greater than 2 (I think)
# algorithm for finding inputs does not work.
# FIXME: two vocabs have the same edge id, they should be the same in the
# Expanded Input Graph as well. Their accum_id needs to be changed.
def condense_accum_ids(rinput_path_id):
# Hack to condense and consolidate graph sources
prev = None
compressed = []
for item in rinput_path_id:
if item == '1' and prev is not None:
pass # done append ones
elif item != prev:
compressed.append(item)
prev = item
#if len(compressed) > 1 and compressed[0] in ['1', '*']:
if len(compressed) > 1 and compressed[0] == '1':
compressed = compressed[1:]
compressed = tuple(compressed)
return compressed
BIG_HACK = True
#BIG_HACK = False
def condense_accum_ids_stars(rinput_path_id):
# Hack to condense and consolidate graph sources
rcompressed = []
has_star = False
# Remove all but the final star (this is a really bad hack)
for item in reversed(rinput_path_id):
is_star = '*' in item
if not (is_star and has_star):
if not has_star:
rcompressed.append(item)
has_star = has_star or is_star
compressed = tuple(rcompressed[::-1])
return compressed
def accumulate_input_ids(edge_list):
"""
python -m dtool.example_depcache2 testdata_depc4 --show
"""
edge_data = ut.take_column(edge_list, 3)
# We are accumulating local input ids
toaccum_list_ = ut.dict_take_column(edge_data, 'local_input_id')
if BIG_HACK and True:
v_list = ut.take_column(edge_list, 1)
# show the local_input_ids at the entire level
pred_ids = ([[
x['local_input_id']
for x in list(graph.pred[node].values())[0].values()
] if len(graph.pred[node]) else [] for node in v_list])
toaccum_list = [
x + ':' + ';'.join(y) for x, y in zip(toaccum_list_, pred_ids)
]
else:
toaccum_list = toaccum_list_
# Default dumb accumulation
accum_ids_ = ut.cumsum(zip(toaccum_list), tuple())
accum_ids = ut.lmap(condense_accum_ids, accum_ids_)
if BIG_HACK:
accum_ids = ut.lmap(condense_accum_ids_stars, accum_ids)
accum_ids = [('t', ) + x for x in accum_ids]
ut.dict_set_column(edge_data, 'accum_id', accum_ids)
return accum_ids
sources = list(ut.nx_source_nodes(graph))
print(sources)
# assert len(sources) == 1, 'expected a unique source'
source = sources[0]
graph = graph.subgraph(ut.nx_all_nodes_between(graph, source,
target)).copy()
# Remove superfluous data
ut.nx_delete_edge_attr(
graph,
[
'edge_type',
'isnwise',
'nwise_idx',
# 'parent_colx',
'ismulti'
])
# Make all '*' edges have distinct local_input_id's.
# TODO: allow non-distinct suffixes
count = ord('a')
for edge in graph.edges(keys=True, data=True):
dat = edge[3]
if dat['local_input_id'] == '*':
dat['local_input_id'] = '*' + chr(count)
dat['taillabel'] = '*' + chr(count)
count += 1
# Augment with dummy super source/sink nodes
source_input = 'source_input'
target_output = 'target_output'
graph.add_edge(source_input, source, local_input_id='s', taillabel='1')
graph.add_edge(target, target_output, local_input_id='t', taillabel='1')
# Find all paths from the table to the source.
paths_to_source = ut.all_multi_paths(graph,
source_input,
target_output,
data=True)
# Build expanded input graph
# The inputs to this table can be derived from this graph.
# The output is a new expanded input graph.
exi_graph = nx.DiGraph()
for path in paths_to_source:
# Accumlate unique identifiers along the reversed path
edge_list = ut.reverse_path_edges(path)
accumulate_input_ids(edge_list)
# A node's output(?) on this path determines its expanded branch id
exi_nodes = [
ExiNode(v, BranchId(d['accum_id'], k, d.get('parent_colx', -1)))
for u, v, k, d in edge_list[:-1]
]
exi_node_to_label = {
node: node[0] + '[' + ','.join([str(x) for x in node[1]]) + ']'
for node in exi_nodes
}
exi_graph.add_nodes_from(exi_nodes)
nx.set_node_attributes(exi_graph,
name='label',
values=exi_node_to_label)
# Undo any accumulation ordering and remove dummy nodes
old_edges = ut.reverse_path_edges(edge_list[1:-1])
new_edges = ut.reverse_path_edges(list(ut.itertwo(exi_nodes)))
for new_edge, old_edge in zip(new_edges, old_edges):
u2, v2 = new_edge[:2]
d = old_edge[3]
taillabel = d['taillabel']
parent_colx = d.get('parent_colx', -1)
if not exi_graph.has_edge(u2, v2):
exi_graph.add_edge(u2,
v2,
taillabel=taillabel,
parent_colx=parent_colx)
sink_nodes = list(ut.nx_sink_nodes(exi_graph))
source_nodes = list(ut.nx_source_nodes(exi_graph))
assert len(sink_nodes) == 1, 'expected a unique sink'
sink_node = sink_nodes[0]
# First identify if a node is root_specifiable
node_dict = ut.nx_node_dict(exi_graph)
for node in exi_graph.nodes():
root_specifiable = False
# for edge in exi_graph.in_edges(node, keys=True):
for edge in exi_graph.in_edges(node):
# key = edge[-1]
# assert key == 0, 'multi di graph is necessary'
edata = exi_graph.get_edge_data(*edge)
if edata.get('taillabel').startswith('*'):
if node != sink_node:
root_specifiable = True
if exi_graph.in_degree(node) == 0:
root_specifiable = True
node_dict[node]['root_specifiable'] = root_specifiable
# Need to specify any combo of red nodes such that
# 1) for each path from a (leaf) to the (root) there is exactly one red
# node along that path. This garentees that all inputs are gievn.
path_list = ut.flatten([
nx.all_simple_paths(exi_graph, source_node, sink_node)
for source_node in source_nodes
])
rootmost_nodes = set([])
for path in path_list:
flags = [node_dict[node]['root_specifiable'] for node in path]
valid_nodes = ut.compress(path, flags)
rootmost_nodes.add(valid_nodes[-1])
# Rootmost nodes are the ones specifiable by default when computing the
# normal property.
for node in rootmost_nodes:
node_dict[node]['rootmost'] = True
# We actually need to hack away any root-most nodes that have another
# rootmost node as the parent. Otherwise, this would cause constraints in
# what the user could specify as valid input combinations.
# ie: specify a vocab and an index, but the index depends on the vocab.
# this forces the user to specify the vocab that was the parent of the index
# the user should either just specify the index and have the vocab inferred
# or for now, we just dont allow this to happen.
nx.get_node_attributes(exi_graph, 'rootmost')
recolor_exi_graph(exi_graph, rootmost_nodes)
return exi_graph
def draw_em_graph(P, Pn, PL, gam, num_labels):
"""
python -m ibeis.algo.hots.testem test_em --show --no-cnn
"""
num_labels = PL.shape[1]
name_nodes = ['N%d' % x for x in list(range(1, num_labels + 1))]
#annot_nodes = ut.chr_range(len(Pn), base='A')
annot_nodes = ['X%d' % x for x in list(range(1, len(Pn) + 1))]
# name_nodes = ut.chr_range(num_labels, base='A')
nodes = name_nodes + annot_nodes
PL2 = gam[:, num_labels:].T
PL2 += .01
PL2 = PL2 / PL2.sum(axis=1)[:, None]
# PL2 = PL2 / np.linalg.norm(PL2, axis=0)
zero_part = np.zeros((num_labels, len(Pn) + num_labels))
prob_part = np.hstack([PL2, Pn])
print(ut.hz_str(' PL2 = ', ut.array_repr2(PL2, precision=2)))
# Redo p with posteriors
if ut.get_argflag('--postem'):
P = np.vstack([zero_part, prob_part])
weight_matrix = P # NOQA
graph = ut.nx_from_matrix(P, nodes=nodes)
graph = graph.to_directed()
# delete graph
dup_edges = []
seen_ = set([])
for u, v in graph.edges():
if u < v:
u, v = v, u
if (u, v) not in seen_:
seen_.add((u, v))
else:
dup_edges.append((u, v))
graph.remove_edges_from(dup_edges)
import plottool as pt
import networkx as nx
if len(name_nodes) == 3 and len(annot_nodes) == 4:
graph.node[annot_nodes[0]]['pos'] = (20., 200.)
graph.node[annot_nodes[1]]['pos'] = (220., 200.)
graph.node[annot_nodes[2]]['pos'] = (20., 100.)
graph.node[annot_nodes[3]]['pos'] = (220., 100.)
graph.node[name_nodes[0]]['pos'] = (10., 300.)
graph.node[name_nodes[1]]['pos'] = (120., 300.)
graph.node[name_nodes[2]]['pos'] = (230., 300.)
nx.set_node_attributes(graph, 'pin', 'true')
print('annot_nodes = %r' % (annot_nodes,))
print('name_nodes = %r' % (name_nodes,))
for u in annot_nodes:
for v in name_nodes:
if graph.has_edge(u, v):
print('1) u, v = %r' % ((u, v),))
graph.edge[u][v]['taillabel'] = graph.edge[u][v]['label']
graph.edge[u][v]['color'] = pt.ORANGE
graph.edge[u][v]['labelcolor'] = pt.BLUE
del graph.edge[u][v]['label']
elif graph.has_edge(v, u):
print('2) u, v = %r' % ((u, v),))
graph.edge[v][u]['headlabel'] = graph.edge[v][u]['label']
graph.edge[v][u]['color'] = pt.ORANGE
graph.edge[v][u]['labelcolor'] = pt.BLUE
del graph.edge[v][u]['label']
else:
print((u, v))
print('!!')
# import itertools
# name_const_edges = [(u, v, {'style': 'invis'}) for u, v in itertools.combinations(name_nodes, 2)]
# graph.add_edges_from(name_const_edges)
# nx.set_edge_attributes(graph, 'constraint', {edge: False for edge in graph.edges() if edge[0] == 'b' or edge[1] == 'b'})
# nx.set_edge_attributes(graph, 'constraint', {edge: False for edge in graph.edges() if edge[0] in annot_nodes and edge[1] in annot_nodes})
# nx.set_edge_attributes(graph, 'constraint', {edge: True for edge in graph.edges() if edge[0] in name_nodes or edge[1] in name_nodes})
# nx.set_edge_attributes(graph, 'constraint', {edge: True for edge in graph.edges() if (edge[0] in ['a', 'b'] and edge[1] in ['a', 'b']) and edge[0] in annot_nodes and edge[1] in annot_nodes})
# nx.set_edge_attributes(graph, 'constraint', {edge: True for edge in graph.edges() if (edge[0] in ['c'] or edge[1] in ['c']) and edge[0] in annot_nodes and edge[1] in annot_nodes})
# nx.set_edge_attributes(graph, 'constraint', {edge: True for edge in graph.edges() if (edge[0] in ['a'] or edge[1] in ['a']) and edge[0] in annot_nodes and edge[1] in annot_nodes})
# nx.set_edge_attributes(graph, 'constraint', {edge: True for edge in graph.edges() if (edge[0] in ['b'] or edge[1] in ['b']) and edge[0] in annot_nodes and edge[1] in annot_nodes})
# graph.add_edges_from([('root', n) for n in nodes])
# {node: 'names' for node in name_nodes})
nx.set_node_attributes(graph, 'color', {node: pt.RED for node in name_nodes})
# nx.set_node_attributes(graph, 'width', {node: 20 for node in nodes})
# nx.set_node_attributes(graph, 'height', {node: 20 for node in nodes})
#nx.set_node_attributes(graph, 'group', {node: 'names' for node in name_nodes})
#nx.set_node_attributes(graph, 'group', {node: 'annots' for node in annot_nodes})
nx.set_node_attributes(graph, 'groupid', {node: 'names' for node in name_nodes})
nx.set_node_attributes(graph, 'groupid', {node: 'annots' for node in annot_nodes})
graph.graph['clusterrank'] = 'local'
# graph.graph['groupattrs'] = {
# 'names': {'rankdir': 'LR', 'rank': 'source'},
# 'annots': {'rankdir': 'TB', 'rank': 'source'},
# }
ut.nx_delete_edge_attr(graph, 'weight')
# pt.show_nx(graph, fontsize=10, layoutkw={'splines': 'spline', 'prog': 'dot', 'sep': 2.0}, verbose=1)
layoutkw = {
# 'rankdir': 'LR',
'splines': 'spline',
# 'splines': 'ortho',
# 'splines': 'curved',
# 'compound': 'True',
# 'prog': 'dot',
'prog': 'neato',
# 'packMode': 'clust',
# 'sep': 4,
# 'nodesep': 1,
# 'ranksep': 1,
}
#pt.show_nx(graph, fontsize=12, layoutkw=layoutkw, verbose=0, as_directed=False)
pt.show_nx(graph, fontsize=6, fontname='Ubuntu', layoutkw=layoutkw, verbose=0, as_directed=False)
pt.interactions.zoom_factory()