def demodata_infr2(defaultdb='PZ_MTEST'):
defaultdb = 'PZ_MTEST'
import wbia
ibs = wbia.opendb(defaultdb=defaultdb)
annots = ibs.annots()
names = list(annots.group_items(annots.nids).values())[0:20]
def dummy_phi(c, n):
x = np.arange(n)
phi = c * x / (c * x + 1)
phi = phi / phi.sum()
phi = np.diff(phi)
return phi
phis = {c: dummy_phi(c, 30) for c in range(1, 4)}
aids = ut.flatten(names)
infr = wbia.AnnotInference(ibs, aids, autoinit=True)
infr.init_termination_criteria(phis)
infr.init_refresh_criteria()
# Partially review
n1, n2, n3, n4 = names[0:4]
for name in names[4:]:
for a, b in ut.itertwo(name.aids):
infr.add_feedback((a, b), POSTV)
for name1, name2 in it.combinations(names[4:], 2):
infr.add_feedback((name1.aids[0], name2.aids[0]), NEGTV)
return infr
def demodata_tarjan_bridge():
"""
CommandLine:
python -m wbia.algo.graph.nx_utils demodata_tarjan_bridge --show
Example:
>>> # ENABLE_DOCTEST
>>> from wbia.algo.graph.nx_utils import * # NOQA
>>> G = demodata_tarjan_bridge()
>>> ut.quit_if_noshow()
>>> import wbia.plottool as pt
>>> pt.show_nx(G)
>>> ut.show_if_requested()
"""
# define 2-connected compoments and bridges
cc2 = [
(1, 2, 4, 3, 1, 4),
(5, 6, 7, 5),
(8, 9, 10, 8),
(17, 18, 16, 15, 17),
(11, 12, 14, 13, 11, 14),
]
bridges = [(4, 8), (3, 5), (3, 17)]
G = nx.Graph(ut.flatten(ut.itertwo(path) for path in cc2 + bridges))
return G
def closest_point_on_verts(p, verts):
import vtool as vt
candidates = [
closest_point_on_line_segment(p, e1, e2)
for e1, e2 in ut.itertwo(verts)
]
dists = np.array([vt.L2_sqrd(p, new_pt) for new_pt in candidates])
new_pts = candidates[dists.argmin()]
return new_pts
def to_networkx(self):
import utool as ut
# n = list(self.tour_tree._traverse_nodes())[0]
# return nx.Graph(ut.itertwo(self.tour))
# In order traversal of the tree is the tour order
# return nx.Graph(ut.itertwo(self.tour_tree.values()))
tour = (n.value for n in self.tour_tree._traverse_nodes())
graph = nx.Graph(ut.itertwo(tour))
return graph
def repr_tree(self):
"""
reconstruct represented tree as a DiGraph to
preserve the current rootedness
"""
import utool as ut
import networkx as nx
repr_tree = nx.DiGraph()
for u, v in ut.itertwo(self.values()):
if not repr_tree.has_edge(v, u):
repr_tree.add_edge(u, v)
return repr_tree
def separate_math(line):
# Break line into math and english parts
mathsep = ut.negative_lookbehind(re.escape('\\')) + re.escape('$')
pos = [0]
for count, match in enumerate(re.finditer(mathsep, line)):
pos.append(match.start() if count % 2 == 0 else match.end())
pos.append(len(line))
english = []
math = []
for count, (l, r) in enumerate(ut.itertwo(pos)):
if count % 2 == 0 and line[l:r]:
english.append(line[l:r])
else:
math.append(line[l:r])
return english, math
def get_hacked_pos(netx_graph, name_nodes=None, prog='dot'):
import pygraphviz
import networkx as netx
# Add "invisible" edges to induce an ordering
# Hack for layout (ordering of top level nodes)
netx_graph2 = netx_graph.copy()
if getattr(netx_graph, 'ttype2_cpds', None) is not None:
grouped_nodes = []
for ttype in netx_graph.ttype2_cpds.keys():
ttype_cpds = netx_graph.ttype2_cpds[ttype]
# use defined ordering
ttype_nodes = ut.list_getattr(ttype_cpds, 'variable')
# ttype_nodes = sorted(ttype_nodes)
invis_edges = list(ut.itertwo(ttype_nodes))
netx_graph2.add_edges_from(invis_edges)
grouped_nodes.append(ttype_nodes)
A = netx.to_agraph(netx_graph2)
for nodes in grouped_nodes:
A.add_subgraph(nodes, rank='same')
else:
A = netx.to_agraph(netx_graph2)
# if name_nodes is not None:
# #netx.set_node_attributes(netx_graph, name='label', values={n: {'label': n} for n in all_nodes})
# invis_edges = list(ut.itertwo(name_nodes))
# netx_graph2.add_edges_from(invis_edges)
# A.add_subgraph(name_nodes, rank='same')
# else:
# A = netx.to_agraph(netx_graph2)
args = ''
G = netx_graph
A.layout(prog=prog, args=args)
# A.draw('example.png', prog='dot')
node_pos = {}
for n in G:
node_ = pygraphviz.Node(A, n)
try:
xx, yy = node_.attr['pos'].split(',')
node_pos[n] = (float(xx), float(yy))
except Exception:
logger.info('no position for node', n)
node_pos[n] = (0.0, 0.0)
return node_pos
def get_hacked_pos(netx_graph, name_nodes=None, prog='dot'):
import pygraphviz
import networkx as netx
# Add "invisible" edges to induce an ordering
# Hack for layout (ordering of top level nodes)
netx_graph2 = netx_graph.copy()
if getattr(netx_graph, 'ttype2_cpds', None) is not None:
grouped_nodes = []
for ttype in netx_graph.ttype2_cpds.keys():
ttype_cpds = netx_graph.ttype2_cpds[ttype]
# use defined ordering
ttype_nodes = ut.list_getattr(ttype_cpds, 'variable')
# ttype_nodes = sorted(ttype_nodes)
invis_edges = list(ut.itertwo(ttype_nodes))
netx_graph2.add_edges_from(invis_edges)
grouped_nodes.append(ttype_nodes)
A = netx.to_agraph(netx_graph2)
for nodes in grouped_nodes:
A.add_subgraph(nodes, rank='same')
else:
A = netx.to_agraph(netx_graph2)
#if name_nodes is not None:
# #netx.set_node_attributes(netx_graph, 'label', {n: {'label': n} for n in all_nodes})
# invis_edges = list(ut.itertwo(name_nodes))
# netx_graph2.add_edges_from(invis_edges)
# A.add_subgraph(name_nodes, rank='same')
#else:
# A = netx.to_agraph(netx_graph2)
args = ''
G = netx_graph
A.layout(prog=prog, args=args)
#A.draw('example.png', prog='dot')
node_pos = {}
for n in G:
node_ = pygraphviz.Node(A, n)
try:
xx, yy = node_.attr["pos"].split(',')
node_pos[n] = (float(xx), float(yy))
except:
print("no position for node", n)
node_pos[n] = (0.0, 0.0)
return node_pos
def get_bayesnet_layout(model, name_nodes=None, prog='dot'):
"""
Ensures ordering of layers is in order of addition via templates
"""
import pygraphviz
import networkx as nx
# Add "invisible" edges to induce an ordering
# Hack for layout (ordering of top level nodes)
netx_graph2 = model.copy()
if getattr(model, 'ttype2_cpds', None) is not None:
grouped_nodes = []
for ttype in model.ttype2_cpds.keys():
ttype_cpds = model.ttype2_cpds[ttype]
# use defined ordering
ttype_nodes = ut.list_getattr(ttype_cpds, 'variable')
# ttype_nodes = sorted(ttype_nodes)
invis_edges = list(ut.itertwo(ttype_nodes))
netx_graph2.add_edges_from(invis_edges)
grouped_nodes.append(ttype_nodes)
agraph = nx.nx_agraph.to_agraph(netx_graph2)
for nodes in grouped_nodes:
agraph.add_subgraph(nodes, rank='same')
else:
agraph = nx.nx_agraph.to_agraph(netx_graph2)
logger.info(agraph)
args = ''
agraph.layout(prog=prog, args=args)
# agraph.draw('example.png', prog='dot')
node_pos = {}
for n in model:
node_ = pygraphviz.Node(agraph, n)
try:
xx, yy = node_.attr['pos'].split(',')
node_pos[n] = (float(xx), float(yy))
except Exception:
logger.info('no position for node', n)
node_pos[n] = (0.0, 0.0)
return node_pos
def get_bayesnet_layout(model, name_nodes=None, prog='dot'):
"""
Ensures ordering of layers is in order of addition via templates
"""
import pygraphviz
import networkx as nx
# Add "invisible" edges to induce an ordering
# Hack for layout (ordering of top level nodes)
netx_graph2 = model.copy()
if getattr(model, 'ttype2_cpds', None) is not None:
grouped_nodes = []
for ttype in model.ttype2_cpds.keys():
ttype_cpds = model.ttype2_cpds[ttype]
# use defined ordering
ttype_nodes = ut.list_getattr(ttype_cpds, 'variable')
# ttype_nodes = sorted(ttype_nodes)
invis_edges = list(ut.itertwo(ttype_nodes))
netx_graph2.add_edges_from(invis_edges)
grouped_nodes.append(ttype_nodes)
agraph = nx.nx_agraph.to_agraph(netx_graph2)
for nodes in grouped_nodes:
agraph.add_subgraph(nodes, rank='same')
else:
agraph = nx.nx_agraph.to_agraph(netx_graph2)
print(agraph)
args = ''
agraph.layout(prog=prog, args=args)
#agraph.draw('example.png', prog='dot')
node_pos = {}
for n in model:
node_ = pygraphviz.Node(agraph, n)
try:
xx, yy = node_.attr['pos'].split(',')
node_pos[n] = (float(xx), float(yy))
except:
print('no position for node', n)
node_pos[n] = (0.0, 0.0)
return node_pos
def get_tier_windows(drive):
nbytes_tiers = [
np.inf,
2**32,
2**30,
2**29,
2**28,
2**27,
2**26,
2**25,
2**24,
2**23,
2**22,
2**21,
2**20,
2**10,
0,
-np.inf,
]
tier_windows = list(ut.itertwo(nbytes_tiers))
return tier_windows
def get_tier_windows(drive):
nbytes_tiers = [
np.inf,
2 ** 32,
2 ** 30,
2 ** 29,
2 ** 28,
2 ** 27,
2 ** 26,
2 ** 25,
2 ** 24,
2 ** 23,
2 ** 22,
2 ** 21,
2 ** 20,
2 ** 10,
0,
-np.inf,
]
tier_windows = list(ut.itertwo(nbytes_tiers))
return tier_windows
def check_image_sizes(data_uri_order, all_kpts, offset_list):
"""
Check if any keypoints go out of bounds wrt their associated images
"""
import vtool as vt
from os.path import join
imgdir = ut.truepath('/raid/work/Oxford/oxbuild_images')
gpath_list = [join(imgdir, imgid + '.jpg') for imgid in data_uri_order]
imgsize_list = [vt.open_image_size(gpath) for gpath in gpath_list]
kpts_list = [all_kpts[l:r] for l, r in ut.itertwo(offset_list)]
kpts_extent = [
vt.get_kpts_image_extent(kpts, outer=False, only_xy=False)
for kpts in ut.ProgIter(kpts_list, 'kpts extent')
]
for i, (size, extent) in enumerate(zip(imgsize_list, kpts_extent)):
w, h = size
_, maxx, _, maxy = extent
assert np.isnan(maxx) or maxx < w
assert np.isnan(maxy) or maxy < h
def run_asmk_script():
with ut.embed_on_exception_context: # NOQA
"""
>>> from wbia.algo.smk.script_smk import *
"""
# NOQA
# ==============================================
# PREPROCESSING CONFIGURATION
# ==============================================
config = {
# 'data_year': 2013,
'data_year': None,
'dtype': 'float32',
# 'root_sift': True,
'root_sift': False,
# 'centering': True,
'centering': False,
'num_words': 2**16,
# 'num_words': 1E6
# 'num_words': 8000,
'kmeans_impl': 'sklearn.mini',
'extern_words': False,
'extern_assign': False,
'assign_algo': 'kdtree',
'checks': 1024,
'int_rvec': True,
'only_xy': False,
}
# Define which params are relevant for which operations
relevance = {}
relevance['feats'] = ['dtype', 'root_sift', 'centering', 'data_year']
relevance['words'] = relevance['feats'] + [
'num_words',
'extern_words',
'kmeans_impl',
]
relevance['assign'] = relevance['words'] + [
'checks',
'extern_assign',
'assign_algo',
]
# relevance['ydata'] = relevance['assign'] + ['int_rvec']
# relevance['xdata'] = relevance['assign'] + ['only_xy', 'int_rvec']
nAssign = 1
class SMKCacher(ut.Cacher):
def __init__(self, fname, ext='.cPkl'):
relevant_params = relevance[fname]
relevant_cfg = ut.dict_subset(config, relevant_params)
cfgstr = ut.get_cfg_lbl(relevant_cfg)
dbdir = ut.truepath('/raid/work/Oxford/')
super(SMKCacher, self).__init__(fname,
cfgstr,
cache_dir=dbdir,
ext=ext)
# ==============================================
# LOAD DATASET, EXTRACT AND POSTPROCESS FEATURES
# ==============================================
if config['data_year'] == 2007:
data = load_oxford_2007()
elif config['data_year'] == 2013:
data = load_oxford_2013()
elif config['data_year'] is None:
data = load_oxford_wbia()
offset_list = data['offset_list']
all_kpts = data['all_kpts']
raw_vecs = data['all_vecs']
query_uri_order = data['query_uri_order']
data_uri_order = data['data_uri_order']
# del data
# ================
# PRE-PROCESS
# ================
import vtool as vt
# Alias names to avoid errors in interactive sessions
proc_vecs = raw_vecs
del raw_vecs
feats_cacher = SMKCacher('feats', ext='.npy')
all_vecs = feats_cacher.tryload()
if all_vecs is None:
if config['dtype'] == 'float32':
logger.info('Converting vecs to float32')
proc_vecs = proc_vecs.astype(np.float32)
else:
proc_vecs = proc_vecs
raise NotImplementedError('other dtype')
if config['root_sift']:
with ut.Timer('Apply root sift'):
np.sqrt(proc_vecs, out=proc_vecs)
vt.normalize(proc_vecs, ord=2, axis=1, out=proc_vecs)
if config['centering']:
with ut.Timer('Apply centering'):
mean_vec = np.mean(proc_vecs, axis=0)
# Center and then re-normalize
np.subtract(proc_vecs, mean_vec[None, :], out=proc_vecs)
vt.normalize(proc_vecs, ord=2, axis=1, out=proc_vecs)
if config['dtype'] == 'int8':
smk_funcs
all_vecs = proc_vecs
feats_cacher.save(all_vecs)
del proc_vecs
# =====================================
# BUILD VISUAL VOCABULARY
# =====================================
if config['extern_words']:
words = data['words']
assert config['num_words'] is None or len(
words) == config['num_words']
else:
word_cacher = SMKCacher('words')
words = word_cacher.tryload()
if words is None:
with ut.embed_on_exception_context:
if config['kmeans_impl'] == 'sklearn.mini':
import sklearn.cluster
rng = np.random.RandomState(13421421)
# init_size = int(config['num_words'] * 8)
init_size = int(config['num_words'] * 4)
# converged after 26043 iterations
clusterer = sklearn.cluster.MiniBatchKMeans(
config['num_words'],
init_size=init_size,
batch_size=1000,
compute_labels=False,
max_iter=20,
random_state=rng,
n_init=1,
verbose=1,
)
clusterer.fit(all_vecs)
words = clusterer.cluster_centers_
elif config['kmeans_impl'] == 'yael':
from yael import ynumpy
centroids, qerr, dis, assign, nassign = ynumpy.kmeans(
all_vecs,
config['num_words'],
init='kmeans++',
verbose=True,
output='all',
)
words = centroids
word_cacher.save(words)
# =====================================
# ASSIGN EACH VECTOR TO ITS NEAREST WORD
# =====================================
if config['extern_assign']:
assert config[
'extern_words'], 'need extern cluster to extern assign'
idx_to_wxs = vt.atleast_nd(data['idx_to_wx'], 2)
idx_to_maws = np.ones(idx_to_wxs.shape, dtype=np.float32)
idx_to_wxs = np.ma.array(idx_to_wxs)
idx_to_maws = np.ma.array(idx_to_maws)
else:
from wbia.algo.smk import vocab_indexer
vocab = vocab_indexer.VisualVocab(words)
dassign_cacher = SMKCacher('assign')
assign_tup = dassign_cacher.tryload()
if assign_tup is None:
vocab.flann_params['algorithm'] = config['assign_algo']
vocab.build()
# Takes 12 minutes to assign jegous vecs to 2**16 vocab
with ut.Timer('assign vocab neighbors'):
_idx_to_wx, _idx_to_wdist = vocab.nn_index(
all_vecs, nAssign, checks=config['checks'])
if nAssign > 1:
idx_to_wxs, idx_to_maws = smk_funcs.weight_multi_assigns(
_idx_to_wx,
_idx_to_wdist,
massign_alpha=1.2,
massign_sigma=80.0,
massign_equal_weights=True,
)
else:
idx_to_wxs = np.ma.masked_array(_idx_to_wx,
fill_value=-1)
idx_to_maws = np.ma.ones(idx_to_wxs.shape,
fill_value=-1,
dtype=np.float32)
idx_to_maws.mask = idx_to_wxs.mask
assign_tup = (idx_to_wxs, idx_to_maws)
dassign_cacher.save(assign_tup)
idx_to_wxs, idx_to_maws = assign_tup
# Breakup vectors, keypoints, and word assignments by annotation
wx_lists = [
idx_to_wxs[left:right] for left, right in ut.itertwo(offset_list)
]
maw_lists = [
idx_to_maws[left:right] for left, right in ut.itertwo(offset_list)
]
vecs_list = [
all_vecs[left:right] for left, right in ut.itertwo(offset_list)
]
kpts_list = [
all_kpts[left:right] for left, right in ut.itertwo(offset_list)
]
# =======================
# FIND QUERY SUBREGIONS
# =======================
ibs, query_annots, data_annots, qx_to_dx = load_ordered_annots(
data_uri_order, query_uri_order)
daids = data_annots.aids
qaids = query_annots.aids
query_super_kpts = ut.take(kpts_list, qx_to_dx)
query_super_vecs = ut.take(vecs_list, qx_to_dx)
query_super_wxs = ut.take(wx_lists, qx_to_dx)
query_super_maws = ut.take(maw_lists, qx_to_dx)
# Mark which keypoints are within the bbox of the query
query_flags_list = []
only_xy = config['only_xy']
for kpts_, bbox in zip(query_super_kpts, query_annots.bboxes):
flags = kpts_inside_bbox(kpts_, bbox, only_xy=only_xy)
query_flags_list.append(flags)
logger.info('Queries are crops of existing database images.')
logger.info('Looking at average percents')
percent_list = [
flags_.sum() / flags_.shape[0] for flags_ in query_flags_list
]
percent_stats = ut.get_stats(percent_list)
logger.info('percent_stats = %s' % (ut.repr4(percent_stats), ))
import vtool as vt
query_kpts = vt.zipcompress(query_super_kpts, query_flags_list, axis=0)
query_vecs = vt.zipcompress(query_super_vecs, query_flags_list, axis=0)
query_wxs = vt.zipcompress(query_super_wxs, query_flags_list, axis=0)
query_maws = vt.zipcompress(query_super_maws, query_flags_list, axis=0)
# =======================
# CONSTRUCT QUERY / DATABASE REPR
# =======================
# int_rvec = not config['dtype'].startswith('float')
int_rvec = config['int_rvec']
X_list = []
_prog = ut.ProgPartial(length=len(qaids),
label='new X',
bs=True,
adjust=True)
for aid, fx_to_wxs, fx_to_maws in _prog(
zip(qaids, query_wxs, query_maws)):
X = new_external_annot(aid, fx_to_wxs, fx_to_maws, int_rvec)
X_list.append(X)
# ydata_cacher = SMKCacher('ydata')
# Y_list = ydata_cacher.tryload()
# if Y_list is None:
Y_list = []
_prog = ut.ProgPartial(length=len(daids),
label='new Y',
bs=True,
adjust=True)
for aid, fx_to_wxs, fx_to_maws in _prog(zip(daids, wx_lists,
maw_lists)):
Y = new_external_annot(aid, fx_to_wxs, fx_to_maws, int_rvec)
Y_list.append(Y)
# ydata_cacher.save(Y_list)
# ======================
# Add in some groundtruth
logger.info('Add in some groundtruth')
for Y, nid in zip(Y_list, ibs.get_annot_nids(daids)):
Y.nid = nid
for X, nid in zip(X_list, ibs.get_annot_nids(qaids)):
X.nid = nid
for Y, qual in zip(Y_list, ibs.get_annot_quality_texts(daids)):
Y.qual = qual
# ======================
# Add in other properties
for Y, vecs, kpts in zip(Y_list, vecs_list, kpts_list):
Y.vecs = vecs
Y.kpts = kpts
imgdir = ut.truepath('/raid/work/Oxford/oxbuild_images')
for Y, imgid in zip(Y_list, data_uri_order):
gpath = ut.unixjoin(imgdir, imgid + '.jpg')
Y.gpath = gpath
for X, vecs, kpts in zip(X_list, query_vecs, query_kpts):
X.kpts = kpts
X.vecs = vecs
# ======================
logger.info('Building inverted list')
daids = [Y.aid for Y in Y_list]
# wx_list = sorted(ut.list_union(*[Y.wx_list for Y in Y_list]))
wx_list = sorted(set.union(*[Y.wx_set for Y in Y_list]))
assert daids == data_annots.aids
assert len(wx_list) <= config['num_words']
wx_to_aids = smk_funcs.invert_lists(daids, [Y.wx_list for Y in Y_list],
all_wxs=wx_list)
# Compute IDF weights
logger.info('Compute IDF weights')
ndocs_total = len(daids)
# Use only the unique number of words
ndocs_per_word = np.array([len(set(wx_to_aids[wx])) for wx in wx_list])
logger.info('ndocs_perword stats: ' +
ut.repr4(ut.get_stats(ndocs_per_word)))
idf_per_word = smk_funcs.inv_doc_freq(ndocs_total, ndocs_per_word)
wx_to_weight = dict(zip(wx_list, idf_per_word))
logger.info('idf stats: ' +
ut.repr4(ut.get_stats(wx_to_weight.values())))
# Filter junk
Y_list_ = [Y for Y in Y_list if Y.qual != 'junk']
# =======================
# CHOOSE QUERY KERNEL
# =======================
params = {
'asmk': dict(alpha=3.0, thresh=0.0),
'bow': dict(),
'bow2': dict(),
}
# method = 'bow'
method = 'bow2'
method = 'asmk'
smk = SMK(wx_to_weight, method=method, **params[method])
# Specific info for the type of query
if method == 'asmk':
# Make residual vectors
if True:
# The stacked way is 50x faster
# TODO: extend for multi-assignment and record fxs
flat_query_vecs = np.vstack(query_vecs)
flat_query_wxs = np.vstack(query_wxs)
flat_query_offsets = np.array(
[0] + ut.cumsum(ut.lmap(len, query_wxs)))
flat_wxs_assign = flat_query_wxs
flat_offsets = flat_query_offsets
flat_vecs = flat_query_vecs
tup = smk_funcs.compute_stacked_agg_rvecs(
words, flat_wxs_assign, flat_vecs, flat_offsets)
all_agg_vecs, all_error_flags, agg_offset_list = tup
if int_rvec:
all_agg_vecs = smk_funcs.cast_residual_integer(
all_agg_vecs)
agg_rvecs_list = [
all_agg_vecs[left:right]
for left, right in ut.itertwo(agg_offset_list)
]
agg_flags_list = [
all_error_flags[left:right]
for left, right in ut.itertwo(agg_offset_list)
]
for X, agg_rvecs, agg_flags in zip(X_list, agg_rvecs_list,
agg_flags_list):
X.agg_rvecs = agg_rvecs
X.agg_flags = agg_flags[:, None]
flat_wxs_assign = idx_to_wxs
flat_offsets = offset_list
flat_vecs = all_vecs
tup = smk_funcs.compute_stacked_agg_rvecs(
words, flat_wxs_assign, flat_vecs, flat_offsets)
all_agg_vecs, all_error_flags, agg_offset_list = tup
if int_rvec:
all_agg_vecs = smk_funcs.cast_residual_integer(
all_agg_vecs)
agg_rvecs_list = [
all_agg_vecs[left:right]
for left, right in ut.itertwo(agg_offset_list)
]
agg_flags_list = [
all_error_flags[left:right]
for left, right in ut.itertwo(agg_offset_list)
]
for Y, agg_rvecs, agg_flags in zip(Y_list, agg_rvecs_list,
agg_flags_list):
Y.agg_rvecs = agg_rvecs
Y.agg_flags = agg_flags[:, None]
else:
# This non-stacked way is about 500x slower
_prog = ut.ProgPartial(label='agg Y rvecs',
bs=True,
adjust=True)
for Y in _prog(Y_list_):
make_agg_vecs(Y, words, Y.vecs)
_prog = ut.ProgPartial(label='agg X rvecs',
bs=True,
adjust=True)
for X in _prog(X_list):
make_agg_vecs(X, words, X.vecs)
elif method == 'bow2':
# Hack for orig tf-idf bow vector
nwords = len(words)
for X in ut.ProgIter(X_list, label='make bow vector'):
ensure_tf(X)
bow_vector(X, wx_to_weight, nwords)
for Y in ut.ProgIter(Y_list_, label='make bow vector'):
ensure_tf(Y)
bow_vector(Y, wx_to_weight, nwords)
if method != 'bow2':
for X in ut.ProgIter(X_list, 'compute X gamma'):
X.gamma = smk.gamma(X)
for Y in ut.ProgIter(Y_list_, 'compute Y gamma'):
Y.gamma = smk.gamma(Y)
# Execute matches (could go faster by enumerating candidates)
scores_list = []
for X in ut.ProgIter(X_list, label='query %s' % (smk, )):
scores = [smk.kernel(X, Y) for Y in Y_list_]
scores = np.array(scores)
scores = np.nan_to_num(scores)
scores_list.append(scores)
import sklearn.metrics
avep_list = []
_iter = list(zip(scores_list, X_list))
_iter = ut.ProgIter(_iter, label='evaluate %s' % (smk, ))
for scores, X in _iter:
truth = [X.nid == Y.nid for Y in Y_list_]
avep = sklearn.metrics.average_precision_score(truth, scores)
avep_list.append(avep)
avep_list = np.array(avep_list)
mAP = np.mean(avep_list)
logger.info('mAP = %r' % (mAP, ))
def compute_stacked_agg_rvecs(words, flat_wxs_assign, flat_vecs, flat_offsets):
"""
More efficient version of agg on a stacked structure
Args:
words (ndarray): entire vocabulary of words
flat_wxs_assign (ndarray): maps a stacked index to word index
flat_vecs (ndarray): stacked SIFT descriptors
flat_offsets (ndarray): offset positions per annotation
Example:
>>> # DISABLE_DOCTEST
>>> from wbia.algo.smk.smk_funcs import * # NOQA
>>> data = testdata_rvecs(dim=2, nvecs=1000, nannots=10)
>>> words = data['words']
>>> flat_offsets = data['offset_list']
>>> flat_wxs_assign, flat_vecs = ut.take(data, ['idx_to_wx', 'vecs'])
>>> tup = compute_stacked_agg_rvecs(words, flat_wxs_assign, flat_vecs, flat_offsets)
>>> all_agg_vecs, all_error_flags, agg_offset_list = tup
>>> agg_rvecs_list = [all_agg_vecs[l:r] for l, r in ut.itertwo(agg_offset_list)]
>>> agg_flags_list = [all_error_flags[l:r] for l, r in ut.itertwo(agg_offset_list)]
>>> assert len(agg_flags_list) == len(flat_offsets) - 1
Example:
>>> # DISABLE_DOCTEST
>>> from wbia.algo.smk.smk_funcs import * # NOQA
>>> data = testdata_rvecs(dim=2, nvecs=100, nannots=5)
>>> words = data['words']
>>> flat_offsets = data['offset_list']
>>> flat_wxs_assign, flat_vecs = ut.take(data, ['idx_to_wx', 'vecs'])
>>> tup = compute_stacked_agg_rvecs(words, flat_wxs_assign, flat_vecs, flat_offsets)
>>> all_agg_vecs, all_error_flags, agg_offset_list = tup
>>> agg_rvecs_list = [all_agg_vecs[l:r] for l, r in ut.itertwo(agg_offset_list)]
>>> agg_flags_list = [all_error_flags[l:r] for l, r in ut.itertwo(agg_offset_list)]
>>> assert len(agg_flags_list) == len(flat_offsets) - 1
"""
grouped_wxs = [
flat_wxs_assign[left:right] for left, right in ut.itertwo(flat_offsets)
]
# Assume single assignment, aggregate everything
# across the entire database
flat_offsets = np.array(flat_offsets)
idx_to_dx = (np.searchsorted(
flat_offsets, np.arange(len(flat_wxs_assign)), side='right') -
1).astype(np.int32)
if isinstance(flat_wxs_assign, np.ma.masked_array):
wx_list = flat_wxs_assign.T[0].compressed()
else:
wx_list = flat_wxs_assign.T[0].ravel()
unique_wx, groupxs = vt.group_indices(wx_list)
dim = flat_vecs.shape[1]
if isinstance(flat_wxs_assign, np.ma.masked_array):
dx_to_wxs = [np.unique(wxs.compressed()) for wxs in grouped_wxs]
else:
dx_to_wxs = [np.unique(wxs.ravel()) for wxs in grouped_wxs]
dx_to_nagg = [len(wxs) for wxs in dx_to_wxs]
num_agg_vecs = sum(dx_to_nagg)
# all_agg_wxs = np.hstack(dx_to_wxs)
agg_offset_list = np.array([0] + ut.cumsum(dx_to_nagg))
# Preallocate agg residuals for all dxs
all_agg_vecs = np.empty((num_agg_vecs, dim), dtype=np.float32)
all_agg_vecs[:, :] = np.nan
# precompute agg residual stack
i_to_dxs = vt.apply_grouping(idx_to_dx, groupxs)
subgroup = [vt.group_indices(dxs) for dxs in ut.ProgIter(i_to_dxs)]
i_to_unique_dxs = ut.take_column(subgroup, 0)
i_to_dx_groupxs = ut.take_column(subgroup, 1)
num_words = len(unique_wx)
# Overall this takes 5 minutes and 21 seconds
# I think the other method takes about 12 minutes
for i in ut.ProgIter(range(num_words), 'agg'):
wx = unique_wx[i]
xs = groupxs[i]
dxs = i_to_unique_dxs[i]
dx_groupxs = i_to_dx_groupxs[i]
word = words[wx:wx + 1]
offsets1 = agg_offset_list.take(dxs)
offsets2 = [np.where(dx_to_wxs[dx] == wx)[0][0] for dx in dxs]
offsets = np.add(offsets1, offsets2, out=offsets1)
# if __debug__:
# assert np.bincount(dxs).max() < 2
# offset = agg_offset_list[dxs[0]]
# assert np.all(dx_to_wxs[dxs[0]] == all_agg_wxs[offset:offset +
# dx_to_nagg[dxs[0]]])
# Compute residuals
rvecs = flat_vecs[xs] - word
vt.normalize(rvecs, axis=1, out=rvecs)
rvecs[np.all(np.isnan(rvecs), axis=1)] = 0
# Aggregate across same images
grouped_rvecs = vt.apply_grouping(rvecs, dx_groupxs, axis=0)
agg_rvecs_ = [rvec_group.sum(axis=0) for rvec_group in grouped_rvecs]
# agg_rvecs = np.vstack(agg_rvecs_)
all_agg_vecs[offsets, :] = agg_rvecs_
assert not np.any(np.isnan(all_agg_vecs))
logger.info('Apply normalization')
vt.normalize(all_agg_vecs, axis=1, out=all_agg_vecs)
all_error_flags = np.all(np.isnan(all_agg_vecs), axis=1)
all_agg_vecs[all_error_flags, :] = 0
# ndocs_per_word1 = np.array(ut.lmap(len, wx_to_unique_dxs))
# ndocs_total1 = len(flat_offsets) - 1
# idf1 = smk_funcs.inv_doc_freq(ndocs_total1, ndocs_per_word1)
tup = all_agg_vecs, all_error_flags, agg_offset_list
return tup
def draw_bayesian_model(model,
evidence={},
soft_evidence={},
fnum=None,
pnum=None,
**kwargs):
from pgmpy.models import BayesianModel
if not isinstance(model, BayesianModel):
model = model.to_bayesian_model()
import plottool as pt
import networkx as nx
kwargs = kwargs.copy()
factor_list = kwargs.pop('factor_list', [])
ttype_colors, ttype_scalars = make_colorcodes(model)
textprops = {
'horizontalalignment': 'left',
'family': 'monospace',
'size': 8,
}
# build graph attrs
tup = get_node_viz_attrs(model, evidence, soft_evidence, factor_list,
ttype_colors, **kwargs)
node_color, pos_list, pos_dict, takws = tup
# draw graph
has_infered = evidence or 'factor_list' in kwargs
if False:
fig = pt.figure(fnum=fnum, pnum=pnum, doclf=True) # NOQA
ax = pt.gca()
drawkw = dict(pos=pos_dict,
ax=ax,
with_labels=True,
node_size=1100,
node_color=node_color)
nx.draw(model, **drawkw)
else:
# BE VERY CAREFUL
if 1:
graph = model.copy()
graph.__class__ = nx.DiGraph
graph.graph['groupattrs'] = ut.ddict(dict)
#graph = model.
if getattr(graph, 'ttype2_cpds', None) is not None:
# Add invis edges and ttype groups
for ttype in model.ttype2_cpds.keys():
ttype_cpds = model.ttype2_cpds[ttype]
# use defined ordering
ttype_nodes = ut.list_getattr(ttype_cpds, 'variable')
# ttype_nodes = sorted(ttype_nodes)
invis_edges = list(ut.itertwo(ttype_nodes))
graph.add_edges_from(invis_edges)
nx.set_edge_attributes(
graph, 'style',
{edge: 'invis'
for edge in invis_edges})
nx.set_node_attributes(
graph, 'groupid',
{node: ttype
for node in ttype_nodes})
graph.graph['groupattrs'][ttype]['rank'] = 'same'
graph.graph['groupattrs'][ttype]['cluster'] = False
else:
graph = model
pt.show_nx(graph,
layout_kw={'prog': 'dot'},
fnum=fnum,
pnum=pnum,
verbose=0)
pt.zoom_factory()
fig = pt.gcf()
ax = pt.gca()
pass
hacks = [
pt.draw_text_annotations(textprops=textprops, **takw) for takw in takws
if takw
]
xmin, ymin = np.array(pos_list).min(axis=0)
xmax, ymax = np.array(pos_list).max(axis=0)
if 'name' in model.ttype2_template:
num_names = len(model.ttype2_template['name'].basis)
num_annots = len(model.ttype2_cpds['name'])
if num_annots > 4:
ax.set_xlim((xmin - 40, xmax + 40))
ax.set_ylim((ymin - 50, ymax + 50))
fig.set_size_inches(30, 7)
else:
ax.set_xlim((xmin - 42, xmax + 42))
ax.set_ylim((ymin - 50, ymax + 50))
fig.set_size_inches(23, 7)
title = 'num_names=%r, num_annots=%r' % (
num_names,
num_annots,
)
else:
title = ''
map_assign = kwargs.get('map_assign', None)
def word_insert(text):
return '' if len(text) == 0 else text + ' '
top_assignments = kwargs.get('top_assignments', None)
if top_assignments is not None:
map_assign, map_prob = top_assignments[0]
if map_assign is not None:
title += '\n%sMAP: ' % (word_insert(kwargs.get('method', '')))
title += map_assign + ' @' + '%.2f%%' % (100 * map_prob, )
if kwargs.get('show_title', True):
pt.set_figtitle(title, size=14)
for hack in hacks:
hack()
if has_infered:
# Hack in colorbars
# if ut.list_type(basis) is int:
# pt.colorbar(scalars, colors, lbl='score', ticklabels=np.array(basis) + 1)
# else:
# pt.colorbar(scalars, colors, lbl='score', ticklabels=basis)
keys = ['name', 'score']
locs = ['left', 'right']
for key, loc in zip(keys, locs):
if key in ttype_colors:
basis = model.ttype2_template[key].basis
# scalars =
colors = ttype_colors[key]
scalars = ttype_scalars[key]
pt.colorbar(scalars,
colors,
lbl=key,
ticklabels=basis,
ticklocation=loc)
def closest_point_on_verts(p, verts):
import vtool as vt
candidates = [closest_point_on_line_segment(p, e1, e2) for e1, e2 in ut.itertwo(verts)]
dists = np.array([vt.L2_sqrd(p, new_pt) for new_pt in candidates])
new_pts = candidates[dists.argmin()]
return new_pts
def fix_sentences():
"""
fixtex --fixsent
"""
text = ut.read_from('main.tex')
root = latex_parser.LatexDocPart.parse_text(text, debug=None)
document = root.find_descendant_type('document')
chapters = list(document.find_descendant_types('chapter'))
def separate_math(line):
# Break line into math and english parts
mathsep = ut.negative_lookbehind(re.escape('\\')) + re.escape('$')
pos = [0]
for count, match in enumerate(re.finditer(mathsep, line)):
pos.append(match.start() if count % 2 == 0 else match.end())
pos.append(len(line))
english = []
math = []
for count, (l, r) in enumerate(ut.itertwo(pos)):
if count % 2 == 0 and line[l:r]:
english.append(line[l:r])
else:
math.append(line[l:r])
return english, math
def print_acronymn_def(english_line):
words = re.split('[~\s]', english_line.rstrip('.'))
words = [w.rstrip(',').rstrip('.') for w in words]
flag = 0
for count, word in enumerate(words):
if re.match('\\([A-Z]+\\)', word):
ut.cprint(word, 'blue')
flag = True
if flag:
print(re.sub('\\\\cite{[^}]*}', '', line))
def has_consec_cap_words(words):
for count, (u, v) in enumerate(ut.itertwo(words)):
if u[0].isupper() and v[0].isupper():
if count > 0:
return True
def gen_sentences():
for chapter in chapters:
# ut.cprint(chapter.fpath_root(), 'yellow')
for line in chapter.find_sentences():
context = {'chapter': chapter}
yield line, context
import re
found = ut.ddict(list)
for line, context in gen_sentences():
english, math = separate_math(line)
english_line = ' '.join(english).replace(',',
'').rstrip('.').strip(' ')
words = re.split('[~\s]+', english_line)
words = [w.rstrip(',').rstrip('.') for w in words]
if has_consec_cap_words(words):
print(line)
# print_acronymn_def(english_line)
if 'locality sensitive' in line:
print("LSH NEEDS DASH")
multicap_words = []
for count, word in enumerate(words):
word = word.strip(')').strip('(')
if sum(c.isupper() for c in word) > 1:
if word.startswith('\\') and word.endswith('{}'):
continue
if word.startswith('\\Cref') and word.endswith('}'):
if count != 0:
print("FIX CREF UPPER")
print(line)
continue
if word.startswith('\\cref') and word.endswith('}'):
if count == 0:
print("FIX CREF LOWER")
print(line)
continue
if not word.isalpha():
continue
multicap_words.append(word)
if multicap_words:
found[context['chapter']].append(multicap_words)
# print(ut.repr4(ut.dict_hist(found)))
def english_tokens(line):
# Break line into math and english parts
mathsep = ut.negative_lookbehind(re.escape('\\')) + re.escape('$')
def clean_word(word):
if word.startswith('``'):
word = word[2:]
if word.endswith("''"):
word = word[:-2]
return word.strip(',').rstrip('.')
prev = 0
tokens = []
for count, match in enumerate(re.finditer(mathsep, line)):
if count % 2 == 0:
curr = match.start()
english = line[prev:curr]
parts = re.split('[~\s]+', english)
parts = (clean_word(p) for p in parts)
parts = (p for p in parts if p)
tokens.extend(parts)
else:
curr = match.end()
math = line[prev:curr]
tokens.append(math)
prev = curr
return tokens
from fixtex.svn_converter.latexparser import DocParser
from fixtex.svn_converter.docnodes import CaptionNode, FigureNode
from fixtex.svn_converter.tokenizer import Tokenizer
def caption_sentences(fpath):
text = ut.readfrom(fpath)
tokenstream = Tokenizer(text).tokenize()
self = DocParser(tokenstream, fpath)
tree = self.parse()
for node in tree.walk():
if isinstance(node, FigureNode):
for x in node.walk():
if isinstance(x, CaptionNode):
for sent in ut.split_sentences2(x.resolve()):
yield sent
def gen_cap():
fpaths = [
ut.truepath('~/latex/crall-thesis-2017/figdef1.tex'),
ut.truepath('~/latex/crall-thesis-2017/figdef2.tex'),
ut.truepath('~/latex/crall-thesis-2017/figdef3.tex'),
ut.truepath('~/latex/crall-thesis-2017/figdef4.tex'),
ut.truepath('~/latex/crall-thesis-2017/figdef5.tex'),
]
for fpath in fpaths:
context = {'fpath': fpath}
for sent in caption_sentences(fpath):
yield sent, context
# Find A, An grammar errors
# Define special cases:
cons_sounds = {
'unit', 'user', 'unique', 'one', 'uniform', 'unified', 'useful'
}
vowel_sounds = {'roc', 'mcc', 'lnbnn', 'l1', 'hour'}
def startswith_vowel_sound(after):
# do our best guess
if after.startswith('$'):
if after[1] == '8':
return True
if after[1] == 'x':
return True
if after in vowel_sounds:
return True
if after in cons_sounds:
return False
return after[0] in 'aeiou'
cmd_map, cmd_map1 = latex_parser.LatexDocPart.read_static_defs()
simple_cmd_re = re.compile('\\\\[A-Za-z]*{}')
print('\nCHECK FOR A / AN ERRORS')
import itertools as it
generators = [
# gen_sentences(),
gen_cap(),
]
for line, context in it.chain(*generators):
words = english_tokens(line)
for u, v in ut.itertwo(words):
v_orig = v
if simple_cmd_re.match(v):
key = v[:-2]
try:
v = cmd_map[key]
except:
print(line)
raise
v = v.split('-')[0]
article = u.lower()
if article in {'a', 'an'}:
after = v.lower()
# TODO ensure v is a singular countable noun
is_vowel_sound = startswith_vowel_sound(after)
flag = False
if article == 'a' and is_vowel_sound:
flag = 'after is a consonent sound should start with a'
if article == 'an' and not is_vowel_sound:
flag = 'after is a vowel sound should start with an'
if flag:
print('---------')
print(flag)
print(article, after)
print('{} {}'.format(u, v_orig))
print(line)
def mark_unreviewed_above_score_as_correct(qres_wgt):
selected_qtindex_list = qres_wgt.selectedRows()
if len(selected_qtindex_list) == 1:
qtindex = selected_qtindex_list[0]
# aid1, aid2 = qres_wgt.get_aidpair_from_qtindex(qtindex)
thresh = qtindex.model().get_header_data('score', qtindex)
logger.info('thresh = %r' % (thresh, ))
rows = qres_wgt.review_api.ider()
scores_ = qres_wgt.review_api.get(
qres_wgt.review_api.col_name_list.index('score'), rows)
valid_rows = ut.compress(rows, scores_ >= thresh)
aids1 = qres_wgt.review_api.get(
qres_wgt.review_api.col_name_list.index('qaid'), valid_rows)
aids2 = qres_wgt.review_api.get(
qres_wgt.review_api.col_name_list.index('aid'), valid_rows)
# ibs = qres_wgt.ibs
ibs = qres_wgt.ibs
am_rowids = ibs.get_annotmatch_rowid_from_undirected_superkey(
aids1, aids2)
reviewed = ibs.get_annotmatch_reviewed(am_rowids)
unreviewed = ut.not_list(reviewed)
valid_rows = ut.compress(valid_rows, unreviewed)
aids1 = ut.compress(aids1, unreviewed)
aids2 = ut.compress(aids2, unreviewed)
import networkx as nx
graph = nx.Graph()
graph.add_edges_from(list(zip(aids1, aids2)),
{'user_thresh_match': True})
review_groups = list(nx.connected_component_subgraphs(graph))
changing_aids = list(graph.nodes())
nids = ibs.get_annot_nids(changing_aids)
nid2_aids = ut.group_items(changing_aids, nids)
for nid, aids in nid2_aids.items():
# Connect all original names in the database to denote merges
for u, v in ut.itertwo(aids):
graph.add_edge(u, v)
dbside_groups = list(nx.connected_component_subgraphs(graph))
options = [
'Accept',
# 'Review More'
]
msg = (ut.codeblock("""
There are %d names and %d annotations in this mass review set.
Mass review has discovered %d internal groups.
Accepting will induce a database grouping of %d names.
""") % (
len(nid2_aids),
len(changing_aids),
len(review_groups),
len(dbside_groups),
))
reply = gt.user_option(msg=msg, options=options)
if reply == options[0]:
# This is not the smartest way to group names.
# Ideally what will happen here, is that reviewed edges will go into
# the new graph name inference algorithm.
# then the chosen point will be used as the threshold. Then
# the graph cut algorithm will be applied.
logger_ = qres_wgt.logger
logger_.debug(msg)
logger_.info('START MASS_THRESHOLD_MERGE')
logger_.info('num_groups=%d thresh=%r' % (
len(dbside_groups),
thresh,
))
for count, subgraph in enumerate(dbside_groups):
thresh_aid_pairs = [
edge for edge, flag in nx.get_edge_attributes(
graph, 'user_thresh_match').items() if flag
]
thresh_uuid_pairs = ibs.unflat_map(ibs.get_annot_uuids,
thresh_aid_pairs)
aids = list(subgraph.nodes())
nids = ibs.get_annot_name_rowids(aids)
flags = ut.not_list(ibs.is_aid_unknown(aids))
previous_names = ibs.get_name_texts(nids)
valid_nids = ut.compress(nids, flags)
if len(valid_nids) == 0:
merge_nid = ibs.make_next_nids(num=1)[0]
type_ = 'new'
else:
merge_nid = min(valid_nids)
type_ = 'existing'
# Need to find other non-exemplar / query names that may
# need merging
other_aids = ibs.get_name_aids(valid_nids)
other_aids = set(ut.flatten(other_aids)) - set(aids)
other_auuids = ibs.get_annot_uuids(other_aids)
other_previous_names = ibs.get_annot_names(other_aids)
merge_name = ibs.get_name_texts(merge_nid)
annot_uuids = ibs.get_annot_uuids(aids)
###
# Set as reviewed (so we dont see them again), but mark it
# with a different code to denote that it was a MASS review
aid1_list = ut.take_column(thresh_aid_pairs, 0)
aid2_list = ut.take_column(thresh_aid_pairs, 1)
am_rowids = ibs.add_annotmatch_undirected(
aid1_list, aid2_list)
ibs.set_annotmatch_reviewer(
am_rowids, ['algo:lnbnn_thresh'] * len(am_rowids))
logger_.info('START GROUP %d' % (count, ))
logger_.info(
'GROUP BASED ON %d ANNOT_PAIRS WITH SCORE ABOVE (thresh=%r)'
% (
len(thresh_uuid_pairs),
thresh,
))
logger_.debug('(uuid_pairs=%r)' % (thresh_uuid_pairs))
logger_.debug('(merge_name=%r)' % (merge_name))
logger_.debug(
'CHANGE NAME OF %d (annot_uuids=%r) WITH (previous_names=%r) TO (%s) (merge_name=%r)'
% (
len(annot_uuids),
annot_uuids,
previous_names,
type_,
merge_name,
))
logger_.debug(
'ADDITIONAL CHANGE NAME OF %d (annot_uuids=%r) WITH (previous_names=%r) TO (%s) (merge_name=%r)'
% (
len(other_auuids),
other_auuids,
other_previous_names,
type_,
merge_name,
))
logger_.info('END GROUP %d' % (count, ))
new_nids = [merge_nid] * len(aids)
ibs.set_annot_name_rowids(aids, new_nids)
logger_.info('END MASS_THRESHOLD_MERGE')
else:
logger.info('[context] Multiple %d selection' %
(len(selected_qtindex_list), ))
def has_consec_cap_words(words):
for count, (u, v) in enumerate(ut.itertwo(words)):
if u[0].isupper() and v[0].isupper():
if count > 0:
return True
def to_networkx(self):
import utool as ut
return nx.Graph(ut.itertwo(self.tour))
def draw_twoday_count(ibs, visit_info_list_):
import copy
visit_info_list = copy.deepcopy(visit_info_list_)
aids_day1, aids_day2 = ut.take_column(visit_info_list_, 'aids')
nids_day1, nids_day2 = ut.take_column(visit_info_list_, 'unique_nids')
resight_nids = ut.isect(nids_day1, nids_day2)
if False:
# HACK REMOVE DATA TO MAKE THIS FASTER
num = 20
for info in visit_info_list:
non_resight_nids = list(set(info['unique_nids']) - set(resight_nids))
sample_nids2 = non_resight_nids[0:num] + resight_nids[:num]
info['grouped_aids'] = ut.dict_subset(info['grouped_aids'], sample_nids2)
info['unique_nids'] = sample_nids2
# Build a graph of matches
if False:
debug = False
for info in visit_info_list:
edges = []
grouped_aids = info['grouped_aids']
aids_list = list(grouped_aids.values())
ams_list = ibs.get_annotmatch_rowids_in_cliques(aids_list)
aids1_list = ibs.unflat_map(ibs.get_annotmatch_aid1, ams_list)
aids2_list = ibs.unflat_map(ibs.get_annotmatch_aid2, ams_list)
for ams, aids, aids1, aids2 in zip(ams_list, aids_list, aids1_list, aids2_list):
edge_nodes = set(aids1 + aids2)
##if len(edge_nodes) != len(set(aids)):
# #print('--')
# #print('aids = %r' % (aids,))
# #print('edge_nodes = %r' % (edge_nodes,))
bad_aids = edge_nodes - set(aids)
if len(bad_aids) > 0:
print('bad_aids = %r' % (bad_aids,))
unlinked_aids = set(aids) - edge_nodes
mst_links = list(ut.itertwo(list(unlinked_aids) + list(edge_nodes)[:1]))
bad_aids.add(None)
user_links = [(u, v) for (u, v) in zip(aids1, aids2) if u not in bad_aids and v not in bad_aids]
new_edges = mst_links + user_links
new_edges = [(int(u), int(v)) for u, v in new_edges if u not in bad_aids and v not in bad_aids]
edges += new_edges
info['edges'] = edges
# Add edges between days
grouped_aids1, grouped_aids2 = ut.take_column(visit_info_list, 'grouped_aids')
nids_day1, nids_day2 = ut.take_column(visit_info_list, 'unique_nids')
resight_nids = ut.isect(nids_day1, nids_day2)
resight_aids1 = ut.take(grouped_aids1, resight_nids)
resight_aids2 = ut.take(grouped_aids2, resight_nids)
#resight_aids3 = [list(aids1) + list(aids2) for aids1, aids2 in zip(resight_aids1, resight_aids2)]
ams_list = ibs.get_annotmatch_rowids_between_groups(resight_aids1, resight_aids2)
aids1_list = ibs.unflat_map(ibs.get_annotmatch_aid1, ams_list)
aids2_list = ibs.unflat_map(ibs.get_annotmatch_aid2, ams_list)
between_edges = []
for ams, aids1, aids2, rawaids1, rawaids2 in zip(ams_list, aids1_list, aids2_list, resight_aids1, resight_aids2):
link_aids = aids1 + aids2
rawaids3 = rawaids1 + rawaids2
badaids = ut.setdiff(link_aids, rawaids3)
assert not badaids
user_links = [(int(u), int(v)) for (u, v) in zip(aids1, aids2)
if u is not None and v is not None]
# HACK THIS OFF
user_links = []
if len(user_links) == 0:
# Hack in an edge
between_edges += [(rawaids1[0], rawaids2[0])]
else:
between_edges += user_links
assert np.all(0 == np.diff(np.array(ibs.unflat_map(ibs.get_annot_nids, between_edges)), axis=1))
import plottool_ibeis as pt
import networkx as nx
#pt.qt4ensure()
#len(list(nx.connected_components(graph1)))
#print(ut.graph_info(graph1))
# Layout graph
layoutkw = dict(
prog='neato',
draw_implicit=False, splines='line',
#splines='curved',
#splines='spline',
#sep=10 / 72,
#prog='dot', rankdir='TB',
)
def translate_graph_to_origin(graph):
x, y, w, h = ut.get_graph_bounding_box(graph)
ut.translate_graph(graph, (-x, -y))
def stack_graphs(graph_list, vert=False, pad=None):
graph_list_ = [g.copy() for g in graph_list]
for g in graph_list_:
translate_graph_to_origin(g)
bbox_list = [ut.get_graph_bounding_box(g) for g in graph_list_]
if vert:
dim1 = 3
dim2 = 2
else:
dim1 = 2
dim2 = 3
dim1_list = np.array([bbox[dim1] for bbox in bbox_list])
dim2_list = np.array([bbox[dim2] for bbox in bbox_list])
if pad is None:
pad = np.mean(dim1_list) / 2
offset1_list = ut.cumsum([0] + [d + pad for d in dim1_list[:-1]])
max_dim2 = max(dim2_list)
offset2_list = [(max_dim2 - d2) / 2 for d2 in dim2_list]
if vert:
t_xy_list = [(d2, d1) for d1, d2 in zip(offset1_list, offset2_list)]
else:
t_xy_list = [(d1, d2) for d1, d2 in zip(offset1_list, offset2_list)]
for g, t_xy in zip(graph_list_, t_xy_list):
ut.translate_graph(g, t_xy)
nx.set_node_attributes(g, name='pin', values='true')
new_graph = nx.compose_all(graph_list_)
#pt.show_nx(new_graph, layout='custom', node_labels=False, as_directed=False) # NOQA
return new_graph
# Construct graph
for count, info in enumerate(visit_info_list):
graph = nx.Graph()
edges = [(int(u), int(v)) for u, v in info['edges']
if u is not None and v is not None]
graph.add_edges_from(edges, attr_dict={'zorder': 10})
nx.set_node_attributes(graph, name='zorder', values=20)
# Layout in neato
_ = pt.nx_agraph_layout(graph, inplace=True, **layoutkw) # NOQA
# Extract components and then flatten in nid ordering
ccs = list(nx.connected_components(graph))
root_aids = []
cc_graphs = []
for cc_nodes in ccs:
cc = graph.subgraph(cc_nodes)
try:
root_aids.append(list(ut.nx_source_nodes(cc.to_directed()))[0])
except nx.NetworkXUnfeasible:
root_aids.append(list(cc.nodes())[0])
cc_graphs.append(cc)
root_nids = ibs.get_annot_nids(root_aids)
nid2_graph = dict(zip(root_nids, cc_graphs))
resight_nids_ = set(resight_nids).intersection(set(root_nids))
noresight_nids_ = set(root_nids) - resight_nids_
n_graph_list = ut.take(nid2_graph, sorted(noresight_nids_))
r_graph_list = ut.take(nid2_graph, sorted(resight_nids_))
if len(n_graph_list) > 0:
n_graph = nx.compose_all(n_graph_list)
_ = pt.nx_agraph_layout(n_graph, inplace=True, **layoutkw) # NOQA
n_graphs = [n_graph]
else:
n_graphs = []
r_graphs = [stack_graphs(chunk) for chunk in ut.ichunks(r_graph_list, 100)]
if count == 0:
new_graph = stack_graphs(n_graphs + r_graphs, vert=True)
else:
new_graph = stack_graphs(r_graphs[::-1] + n_graphs, vert=True)
#pt.show_nx(new_graph, layout='custom', node_labels=False, as_directed=False) # NOQA
info['graph'] = new_graph
graph1_, graph2_ = ut.take_column(visit_info_list, 'graph')
if False:
_ = pt.show_nx(graph1_, layout='custom', node_labels=False, as_directed=False) # NOQA
_ = pt.show_nx(graph2_, layout='custom', node_labels=False, as_directed=False) # NOQA
graph_list = [graph1_, graph2_]
twoday_graph = stack_graphs(graph_list, vert=True, pad=None)
nx.set_node_attributes(twoday_graph, name='pin', values='true')
if debug:
ut.nx_delete_None_edge_attr(twoday_graph)
ut.nx_delete_None_node_attr(twoday_graph)
print('twoday_graph(pre) info' + ut.repr3(ut.graph_info(twoday_graph), nl=2))
# Hack, no idea why there are nodes that dont exist here
between_edges_ = [edge for edge in between_edges
if twoday_graph.has_node(edge[0]) and twoday_graph.has_node(edge[1])]
twoday_graph.add_edges_from(between_edges_, attr_dict={'alpha': .2, 'zorder': 0})
ut.nx_ensure_agraph_color(twoday_graph)
layoutkw['splines'] = 'line'
layoutkw['prog'] = 'neato'
agraph = pt.nx_agraph_layout(twoday_graph, inplace=True, return_agraph=True, **layoutkw)[-1] # NOQA
if False:
fpath = ut.truepath('~/ggr_graph.png')
agraph.draw(fpath)
ut.startfile(fpath)
if debug:
print('twoday_graph(post) info' + ut.repr3(ut.graph_info(twoday_graph)))
_ = pt.show_nx(twoday_graph, layout='custom', node_labels=False, as_directed=False) # NOQA
def make_expanded_input_graph(graph, target):
"""
Starting from the `target` property we trace all possible paths in the
`graph` back to all sources.
Args:
graph (nx.DiMultiGraph): the dependency graph with a single source.
target (str): a single target node in graph
Notes:
Each edge in the graph must have a `local_input_id` that defines the
type of edge it is: (eg one-to-many, one-to-one, nwise/multi).
# Step 1: Extracting the Relevant Subgraph
We start by searching for all sources of the graph (we assume there is
only one). Then we extract the subgraph defined by all edges between
the sources and the target. We augment this graph with a dummy super
source `s` and super sink `t`. This allows us to associate an edge with
the real source and sink.
# Step 2: Trace all paths from `s` to `t`.
Create a set of all paths from the source to the sink and accumulate
the `local_input_id` of each edge along the path. This will uniquely
identify each path. We use a hack to condense the accumualated ids in
order to display them nicely.
# Step 3: Create the new `exi_graph`
Using the traced paths with ids we construct a new graph representing
expanded inputs. The nodes in the original graph will be copied for each
unique path that passes through the node. We identify these nodes using
the accumulated ids built along the edges in our path set. For each
path starting from the target we add each node augmented with the
accumulated ids on its output(?) edge. We also add the edges along
these paths which results in the final `exi_graph`.
# Step 4: Identify valid inputs candidates
The purpose of this graph is to identify which inputs are needed
to compute dependant properties. One valid set of inputs is all
sources of the graph. However, sometimes it is preferable to specify
a model that may have been trained from many inputs. Therefore any
node with a one-to-many input edge may also be specified as an input.
# Step 5: Identify root-most inputs
The user will only specify one possible set of the inputs. We refer to
this set as the "root-most" inputs. This is a set of candiate nodes
such that all paths from the sink to the super source are blocked. We
default to the set of inputs which results in the fewest dependency
computations. However this is arbitary.
The last step that is not represented here is to compute the order that
the branches must be specified in when given to the depcache for a
computation.
Returns:
nx.DiGraph: exi_graph: the expanded input graph
Notes:
All * nodes are defined to be distinct.
TODO: To make a * node non-distinct it must be suffixed with an
identifier.
CommandLine:
python -m dtool.input_helpers make_expanded_input_graph --show
Example:
>>> # ENABLE_DOCTEST
>>> from dtool.input_helpers import * # NOQA
>>> from dtool.example_depcache2 import * # NOQA
>>> depc = testdata_depc3()
>>> table = depc['smk_match']
>>> table = depc['vsone']
>>> graph = table.depc.explicit_graph.copy()
>>> target = table.tablename
>>> exi_graph = make_expanded_input_graph(graph, target)
>>> x = list(exi_graph.nodes())[0]
>>> print('x = %r' % (x,))
>>> ut.quit_if_noshow()
>>> import plottool as pt
>>> pt.show_nx(graph, fnum=1, pnum=(1, 2, 1))
>>> pt.show_nx(exi_graph, fnum=1, pnum=(1, 2, 2))
>>> ut.show_if_requested()
"""
# FIXME: this does not work correctly when
# The nesting of non-1-to-1 dependencies is greater than 2 (I think)
# algorithm for finding inputs does not work.
# FIXME: two vocabs have the same edge id, they should be the same in the
# Expanded Input Graph as well. Their accum_id needs to be changed.
def condense_accum_ids(rinput_path_id):
# Hack to condense and consolidate graph sources
prev = None
compressed = []
for item in rinput_path_id:
if item == '1' and prev is not None:
pass # done append ones
elif item != prev:
compressed.append(item)
prev = item
#if len(compressed) > 1 and compressed[0] in ['1', '*']:
if len(compressed) > 1 and compressed[0] == '1':
compressed = compressed[1:]
compressed = tuple(compressed)
return compressed
BIG_HACK = True
#BIG_HACK = False
def condense_accum_ids_stars(rinput_path_id):
# Hack to condense and consolidate graph sources
rcompressed = []
has_star = False
# Remove all but the final star (this is a really bad hack)
for item in reversed(rinput_path_id):
is_star = '*' in item
if not (is_star and has_star):
if not has_star:
rcompressed.append(item)
has_star = has_star or is_star
compressed = tuple(rcompressed[::-1])
return compressed
def accumulate_input_ids(edge_list):
"""
python -m dtool.example_depcache2 testdata_depc4 --show
"""
edge_data = ut.take_column(edge_list, 3)
# We are accumulating local input ids
toaccum_list_ = ut.dict_take_column(edge_data, 'local_input_id')
if BIG_HACK and True:
v_list = ut.take_column(edge_list, 1)
# show the local_input_ids at the entire level
pred_ids = ([[
x['local_input_id']
for x in list(graph.pred[node].values())[0].values()
] if len(graph.pred[node]) else [] for node in v_list])
toaccum_list = [
x + ':' + ';'.join(y) for x, y in zip(toaccum_list_, pred_ids)
]
else:
toaccum_list = toaccum_list_
# Default dumb accumulation
accum_ids_ = ut.cumsum(zip(toaccum_list), tuple())
accum_ids = ut.lmap(condense_accum_ids, accum_ids_)
if BIG_HACK:
accum_ids = ut.lmap(condense_accum_ids_stars, accum_ids)
accum_ids = [('t', ) + x for x in accum_ids]
ut.dict_set_column(edge_data, 'accum_id', accum_ids)
return accum_ids
sources = list(ut.nx_source_nodes(graph))
print(sources)
# assert len(sources) == 1, 'expected a unique source'
source = sources[0]
graph = graph.subgraph(ut.nx_all_nodes_between(graph, source,
target)).copy()
# Remove superfluous data
ut.nx_delete_edge_attr(
graph,
[
'edge_type',
'isnwise',
'nwise_idx',
# 'parent_colx',
'ismulti'
])
# Make all '*' edges have distinct local_input_id's.
# TODO: allow non-distinct suffixes
count = ord('a')
for edge in graph.edges(keys=True, data=True):
dat = edge[3]
if dat['local_input_id'] == '*':
dat['local_input_id'] = '*' + chr(count)
dat['taillabel'] = '*' + chr(count)
count += 1
# Augment with dummy super source/sink nodes
source_input = 'source_input'
target_output = 'target_output'
graph.add_edge(source_input, source, local_input_id='s', taillabel='1')
graph.add_edge(target, target_output, local_input_id='t', taillabel='1')
# Find all paths from the table to the source.
paths_to_source = ut.all_multi_paths(graph,
source_input,
target_output,
data=True)
# Build expanded input graph
# The inputs to this table can be derived from this graph.
# The output is a new expanded input graph.
exi_graph = nx.DiGraph()
for path in paths_to_source:
# Accumlate unique identifiers along the reversed path
edge_list = ut.reverse_path_edges(path)
accumulate_input_ids(edge_list)
# A node's output(?) on this path determines its expanded branch id
exi_nodes = [
ExiNode(v, BranchId(d['accum_id'], k, d.get('parent_colx', -1)))
for u, v, k, d in edge_list[:-1]
]
exi_node_to_label = {
node: node[0] + '[' + ','.join([str(x) for x in node[1]]) + ']'
for node in exi_nodes
}
exi_graph.add_nodes_from(exi_nodes)
nx.set_node_attributes(exi_graph,
name='label',
values=exi_node_to_label)
# Undo any accumulation ordering and remove dummy nodes
old_edges = ut.reverse_path_edges(edge_list[1:-1])
new_edges = ut.reverse_path_edges(list(ut.itertwo(exi_nodes)))
for new_edge, old_edge in zip(new_edges, old_edges):
u2, v2 = new_edge[:2]
d = old_edge[3]
taillabel = d['taillabel']
parent_colx = d.get('parent_colx', -1)
if not exi_graph.has_edge(u2, v2):
exi_graph.add_edge(u2,
v2,
taillabel=taillabel,
parent_colx=parent_colx)
sink_nodes = list(ut.nx_sink_nodes(exi_graph))
source_nodes = list(ut.nx_source_nodes(exi_graph))
assert len(sink_nodes) == 1, 'expected a unique sink'
sink_node = sink_nodes[0]
# First identify if a node is root_specifiable
node_dict = ut.nx_node_dict(exi_graph)
for node in exi_graph.nodes():
root_specifiable = False
# for edge in exi_graph.in_edges(node, keys=True):
for edge in exi_graph.in_edges(node):
# key = edge[-1]
# assert key == 0, 'multi di graph is necessary'
edata = exi_graph.get_edge_data(*edge)
if edata.get('taillabel').startswith('*'):
if node != sink_node:
root_specifiable = True
if exi_graph.in_degree(node) == 0:
root_specifiable = True
node_dict[node]['root_specifiable'] = root_specifiable
# Need to specify any combo of red nodes such that
# 1) for each path from a (leaf) to the (root) there is exactly one red
# node along that path. This garentees that all inputs are gievn.
path_list = ut.flatten([
nx.all_simple_paths(exi_graph, source_node, sink_node)
for source_node in source_nodes
])
rootmost_nodes = set([])
for path in path_list:
flags = [node_dict[node]['root_specifiable'] for node in path]
valid_nodes = ut.compress(path, flags)
rootmost_nodes.add(valid_nodes[-1])
# Rootmost nodes are the ones specifiable by default when computing the
# normal property.
for node in rootmost_nodes:
node_dict[node]['rootmost'] = True
# We actually need to hack away any root-most nodes that have another
# rootmost node as the parent. Otherwise, this would cause constraints in
# what the user could specify as valid input combinations.
# ie: specify a vocab and an index, but the index depends on the vocab.
# this forces the user to specify the vocab that was the parent of the index
# the user should either just specify the index and have the vocab inferred
# or for now, we just dont allow this to happen.
nx.get_node_attributes(exi_graph, 'rootmost')
recolor_exi_graph(exi_graph, rootmost_nodes)
return exi_graph
def demodata_bridge():
# define 2-connected compoments and bridges
cc2 = [(1, 2, 4, 3, 1, 4), (8, 9, 10, 8), (11, 12, 13, 11)]
bridges = [(4, 8), (3, 5), (20, 21), (22, 23, 24)]
G = nx.Graph(ut.flatten(ut.itertwo(path) for path in cc2 + bridges))
return G
def _enrich_matches_lnbnn(extr, matches, other_aids, other_nids,
inplace=False):
"""
Given a set of one-vs-one matches, searches for LNBNN normalizers in a
larger database to enrich the matches with database-level
distinctiveness.
"""
from ibeis.algo.hots import nn_weights
raise NotImplementedError('havent tested since the re-work. '
'Need to ensure that things work correctly.')
ibs = extr.ibs
cfgdict = {
'can_match_samename': False,
'can_match_sameimg': True,
'K': 3,
'Knorm': 3,
'prescore_method': 'csum',
'score_method': 'csum'
}
custom_nid_lookup = ut.dzip(other_aids, other_nids)
aids = [m.annot2['aid'] for m in matches]
qreq_ = ibs.new_query_request(aids, other_aids, cfgdict=cfgdict,
custom_nid_lookup=custom_nid_lookup,
verbose=extr.verbose >= 2)
qreq_.load_indexer()
indexer = qreq_.indexer
if not inplace:
matches_ = [match.copy() for match in matches]
else:
matches_ = matches
K = qreq_.qparams.K
Knorm = qreq_.qparams.Knorm
normalizer_rule = qreq_.qparams.normalizer_rule
extr.print('Stacking vecs for batch lnbnn matching')
offset_list = np.cumsum([0] + [match_.fm.shape[0] for match_ in matches_])
stacked_vecs = np.vstack([
match_.matched_vecs2()
for match_ in ut.ProgIter(matches_, label='stack matched vecs')
])
vecs = stacked_vecs
num = (K + Knorm)
idxs, dists = indexer.batch_knn(vecs, num, chunksize=8192,
label='lnbnn scoring')
idx_list = [idxs[l:r] for l, r in ut.itertwo(offset_list)]
dist_list = [dists[l:r] for l, r in ut.itertwo(offset_list)]
iter_ = zip(matches_, idx_list, dist_list)
prog = ut.ProgIter(iter_, length=len(matches_), label='lnbnn scoring')
for match_, neighb_idx, neighb_dist in prog:
qaid = match_.annot2['aid']
norm_k = nn_weights.get_normk(qreq_, qaid, neighb_idx, Knorm,
normalizer_rule)
ndist = vt.take_col_per_row(neighb_dist, norm_k)
vdist = match_.local_measures['match_dist']
lnbnn_dist = nn_weights.lnbnn_fn(vdist, ndist)
lnbnn_clip_dist = np.clip(lnbnn_dist, 0, np.inf)
match_.local_measures['lnbnn_norm_dist'] = ndist
match_.local_measures['lnbnn'] = lnbnn_dist
match_.local_measures['lnbnn_clip'] = lnbnn_clip_dist
match_.fs = lnbnn_dist
return matches_
aug_graph = graph.copy()
# remove cut edges from augmented graph
edge_to_iscut = nx.get_edge_attributes(aug_graph, 'is_cut')
cut_edges = [
(u, v)
for (u, v, d) in aug_graph.edges(data=True)
if not (d.get('is_cut') or d.get('decision', 'unreviewed') in ['nomatch'])
]
cut_edges = [edge for edge, flag in edge_to_iscut.items() if flag]
aug_graph.remove_edges_from(cut_edges)
# Enumerate cliques inside labels
unflat_edges = [list(ut.itertwo(nodes)) for nodes in label_to_nodes.values()]
node_pairs = [tup for tup in ut.iflatten(unflat_edges) if tup[0] != tup[1]]
# Remove candidate MST edges that exist in the original graph
orig_edges = list(aug_graph.edges())
candidate_mst_edges = [edge for edge in node_pairs if not aug_graph.has_edge(*edge)]
# randomness prevents chains and visually looks better
rng = np.random.RandomState(42)
def _randint():
return 0
return rng.randint(0, 100)
aug_graph.add_edges_from(candidate_mst_edges)
def nx_transitive_reduction(G, mode=1):
"""
References:
https://en.wikipedia.org/wiki/Transitive_reduction#Computing_the_reduction_using_the_closure
http://dept-info.labri.fr/~thibault/tmp/0201008.pdf
http://stackoverflow.com/questions/17078696/im-trying-to-perform-the-transitive-reduction-of-directed-graph-in-python
CommandLine:
python -m utool.util_graph nx_transitive_reduction --show
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_graph import * # NOQA
>>> import utool as ut
>>> import networkx as nx
>>> G = nx.DiGraph([('a', 'b'), ('a', 'c'), ('a', 'e'),
>>> ('a', 'd'), ('b', 'd'), ('c', 'e'),
>>> ('d', 'e'), ('c', 'e'), ('c', 'd')])
>>> G = testdata_graph()[1]
>>> G_tr = nx_transitive_reduction(G, mode=1)
>>> G_tr2 = nx_transitive_reduction(G, mode=1)
>>> ut.quit_if_noshow()
>>> import plottool as pt
>>> G_ = nx.dag.transitive_closure(G)
>>> pt.show_nx(G , pnum=(1, 5, 1), fnum=1)
>>> pt.show_nx(G_tr , pnum=(1, 5, 2), fnum=1)
>>> pt.show_nx(G_tr2 , pnum=(1, 5, 3), fnum=1)
>>> pt.show_nx(G_ , pnum=(1, 5, 4), fnum=1)
>>> pt.show_nx(nx.dag.transitive_closure(G_tr), pnum=(1, 5, 5), fnum=1)
>>> ut.show_if_requested()
"""
import utool as ut
import networkx as nx
has_cycles = not nx.is_directed_acyclic_graph(G)
if has_cycles:
# FIXME: this does not work for cycle graphs.
# Need to do algorithm on SCCs
G_orig = G
G = nx.condensation(G_orig)
nodes = list(G.nodes())
node2_idx = ut.make_index_lookup(nodes)
# For each node u, perform DFS consider its set of (non-self) children C.
# For each descendant v, of a node in C, remove any edge from u to v.
if mode == 1:
G_tr = G.copy()
for parent in G_tr.nodes():
# Remove self loops
if G_tr.has_edge(parent, parent):
G_tr.remove_edge(parent, parent)
# For each child of the parent
for child in list(G_tr.successors(parent)):
# Preorder nodes includes its argument (no added complexity)
for gchild in list(G_tr.successors(child)):
# Remove all edges from parent to non-child descendants
for descendant in nx.dfs_preorder_nodes(G_tr, gchild):
if G_tr.has_edge(parent, descendant):
G_tr.remove_edge(parent, descendant)
if has_cycles:
# Uncondense graph
uncondensed_G_tr = G.__class__()
mapping = G.graph['mapping']
uncondensed_G_tr.add_nodes_from(mapping.keys())
inv_mapping = ut.invert_dict(mapping, unique_vals=False)
for u, v in G_tr.edges():
u_ = inv_mapping[u][0]
v_ = inv_mapping[v][0]
uncondensed_G_tr.add_edge(u_, v_)
for key, path in inv_mapping.items():
if len(path) > 1:
directed_cycle = list(ut.itertwo(path, wrap=True))
uncondensed_G_tr.add_edges_from(directed_cycle)
G_tr = uncondensed_G_tr
else:
def make_adj_matrix(G):
edges = list(G.edges())
edge2_idx = ut.partial(ut.dict_take, node2_idx)
uv_list = ut.lmap(edge2_idx, edges)
A = np.zeros((len(nodes), len(nodes)))
A[tuple(np.array(uv_list).T)] = 1
return A
G_ = nx.dag.transitive_closure(G)
A = make_adj_matrix(G)
B = make_adj_matrix(G_)
#AB = A * B
#AB = A.T.dot(B)
AB = A.dot(B)
#AB = A.dot(B.T)
A_and_notAB = np.logical_and(A, np.logical_not(AB))
tr_uvs = np.where(A_and_notAB)
#nodes = G.nodes()
edges = list(zip(*ut.unflat_take(nodes, tr_uvs)))
G_tr = G.__class__()
G_tr.add_nodes_from(nodes)
G_tr.add_edges_from(edges)
if has_cycles:
# Uncondense graph
uncondensed_G_tr = G.__class__()
mapping = G.graph['mapping']
uncondensed_G_tr.add_nodes_from(mapping.keys())
inv_mapping = ut.invert_dict(mapping, unique_vals=False)
for u, v in G_tr.edges():
u_ = inv_mapping[u][0]
v_ = inv_mapping[v][0]
uncondensed_G_tr.add_edge(u_, v_)
for key, path in inv_mapping.items():
if len(path) > 1:
directed_cycle = list(ut.itertwo(path, wrap=True))
uncondensed_G_tr.add_edges_from(directed_cycle)
G_tr = uncondensed_G_tr
return G_tr
def draw_bayesian_model(model, evidence={}, soft_evidence={}, fnum=None,
pnum=None, **kwargs):
from pgmpy.models import BayesianModel
if not isinstance(model, BayesianModel):
model = model.to_bayesian_model()
import plottool as pt
import networkx as nx
kwargs = kwargs.copy()
factor_list = kwargs.pop('factor_list', [])
ttype_colors, ttype_scalars = make_colorcodes(model)
textprops = {
'horizontalalignment': 'left', 'family': 'monospace', 'size': 8, }
# build graph attrs
tup = get_node_viz_attrs(
model, evidence, soft_evidence, factor_list, ttype_colors, **kwargs)
node_color, pos_list, pos_dict, takws = tup
# draw graph
has_infered = evidence or 'factor_list' in kwargs
if False:
fig = pt.figure(fnum=fnum, pnum=pnum, doclf=True) # NOQA
ax = pt.gca()
drawkw = dict(pos=pos_dict, ax=ax, with_labels=True, node_size=1100,
node_color=node_color)
nx.draw(model, **drawkw)
else:
# BE VERY CAREFUL
if 1:
graph = model.copy()
graph.__class__ = nx.DiGraph
graph.graph['groupattrs'] = ut.ddict(dict)
#graph = model.
if getattr(graph, 'ttype2_cpds', None) is not None:
# Add invis edges and ttype groups
for ttype in model.ttype2_cpds.keys():
ttype_cpds = model.ttype2_cpds[ttype]
# use defined ordering
ttype_nodes = ut.list_getattr(ttype_cpds, 'variable')
# ttype_nodes = sorted(ttype_nodes)
invis_edges = list(ut.itertwo(ttype_nodes))
graph.add_edges_from(invis_edges)
nx.set_edge_attributes(graph, 'style', {edge: 'invis' for edge in invis_edges})
nx.set_node_attributes(graph, 'groupid', {node: ttype for node in ttype_nodes})
graph.graph['groupattrs'][ttype]['rank'] = 'same'
graph.graph['groupattrs'][ttype]['cluster'] = False
else:
graph = model
pt.show_nx(graph, layout_kw={'prog': 'dot'}, fnum=fnum, pnum=pnum, verbose=0)
pt.zoom_factory()
fig = pt.gcf()
ax = pt.gca()
pass
hacks = [pt.draw_text_annotations(textprops=textprops, **takw)
for takw in takws if takw]
xmin, ymin = np.array(pos_list).min(axis=0)
xmax, ymax = np.array(pos_list).max(axis=0)
if 'name' in model.ttype2_template:
num_names = len(model.ttype2_template['name'].basis)
num_annots = len(model.ttype2_cpds['name'])
if num_annots > 4:
ax.set_xlim((xmin - 40, xmax + 40))
ax.set_ylim((ymin - 50, ymax + 50))
fig.set_size_inches(30, 7)
else:
ax.set_xlim((xmin - 42, xmax + 42))
ax.set_ylim((ymin - 50, ymax + 50))
fig.set_size_inches(23, 7)
title = 'num_names=%r, num_annots=%r' % (num_names, num_annots,)
else:
title = ''
map_assign = kwargs.get('map_assign', None)
def word_insert(text):
return '' if len(text) == 0 else text + ' '
top_assignments = kwargs.get('top_assignments', None)
if top_assignments is not None:
map_assign, map_prob = top_assignments[0]
if map_assign is not None:
title += '\n%sMAP: ' % (word_insert(kwargs.get('method', '')))
title += map_assign + ' @' + '%.2f%%' % (100 * map_prob,)
if kwargs.get('show_title', True):
pt.set_figtitle(title, size=14)
for hack in hacks:
hack()
if has_infered:
# Hack in colorbars
# if ut.list_type(basis) is int:
# pt.colorbar(scalars, colors, lbl='score', ticklabels=np.array(basis) + 1)
# else:
# pt.colorbar(scalars, colors, lbl='score', ticklabels=basis)
keys = ['name', 'score']
locs = ['left', 'right']
for key, loc in zip(keys, locs):
if key in ttype_colors:
basis = model.ttype2_template[key].basis
# scalars =
colors = ttype_colors[key]
scalars = ttype_scalars[key]
pt.colorbar(scalars, colors, lbl=key, ticklabels=basis,
ticklocation=loc)
