def setcover_example():
"""
CommandLine:
python -m ibeis.scripts.specialdraw setcover_example --show
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.scripts.specialdraw import * # NOQA
>>> result = setcover_example()
>>> print(result)
>>> ut.quit_if_noshow()
>>> import plottool as pt
>>> ut.show_if_requested()
"""
import ibeis
import plottool as pt
from ibeis.viz import viz_graph
import networkx as nx
pt.ensure_pylab_qt4()
ibs = ibeis.opendb(defaultdb='testdb2')
if False:
# Select a good set
aids = ibs.get_name_aids(ibs.get_valid_nids())
# ibeis.testdata_aids('testdb2', a='default:mingt=2')
aids = [a for a in aids if len(a) > 1]
for a in aids:
print(ut.repr3(ibs.get_annot_stats_dict(a)))
print(aids[-2])
#aids = [78, 79, 80, 81, 88, 91]
aids = [78, 79, 81, 88, 91]
qreq_ = ibs.depc.new_request('vsone', aids, aids, cfgdict={})
cm_list = qreq_.execute()
from ibeis.algo.hots import graph_iden
infr = graph_iden.AnnotInference(cm_list)
unique_aids, prob_annots = infr.make_prob_annots()
import numpy as np
print(
ut.hz_str(
'prob_annots = ',
ut.array2string2(prob_annots,
precision=2,
max_line_width=140,
suppress_small=True)))
# ut.setcover_greedy(candidate_sets_dict)
max_weight = 3
prob_annots[np.diag_indices(len(prob_annots))] = np.inf
prob_annots = prob_annots
thresh_points = np.sort(prob_annots[np.isfinite(prob_annots)])
# probably not the best way to go about searching for these thresholds
# but when you have a hammer...
if False:
quant = sorted(np.diff(thresh_points))[(len(thresh_points) - 1) // 2]
candset = {
point: thresh_points[np.abs(thresh_points - point) < quant]
for point in thresh_points
}
check_thresholds = len(aids) * 2
thresh_points2 = np.array(
ut.setcover_greedy(candset, max_weight=check_thresholds).keys())
thresh_points = thresh_points2
# pt.plot(sorted(thresh_points), 'rx')
# pt.plot(sorted(thresh_points2), 'o')
# prob_annots = prob_annots.T
# thresh_start = np.mean(thresh_points)
current_idxs = []
current_covers = []
current_val = np.inf
for thresh in thresh_points:
covering_sets = [np.where(row >= thresh)[0] for row in (prob_annots)]
candidate_sets_dict = {
ax: others
for ax, others in enumerate(covering_sets)
}
soln_cover = ut.setcover_ilp(candidate_sets_dict,
max_weight=max_weight)
exemplar_idxs = list(soln_cover.keys())
soln_weight = len(exemplar_idxs)
val = max_weight - soln_weight
# print('val = %r' % (val,))
# print('soln_weight = %r' % (soln_weight,))
if val < current_val:
current_val = val
current_covers = covering_sets
current_idxs = exemplar_idxs
exemplars = ut.take(aids, current_idxs)
ensure_edges = [(aids[ax], aids[ax2])
for ax, other_xs in enumerate(current_covers)
for ax2 in other_xs]
graph = viz_graph.make_netx_graph_from_aid_groups(
ibs, [aids],
allow_directed=True,
ensure_edges=ensure_edges,
temp_nids=[1] * len(aids))
viz_graph.ensure_node_images(ibs, graph)
nx.set_node_attributes(graph, 'framewidth', False)
nx.set_node_attributes(graph, 'framewidth',
{aid: 4.0
for aid in exemplars})
nx.set_edge_attributes(graph, 'color', pt.ORANGE)
nx.set_node_attributes(graph, 'color', pt.LIGHT_BLUE)
nx.set_node_attributes(graph, 'shape', 'rect')
layoutkw = {
'sep': 1 / 10,
'prog': 'neato',
'overlap': 'false',
#'splines': 'ortho',
'splines': 'spline',
}
pt.show_nx(graph, layout='agraph', layoutkw=layoutkw)
pt.zoom_factory()
def make_graph(infr, show=False):
import networkx as nx
import itertools
cm_list = infr.cm_list
unique_nids, prob_names = infr.make_prob_names()
thresh = infr.choose_thresh()
# Simply cut any edge with a weight less than a threshold
qaid_list = [cm.qaid for cm in cm_list]
postcut = prob_names > thresh
qxs, nxs = np.where(postcut)
if False:
kw = dict(precision=2, max_line_width=140, suppress_small=True)
print(
ut.hz_str('prob_names = ', ut.array2string2((prob_names),
**kw)))
print(
ut.hz_str('postcut = ',
ut.array2string2((postcut).astype(np.int), **kw)))
matching_qaids = ut.take(qaid_list, qxs)
matched_nids = ut.take(unique_nids, nxs)
qreq_ = infr.qreq_
nodes = ut.unique(qreq_.qaids.tolist() + qreq_.daids.tolist())
if not hasattr(qreq_, 'dnids'):
qreq_.dnids = qreq_.ibs.get_annot_nids(qreq_.daids)
qreq_.qnids = qreq_.ibs.get_annot_nids(qreq_.qaids)
dnid2_daids = ut.group_items(qreq_.daids, qreq_.dnids)
grouped_aids = dnid2_daids.values()
matched_daids = ut.take(dnid2_daids, matched_nids)
name_cliques = [
list(itertools.combinations(aids, 2)) for aids in grouped_aids
]
aid_matches = [
list(ut.product([qaid], daids))
for qaid, daids in zip(matching_qaids, matched_daids)
]
graph = nx.Graph()
graph.add_nodes_from(nodes)
graph.add_edges_from(ut.flatten(name_cliques))
graph.add_edges_from(ut.flatten(aid_matches))
#matchless_quries = ut.take(qaid_list, ut.index_complement(qxs, len(qaid_list)))
name_nodes = [('nid', l) for l in qreq_.dnids]
db_aid_nid_edges = list(zip(qreq_.daids, name_nodes))
#query_aid_nid_edges = list(zip(matching_qaids, [('nid', l) for l in matched_nids]))
#G = nx.Graph()
#G.add_nodes_from(matchless_quries)
#G.add_edges_from(db_aid_nid_edges)
#G.add_edges_from(query_aid_nid_edges)
graph.add_edges_from(db_aid_nid_edges)
if infr.user_feedback is not None:
user_feedback = ut.map_dict_vals(np.array, infr.user_feedback)
p_bg = 0.0
part1 = user_feedback['p_match'] * (1 - user_feedback['p_notcomp'])
part2 = p_bg * user_feedback['p_notcomp']
p_same_list = part1 + part2
for aid1, aid2, p_same in zip(user_feedback['aid1'],
user_feedback['aid2'], p_same_list):
if p_same > .5:
if not graph.has_edge(aid1, aid2):
graph.add_edge(aid1, aid2)
else:
if graph.has_edge(aid1, aid2):
graph.remove_edge(aid1, aid2)
if show:
import plottool as pt
nx.set_node_attributes(graph, 'color',
{aid: pt.LIGHT_PINK
for aid in qreq_.daids})
nx.set_node_attributes(graph, 'color',
{aid: pt.TRUE_BLUE
for aid in qreq_.qaids})
nx.set_node_attributes(
graph, 'color', {
aid: pt.LIGHT_PURPLE
for aid in np.intersect1d(qreq_.qaids, qreq_.daids)
})
nx.set_node_attributes(
graph, 'label',
{node: 'n%r' % (node[1], )
for node in name_nodes})
nx.set_node_attributes(
graph, 'color', {node: pt.LIGHT_GREEN
for node in name_nodes})
if show:
import plottool as pt
pt.show_nx(graph, layoutkw={'prog': 'neato'}, verbose=False)
return graph
def test_em2(prob_names, prob_annots=None):
"""
assert prob_names.shape == (nAnnots, nNames)
"""
learn_rate = 0.05
num_iters = 1
# Matrix if unary probabilites, The probability that each node takes on a
# given label, independent of its edges.
num_annots, num_names = prob_names.shape
# prevent zero probabilities
prob_names_ = prob_names + 1E-9
prob_names_ /= prob_names_.sum(axis=1)[:, None]
if prob_annots is None:
prob_annots_ = np.full((num_annots, num_annots), 1 / num_annots)
prob_annots_[np.diag_indices(num_annots)] *= 1.01
# perterb
rng = np.random.RandomState(0)
prob_annots_ += (rng.randn(*prob_annots_.shape)) / 100
prob_annots_ /= prob_annots_.sum(axis=1)[:, None]
prob_annots_ = (prob_annots_.T + prob_annots_) / 2
else:
prob_annots_ = prob_annots + 1E-9
prob_annots_ /= prob_annots_.sum(axis=1)[:, None]
# Stack everything into a single matrix
prob_part = np.hstack([prob_names_, prob_annots_])
zero_part = np.zeros((num_names, num_annots + num_names))
prior = np.vstack([zero_part, prob_part])
# Gamma will hold a probability distribution over the nodes
# The labeled nodes must match themselves.
# The unlabeld nodes are initialized with a uniform distribution.
gam = np.hstack([np.eye(num_names), np.ones((num_names, num_annots)) / num_names])
verbose = 1
if verbose:
print('Initialize')
print('num_names = %r' % (num_names,))
print(ut.hz_str('prior = ', ut.array2string2(prob_part[:, :], precision=2, max_line_width=140, suppress_small=True)))
print(ut.hz_str('gamma = ', ut.array2string2(gam[:, :], max_line_width=140, precision=2, suppress_small=True)))
#print(ut.hz_str(' gamma = ', ut.array_repr2(gam, max_line_width=140, precision=2)))
delta_i = np.zeros(num_names)
def dErr(i, gam, prior, delta_i=delta_i, num_names=num_names):
# exepcted liklihood is cross entropy error
delta_i[:] = 0
# Compute the gradient of the cross entropy error
# This is over both names and annotations
jdxs = [j for j in range(prior.shape[0]) if j != i]
prior_ij = prior[i, jdxs]
np.log(prior_ij / (1 - prior_ij))
gam[:, jdxs]
for j in range(prior.shape[0]):
if i != j:
delta_i += gam[:, j] * np.log(prior[i, j] / (1 - prior[i, j]))
# compute the projected gradient
delta_i_hat = delta_i - delta_i.sum() / num_names
return delta_i_hat
# Build node for each annot and each name
num_nodes = num_annots + num_names
# Maximies the expected liklihood of gamma
dGam = np.zeros(gam.shape)
# for count in range(num_iters):
for count in ut.ProgIter(range(num_iters), label='EM', bs=True):
# Compute error gradient
for i in range(num_names, num_nodes):
dGam[:, i] = dErr(i, gam, prior)
# Make a step in the gradient direction
# print(ut.hz_str(' dGam = ', ut.array_repr2(dGam, max_line_width=140, precision=2)))
gam = gam + learn_rate * dGam
# Normalize
gam = np.clip(gam, 0, 1)
for i in range(num_names, num_nodes):
gam[:, i] = gam[:, i] / np.sum(gam[:, i])
# print(ut.hz_str(' gamma = ', ut.array_repr2(gam, max_line_width=140, precision=2)))
if verbose:
print(ut.hz_str(' gamma = ', ut.array2string2(gam[:, num_names:], max_line_width=140, precision=2, suppress_small=True)))
print('Finished')
return gam
def make_graph(infr, show=False):
import networkx as nx
import itertools
cm_list = infr.cm_list
unique_nids, prob_names = infr.make_prob_names()
thresh = infr.choose_thresh()
# Simply cut any edge with a weight less than a threshold
qaid_list = [cm.qaid for cm in cm_list]
postcut = prob_names > thresh
qxs, nxs = np.where(postcut)
if False:
kw = dict(precision=2, max_line_width=140, suppress_small=True)
print(ut.hz_str('prob_names = ', ut.array2string2((prob_names), **kw)))
print(ut.hz_str('postcut = ', ut.array2string2((postcut).astype(np.int), **kw)))
matching_qaids = ut.take(qaid_list, qxs)
matched_nids = ut.take(unique_nids, nxs)
qreq_ = infr.qreq_
nodes = ut.unique(qreq_.qaids.tolist() + qreq_.daids.tolist())
if not hasattr(qreq_, 'dnids'):
qreq_.dnids = qreq_.ibs.get_annot_nids(qreq_.daids)
qreq_.qnids = qreq_.ibs.get_annot_nids(qreq_.qaids)
dnid2_daids = ut.group_items(qreq_.daids, qreq_.dnids)
grouped_aids = dnid2_daids.values()
matched_daids = ut.take(dnid2_daids, matched_nids)
name_cliques = [list(itertools.combinations(aids, 2)) for aids in grouped_aids]
aid_matches = [list(ut.product([qaid], daids)) for qaid, daids in
zip(matching_qaids, matched_daids)]
graph = nx.Graph()
graph.add_nodes_from(nodes)
graph.add_edges_from(ut.flatten(name_cliques))
graph.add_edges_from(ut.flatten(aid_matches))
#matchless_quries = ut.take(qaid_list, ut.index_complement(qxs, len(qaid_list)))
name_nodes = [('nid', l) for l in qreq_.dnids]
db_aid_nid_edges = list(zip(qreq_.daids, name_nodes))
#query_aid_nid_edges = list(zip(matching_qaids, [('nid', l) for l in matched_nids]))
#G = nx.Graph()
#G.add_nodes_from(matchless_quries)
#G.add_edges_from(db_aid_nid_edges)
#G.add_edges_from(query_aid_nid_edges)
graph.add_edges_from(db_aid_nid_edges)
if infr.user_feedback is not None:
user_feedback = ut.map_dict_vals(np.array, infr.user_feedback)
p_bg = 0.0
part1 = user_feedback['p_match'] * (1 - user_feedback['p_notcomp'])
part2 = p_bg * user_feedback['p_notcomp']
p_same_list = part1 + part2
for aid1, aid2, p_same in zip(user_feedback['aid1'],
user_feedback['aid2'], p_same_list):
if p_same > .5:
if not graph.has_edge(aid1, aid2):
graph.add_edge(aid1, aid2)
else:
if graph.has_edge(aid1, aid2):
graph.remove_edge(aid1, aid2)
if show:
import plottool as pt
nx.set_node_attributes(graph, 'color', {aid: pt.LIGHT_PINK
for aid in qreq_.daids})
nx.set_node_attributes(graph, 'color', {aid: pt.TRUE_BLUE
for aid in qreq_.qaids})
nx.set_node_attributes(graph, 'color', {
aid: pt.LIGHT_PURPLE
for aid in np.intersect1d(qreq_.qaids, qreq_.daids)})
nx.set_node_attributes(graph, 'label', {node: 'n%r' % (node[1],)
for node in name_nodes})
nx.set_node_attributes(graph, 'color', {node: pt.LIGHT_GREEN
for node in name_nodes})
if show:
import plottool as pt
pt.show_nx(graph, layoutkw={'prog': 'neato'}, verbose=False)
return graph
def setcover_example():
"""
CommandLine:
python -m ibeis.scripts.specialdraw setcover_example --show
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.scripts.specialdraw import * # NOQA
>>> result = setcover_example()
>>> print(result)
>>> ut.quit_if_noshow()
>>> import plottool as pt
>>> ut.show_if_requested()
"""
import ibeis
import plottool as pt
from ibeis.viz import viz_graph
import networkx as nx
pt.ensure_pylab_qt4()
ibs = ibeis.opendb(defaultdb='testdb2')
if False:
# Select a good set
aids = ibs.get_name_aids(ibs.get_valid_nids())
# ibeis.testdata_aids('testdb2', a='default:mingt=2')
aids = [a for a in aids if len(a) > 1]
for a in aids:
print(ut.repr3(ibs.get_annot_stats_dict(a)))
print(aids[-2])
#aids = [78, 79, 80, 81, 88, 91]
aids = [78, 79, 81, 88, 91]
qreq_ = ibs.depc.new_request('vsone', aids, aids, cfgdict={})
cm_list = qreq_.execute()
from ibeis.algo.hots import graph_iden
infr = graph_iden.AnnotInference(cm_list)
unique_aids, prob_annots = infr.make_prob_annots()
import numpy as np
print(ut.hz_str('prob_annots = ', ut.array2string2(prob_annots, precision=2, max_line_width=140, suppress_small=True)))
# ut.setcover_greedy(candidate_sets_dict)
max_weight = 3
prob_annots[np.diag_indices(len(prob_annots))] = np.inf
prob_annots = prob_annots
thresh_points = np.sort(prob_annots[np.isfinite(prob_annots)])
# probably not the best way to go about searching for these thresholds
# but when you have a hammer...
if False:
quant = sorted(np.diff(thresh_points))[(len(thresh_points) - 1) // 2 ]
candset = {point: thresh_points[np.abs(thresh_points - point) < quant] for point in thresh_points}
check_thresholds = len(aids) * 2
thresh_points2 = np.array(ut.setcover_greedy(candset, max_weight=check_thresholds).keys())
thresh_points = thresh_points2
# pt.plot(sorted(thresh_points), 'rx')
# pt.plot(sorted(thresh_points2), 'o')
# prob_annots = prob_annots.T
# thresh_start = np.mean(thresh_points)
current_idxs = []
current_covers = []
current_val = np.inf
for thresh in thresh_points:
covering_sets = [np.where(row >= thresh)[0] for row in (prob_annots)]
candidate_sets_dict = {ax: others for ax, others in enumerate(covering_sets)}
soln_cover = ut.setcover_ilp(candidate_sets_dict, max_weight=max_weight)
exemplar_idxs = list(soln_cover.keys())
soln_weight = len(exemplar_idxs)
val = max_weight - soln_weight
# print('val = %r' % (val,))
# print('soln_weight = %r' % (soln_weight,))
if val < current_val:
current_val = val
current_covers = covering_sets
current_idxs = exemplar_idxs
exemplars = ut.take(aids, current_idxs)
ensure_edges = [(aids[ax], aids[ax2]) for ax, other_xs in enumerate(current_covers) for ax2 in other_xs]
graph = viz_graph.make_netx_graph_from_aid_groups(
ibs, [aids], allow_directed=True, ensure_edges=ensure_edges,
temp_nids=[1] * len(aids))
viz_graph.ensure_node_images(ibs, graph)
nx.set_node_attributes(graph, 'framewidth', False)
nx.set_node_attributes(graph, 'framewidth', {aid: 4.0 for aid in exemplars})
nx.set_edge_attributes(graph, 'color', pt.ORANGE)
nx.set_node_attributes(graph, 'color', pt.LIGHT_BLUE)
nx.set_node_attributes(graph, 'shape', 'rect')
layoutkw = {
'sep' : 1 / 10,
'prog': 'neato',
'overlap': 'false',
#'splines': 'ortho',
'splines': 'spline',
}
pt.show_nx(graph, layout='agraph', layoutkw=layoutkw)
pt.zoom_factory()