def setcover_example(): """ CommandLine: python -m ibeis.scripts.specialdraw setcover_example --show Example: >>> # DISABLE_DOCTEST >>> from ibeis.scripts.specialdraw import * # NOQA >>> result = setcover_example() >>> print(result) >>> ut.quit_if_noshow() >>> import plottool as pt >>> ut.show_if_requested() """ import ibeis import plottool as pt from ibeis.viz import viz_graph import networkx as nx pt.ensure_pylab_qt4() ibs = ibeis.opendb(defaultdb='testdb2') if False: # Select a good set aids = ibs.get_name_aids(ibs.get_valid_nids()) # ibeis.testdata_aids('testdb2', a='default:mingt=2') aids = [a for a in aids if len(a) > 1] for a in aids: print(ut.repr3(ibs.get_annot_stats_dict(a))) print(aids[-2]) #aids = [78, 79, 80, 81, 88, 91] aids = [78, 79, 81, 88, 91] qreq_ = ibs.depc.new_request('vsone', aids, aids, cfgdict={}) cm_list = qreq_.execute() from ibeis.algo.hots import graph_iden infr = graph_iden.AnnotInference(cm_list) unique_aids, prob_annots = infr.make_prob_annots() import numpy as np print( ut.hz_str( 'prob_annots = ', ut.array2string2(prob_annots, precision=2, max_line_width=140, suppress_small=True))) # ut.setcover_greedy(candidate_sets_dict) max_weight = 3 prob_annots[np.diag_indices(len(prob_annots))] = np.inf prob_annots = prob_annots thresh_points = np.sort(prob_annots[np.isfinite(prob_annots)]) # probably not the best way to go about searching for these thresholds # but when you have a hammer... if False: quant = sorted(np.diff(thresh_points))[(len(thresh_points) - 1) // 2] candset = { point: thresh_points[np.abs(thresh_points - point) < quant] for point in thresh_points } check_thresholds = len(aids) * 2 thresh_points2 = np.array( ut.setcover_greedy(candset, max_weight=check_thresholds).keys()) thresh_points = thresh_points2 # pt.plot(sorted(thresh_points), 'rx') # pt.plot(sorted(thresh_points2), 'o') # prob_annots = prob_annots.T # thresh_start = np.mean(thresh_points) current_idxs = [] current_covers = [] current_val = np.inf for thresh in thresh_points: covering_sets = [np.where(row >= thresh)[0] for row in (prob_annots)] candidate_sets_dict = { ax: others for ax, others in enumerate(covering_sets) } soln_cover = ut.setcover_ilp(candidate_sets_dict, max_weight=max_weight) exemplar_idxs = list(soln_cover.keys()) soln_weight = len(exemplar_idxs) val = max_weight - soln_weight # print('val = %r' % (val,)) # print('soln_weight = %r' % (soln_weight,)) if val < current_val: current_val = val current_covers = covering_sets current_idxs = exemplar_idxs exemplars = ut.take(aids, current_idxs) ensure_edges = [(aids[ax], aids[ax2]) for ax, other_xs in enumerate(current_covers) for ax2 in other_xs] graph = viz_graph.make_netx_graph_from_aid_groups( ibs, [aids], allow_directed=True, ensure_edges=ensure_edges, temp_nids=[1] * len(aids)) viz_graph.ensure_node_images(ibs, graph) nx.set_node_attributes(graph, 'framewidth', False) nx.set_node_attributes(graph, 'framewidth', {aid: 4.0 for aid in exemplars}) nx.set_edge_attributes(graph, 'color', pt.ORANGE) nx.set_node_attributes(graph, 'color', pt.LIGHT_BLUE) nx.set_node_attributes(graph, 'shape', 'rect') layoutkw = { 'sep': 1 / 10, 'prog': 'neato', 'overlap': 'false', #'splines': 'ortho', 'splines': 'spline', } pt.show_nx(graph, layout='agraph', layoutkw=layoutkw) pt.zoom_factory()
def make_graph(infr, show=False): import networkx as nx import itertools cm_list = infr.cm_list unique_nids, prob_names = infr.make_prob_names() thresh = infr.choose_thresh() # Simply cut any edge with a weight less than a threshold qaid_list = [cm.qaid for cm in cm_list] postcut = prob_names > thresh qxs, nxs = np.where(postcut) if False: kw = dict(precision=2, max_line_width=140, suppress_small=True) print( ut.hz_str('prob_names = ', ut.array2string2((prob_names), **kw))) print( ut.hz_str('postcut = ', ut.array2string2((postcut).astype(np.int), **kw))) matching_qaids = ut.take(qaid_list, qxs) matched_nids = ut.take(unique_nids, nxs) qreq_ = infr.qreq_ nodes = ut.unique(qreq_.qaids.tolist() + qreq_.daids.tolist()) if not hasattr(qreq_, 'dnids'): qreq_.dnids = qreq_.ibs.get_annot_nids(qreq_.daids) qreq_.qnids = qreq_.ibs.get_annot_nids(qreq_.qaids) dnid2_daids = ut.group_items(qreq_.daids, qreq_.dnids) grouped_aids = dnid2_daids.values() matched_daids = ut.take(dnid2_daids, matched_nids) name_cliques = [ list(itertools.combinations(aids, 2)) for aids in grouped_aids ] aid_matches = [ list(ut.product([qaid], daids)) for qaid, daids in zip(matching_qaids, matched_daids) ] graph = nx.Graph() graph.add_nodes_from(nodes) graph.add_edges_from(ut.flatten(name_cliques)) graph.add_edges_from(ut.flatten(aid_matches)) #matchless_quries = ut.take(qaid_list, ut.index_complement(qxs, len(qaid_list))) name_nodes = [('nid', l) for l in qreq_.dnids] db_aid_nid_edges = list(zip(qreq_.daids, name_nodes)) #query_aid_nid_edges = list(zip(matching_qaids, [('nid', l) for l in matched_nids])) #G = nx.Graph() #G.add_nodes_from(matchless_quries) #G.add_edges_from(db_aid_nid_edges) #G.add_edges_from(query_aid_nid_edges) graph.add_edges_from(db_aid_nid_edges) if infr.user_feedback is not None: user_feedback = ut.map_dict_vals(np.array, infr.user_feedback) p_bg = 0.0 part1 = user_feedback['p_match'] * (1 - user_feedback['p_notcomp']) part2 = p_bg * user_feedback['p_notcomp'] p_same_list = part1 + part2 for aid1, aid2, p_same in zip(user_feedback['aid1'], user_feedback['aid2'], p_same_list): if p_same > .5: if not graph.has_edge(aid1, aid2): graph.add_edge(aid1, aid2) else: if graph.has_edge(aid1, aid2): graph.remove_edge(aid1, aid2) if show: import plottool as pt nx.set_node_attributes(graph, 'color', {aid: pt.LIGHT_PINK for aid in qreq_.daids}) nx.set_node_attributes(graph, 'color', {aid: pt.TRUE_BLUE for aid in qreq_.qaids}) nx.set_node_attributes( graph, 'color', { aid: pt.LIGHT_PURPLE for aid in np.intersect1d(qreq_.qaids, qreq_.daids) }) nx.set_node_attributes( graph, 'label', {node: 'n%r' % (node[1], ) for node in name_nodes}) nx.set_node_attributes( graph, 'color', {node: pt.LIGHT_GREEN for node in name_nodes}) if show: import plottool as pt pt.show_nx(graph, layoutkw={'prog': 'neato'}, verbose=False) return graph
def test_em2(prob_names, prob_annots=None): """ assert prob_names.shape == (nAnnots, nNames) """ learn_rate = 0.05 num_iters = 1 # Matrix if unary probabilites, The probability that each node takes on a # given label, independent of its edges. num_annots, num_names = prob_names.shape # prevent zero probabilities prob_names_ = prob_names + 1E-9 prob_names_ /= prob_names_.sum(axis=1)[:, None] if prob_annots is None: prob_annots_ = np.full((num_annots, num_annots), 1 / num_annots) prob_annots_[np.diag_indices(num_annots)] *= 1.01 # perterb rng = np.random.RandomState(0) prob_annots_ += (rng.randn(*prob_annots_.shape)) / 100 prob_annots_ /= prob_annots_.sum(axis=1)[:, None] prob_annots_ = (prob_annots_.T + prob_annots_) / 2 else: prob_annots_ = prob_annots + 1E-9 prob_annots_ /= prob_annots_.sum(axis=1)[:, None] # Stack everything into a single matrix prob_part = np.hstack([prob_names_, prob_annots_]) zero_part = np.zeros((num_names, num_annots + num_names)) prior = np.vstack([zero_part, prob_part]) # Gamma will hold a probability distribution over the nodes # The labeled nodes must match themselves. # The unlabeld nodes are initialized with a uniform distribution. gam = np.hstack([np.eye(num_names), np.ones((num_names, num_annots)) / num_names]) verbose = 1 if verbose: print('Initialize') print('num_names = %r' % (num_names,)) print(ut.hz_str('prior = ', ut.array2string2(prob_part[:, :], precision=2, max_line_width=140, suppress_small=True))) print(ut.hz_str('gamma = ', ut.array2string2(gam[:, :], max_line_width=140, precision=2, suppress_small=True))) #print(ut.hz_str(' gamma = ', ut.array_repr2(gam, max_line_width=140, precision=2))) delta_i = np.zeros(num_names) def dErr(i, gam, prior, delta_i=delta_i, num_names=num_names): # exepcted liklihood is cross entropy error delta_i[:] = 0 # Compute the gradient of the cross entropy error # This is over both names and annotations jdxs = [j for j in range(prior.shape[0]) if j != i] prior_ij = prior[i, jdxs] np.log(prior_ij / (1 - prior_ij)) gam[:, jdxs] for j in range(prior.shape[0]): if i != j: delta_i += gam[:, j] * np.log(prior[i, j] / (1 - prior[i, j])) # compute the projected gradient delta_i_hat = delta_i - delta_i.sum() / num_names return delta_i_hat # Build node for each annot and each name num_nodes = num_annots + num_names # Maximies the expected liklihood of gamma dGam = np.zeros(gam.shape) # for count in range(num_iters): for count in ut.ProgIter(range(num_iters), label='EM', bs=True): # Compute error gradient for i in range(num_names, num_nodes): dGam[:, i] = dErr(i, gam, prior) # Make a step in the gradient direction # print(ut.hz_str(' dGam = ', ut.array_repr2(dGam, max_line_width=140, precision=2))) gam = gam + learn_rate * dGam # Normalize gam = np.clip(gam, 0, 1) for i in range(num_names, num_nodes): gam[:, i] = gam[:, i] / np.sum(gam[:, i]) # print(ut.hz_str(' gamma = ', ut.array_repr2(gam, max_line_width=140, precision=2))) if verbose: print(ut.hz_str(' gamma = ', ut.array2string2(gam[:, num_names:], max_line_width=140, precision=2, suppress_small=True))) print('Finished') return gam
def make_graph(infr, show=False): import networkx as nx import itertools cm_list = infr.cm_list unique_nids, prob_names = infr.make_prob_names() thresh = infr.choose_thresh() # Simply cut any edge with a weight less than a threshold qaid_list = [cm.qaid for cm in cm_list] postcut = prob_names > thresh qxs, nxs = np.where(postcut) if False: kw = dict(precision=2, max_line_width=140, suppress_small=True) print(ut.hz_str('prob_names = ', ut.array2string2((prob_names), **kw))) print(ut.hz_str('postcut = ', ut.array2string2((postcut).astype(np.int), **kw))) matching_qaids = ut.take(qaid_list, qxs) matched_nids = ut.take(unique_nids, nxs) qreq_ = infr.qreq_ nodes = ut.unique(qreq_.qaids.tolist() + qreq_.daids.tolist()) if not hasattr(qreq_, 'dnids'): qreq_.dnids = qreq_.ibs.get_annot_nids(qreq_.daids) qreq_.qnids = qreq_.ibs.get_annot_nids(qreq_.qaids) dnid2_daids = ut.group_items(qreq_.daids, qreq_.dnids) grouped_aids = dnid2_daids.values() matched_daids = ut.take(dnid2_daids, matched_nids) name_cliques = [list(itertools.combinations(aids, 2)) for aids in grouped_aids] aid_matches = [list(ut.product([qaid], daids)) for qaid, daids in zip(matching_qaids, matched_daids)] graph = nx.Graph() graph.add_nodes_from(nodes) graph.add_edges_from(ut.flatten(name_cliques)) graph.add_edges_from(ut.flatten(aid_matches)) #matchless_quries = ut.take(qaid_list, ut.index_complement(qxs, len(qaid_list))) name_nodes = [('nid', l) for l in qreq_.dnids] db_aid_nid_edges = list(zip(qreq_.daids, name_nodes)) #query_aid_nid_edges = list(zip(matching_qaids, [('nid', l) for l in matched_nids])) #G = nx.Graph() #G.add_nodes_from(matchless_quries) #G.add_edges_from(db_aid_nid_edges) #G.add_edges_from(query_aid_nid_edges) graph.add_edges_from(db_aid_nid_edges) if infr.user_feedback is not None: user_feedback = ut.map_dict_vals(np.array, infr.user_feedback) p_bg = 0.0 part1 = user_feedback['p_match'] * (1 - user_feedback['p_notcomp']) part2 = p_bg * user_feedback['p_notcomp'] p_same_list = part1 + part2 for aid1, aid2, p_same in zip(user_feedback['aid1'], user_feedback['aid2'], p_same_list): if p_same > .5: if not graph.has_edge(aid1, aid2): graph.add_edge(aid1, aid2) else: if graph.has_edge(aid1, aid2): graph.remove_edge(aid1, aid2) if show: import plottool as pt nx.set_node_attributes(graph, 'color', {aid: pt.LIGHT_PINK for aid in qreq_.daids}) nx.set_node_attributes(graph, 'color', {aid: pt.TRUE_BLUE for aid in qreq_.qaids}) nx.set_node_attributes(graph, 'color', { aid: pt.LIGHT_PURPLE for aid in np.intersect1d(qreq_.qaids, qreq_.daids)}) nx.set_node_attributes(graph, 'label', {node: 'n%r' % (node[1],) for node in name_nodes}) nx.set_node_attributes(graph, 'color', {node: pt.LIGHT_GREEN for node in name_nodes}) if show: import plottool as pt pt.show_nx(graph, layoutkw={'prog': 'neato'}, verbose=False) return graph
def setcover_example(): """ CommandLine: python -m ibeis.scripts.specialdraw setcover_example --show Example: >>> # DISABLE_DOCTEST >>> from ibeis.scripts.specialdraw import * # NOQA >>> result = setcover_example() >>> print(result) >>> ut.quit_if_noshow() >>> import plottool as pt >>> ut.show_if_requested() """ import ibeis import plottool as pt from ibeis.viz import viz_graph import networkx as nx pt.ensure_pylab_qt4() ibs = ibeis.opendb(defaultdb='testdb2') if False: # Select a good set aids = ibs.get_name_aids(ibs.get_valid_nids()) # ibeis.testdata_aids('testdb2', a='default:mingt=2') aids = [a for a in aids if len(a) > 1] for a in aids: print(ut.repr3(ibs.get_annot_stats_dict(a))) print(aids[-2]) #aids = [78, 79, 80, 81, 88, 91] aids = [78, 79, 81, 88, 91] qreq_ = ibs.depc.new_request('vsone', aids, aids, cfgdict={}) cm_list = qreq_.execute() from ibeis.algo.hots import graph_iden infr = graph_iden.AnnotInference(cm_list) unique_aids, prob_annots = infr.make_prob_annots() import numpy as np print(ut.hz_str('prob_annots = ', ut.array2string2(prob_annots, precision=2, max_line_width=140, suppress_small=True))) # ut.setcover_greedy(candidate_sets_dict) max_weight = 3 prob_annots[np.diag_indices(len(prob_annots))] = np.inf prob_annots = prob_annots thresh_points = np.sort(prob_annots[np.isfinite(prob_annots)]) # probably not the best way to go about searching for these thresholds # but when you have a hammer... if False: quant = sorted(np.diff(thresh_points))[(len(thresh_points) - 1) // 2 ] candset = {point: thresh_points[np.abs(thresh_points - point) < quant] for point in thresh_points} check_thresholds = len(aids) * 2 thresh_points2 = np.array(ut.setcover_greedy(candset, max_weight=check_thresholds).keys()) thresh_points = thresh_points2 # pt.plot(sorted(thresh_points), 'rx') # pt.plot(sorted(thresh_points2), 'o') # prob_annots = prob_annots.T # thresh_start = np.mean(thresh_points) current_idxs = [] current_covers = [] current_val = np.inf for thresh in thresh_points: covering_sets = [np.where(row >= thresh)[0] for row in (prob_annots)] candidate_sets_dict = {ax: others for ax, others in enumerate(covering_sets)} soln_cover = ut.setcover_ilp(candidate_sets_dict, max_weight=max_weight) exemplar_idxs = list(soln_cover.keys()) soln_weight = len(exemplar_idxs) val = max_weight - soln_weight # print('val = %r' % (val,)) # print('soln_weight = %r' % (soln_weight,)) if val < current_val: current_val = val current_covers = covering_sets current_idxs = exemplar_idxs exemplars = ut.take(aids, current_idxs) ensure_edges = [(aids[ax], aids[ax2]) for ax, other_xs in enumerate(current_covers) for ax2 in other_xs] graph = viz_graph.make_netx_graph_from_aid_groups( ibs, [aids], allow_directed=True, ensure_edges=ensure_edges, temp_nids=[1] * len(aids)) viz_graph.ensure_node_images(ibs, graph) nx.set_node_attributes(graph, 'framewidth', False) nx.set_node_attributes(graph, 'framewidth', {aid: 4.0 for aid in exemplars}) nx.set_edge_attributes(graph, 'color', pt.ORANGE) nx.set_node_attributes(graph, 'color', pt.LIGHT_BLUE) nx.set_node_attributes(graph, 'shape', 'rect') layoutkw = { 'sep' : 1 / 10, 'prog': 'neato', 'overlap': 'false', #'splines': 'ortho', 'splines': 'spline', } pt.show_nx(graph, layout='agraph', layoutkw=layoutkw) pt.zoom_factory()