Beispiel #1
0
def setcover_example():
    """
    CommandLine:
        python -m ibeis.scripts.specialdraw setcover_example --show

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis.scripts.specialdraw import *  # NOQA
        >>> result = setcover_example()
        >>> print(result)
        >>> ut.quit_if_noshow()
        >>> import plottool as pt
        >>> ut.show_if_requested()
    """
    import ibeis
    import plottool as pt
    from ibeis.viz import viz_graph
    import networkx as nx
    pt.ensure_pylab_qt4()
    ibs = ibeis.opendb(defaultdb='testdb2')

    if False:
        # Select a good set
        aids = ibs.get_name_aids(ibs.get_valid_nids())
        # ibeis.testdata_aids('testdb2', a='default:mingt=2')
        aids = [a for a in aids if len(a) > 1]
        for a in aids:
            print(ut.repr3(ibs.get_annot_stats_dict(a)))
        print(aids[-2])
    #aids = [78, 79, 80, 81, 88, 91]
    aids = [78, 79, 81, 88, 91]
    qreq_ = ibs.depc.new_request('vsone', aids, aids, cfgdict={})
    cm_list = qreq_.execute()
    from ibeis.algo.hots import graph_iden
    infr = graph_iden.AnnotInference(cm_list)
    unique_aids, prob_annots = infr.make_prob_annots()
    import numpy as np
    print(
        ut.hz_str(
            'prob_annots = ',
            ut.array2string2(prob_annots,
                             precision=2,
                             max_line_width=140,
                             suppress_small=True)))
    # ut.setcover_greedy(candidate_sets_dict)
    max_weight = 3
    prob_annots[np.diag_indices(len(prob_annots))] = np.inf
    prob_annots = prob_annots
    thresh_points = np.sort(prob_annots[np.isfinite(prob_annots)])

    # probably not the best way to go about searching for these thresholds
    # but when you have a hammer...
    if False:
        quant = sorted(np.diff(thresh_points))[(len(thresh_points) - 1) // 2]
        candset = {
            point: thresh_points[np.abs(thresh_points - point) < quant]
            for point in thresh_points
        }
        check_thresholds = len(aids) * 2
        thresh_points2 = np.array(
            ut.setcover_greedy(candset, max_weight=check_thresholds).keys())
        thresh_points = thresh_points2

    # pt.plot(sorted(thresh_points), 'rx')
    # pt.plot(sorted(thresh_points2), 'o')

    # prob_annots = prob_annots.T

    # thresh_start = np.mean(thresh_points)
    current_idxs = []
    current_covers = []
    current_val = np.inf
    for thresh in thresh_points:
        covering_sets = [np.where(row >= thresh)[0] for row in (prob_annots)]
        candidate_sets_dict = {
            ax: others
            for ax, others in enumerate(covering_sets)
        }
        soln_cover = ut.setcover_ilp(candidate_sets_dict,
                                     max_weight=max_weight)
        exemplar_idxs = list(soln_cover.keys())
        soln_weight = len(exemplar_idxs)
        val = max_weight - soln_weight
        # print('val = %r' % (val,))
        # print('soln_weight = %r' % (soln_weight,))
        if val < current_val:
            current_val = val
            current_covers = covering_sets
            current_idxs = exemplar_idxs
    exemplars = ut.take(aids, current_idxs)
    ensure_edges = [(aids[ax], aids[ax2])
                    for ax, other_xs in enumerate(current_covers)
                    for ax2 in other_xs]
    graph = viz_graph.make_netx_graph_from_aid_groups(
        ibs, [aids],
        allow_directed=True,
        ensure_edges=ensure_edges,
        temp_nids=[1] * len(aids))
    viz_graph.ensure_node_images(ibs, graph)

    nx.set_node_attributes(graph, 'framewidth', False)
    nx.set_node_attributes(graph, 'framewidth',
                           {aid: 4.0
                            for aid in exemplars})
    nx.set_edge_attributes(graph, 'color', pt.ORANGE)
    nx.set_node_attributes(graph, 'color', pt.LIGHT_BLUE)
    nx.set_node_attributes(graph, 'shape', 'rect')

    layoutkw = {
        'sep': 1 / 10,
        'prog': 'neato',
        'overlap': 'false',
        #'splines': 'ortho',
        'splines': 'spline',
    }
    pt.show_nx(graph, layout='agraph', layoutkw=layoutkw)
    pt.zoom_factory()
Beispiel #2
0
    def make_graph(infr, show=False):
        import networkx as nx
        import itertools
        cm_list = infr.cm_list
        unique_nids, prob_names = infr.make_prob_names()
        thresh = infr.choose_thresh()

        # Simply cut any edge with a weight less than a threshold
        qaid_list = [cm.qaid for cm in cm_list]
        postcut = prob_names > thresh
        qxs, nxs = np.where(postcut)
        if False:
            kw = dict(precision=2, max_line_width=140, suppress_small=True)
            print(
                ut.hz_str('prob_names = ', ut.array2string2((prob_names),
                                                            **kw)))
            print(
                ut.hz_str('postcut = ',
                          ut.array2string2((postcut).astype(np.int), **kw)))
        matching_qaids = ut.take(qaid_list, qxs)
        matched_nids = ut.take(unique_nids, nxs)

        qreq_ = infr.qreq_

        nodes = ut.unique(qreq_.qaids.tolist() + qreq_.daids.tolist())
        if not hasattr(qreq_, 'dnids'):
            qreq_.dnids = qreq_.ibs.get_annot_nids(qreq_.daids)
            qreq_.qnids = qreq_.ibs.get_annot_nids(qreq_.qaids)
        dnid2_daids = ut.group_items(qreq_.daids, qreq_.dnids)
        grouped_aids = dnid2_daids.values()
        matched_daids = ut.take(dnid2_daids, matched_nids)
        name_cliques = [
            list(itertools.combinations(aids, 2)) for aids in grouped_aids
        ]
        aid_matches = [
            list(ut.product([qaid], daids))
            for qaid, daids in zip(matching_qaids, matched_daids)
        ]

        graph = nx.Graph()
        graph.add_nodes_from(nodes)
        graph.add_edges_from(ut.flatten(name_cliques))
        graph.add_edges_from(ut.flatten(aid_matches))

        #matchless_quries = ut.take(qaid_list, ut.index_complement(qxs, len(qaid_list)))
        name_nodes = [('nid', l) for l in qreq_.dnids]
        db_aid_nid_edges = list(zip(qreq_.daids, name_nodes))
        #query_aid_nid_edges = list(zip(matching_qaids, [('nid', l) for l in matched_nids]))
        #G = nx.Graph()
        #G.add_nodes_from(matchless_quries)
        #G.add_edges_from(db_aid_nid_edges)
        #G.add_edges_from(query_aid_nid_edges)

        graph.add_edges_from(db_aid_nid_edges)

        if infr.user_feedback is not None:
            user_feedback = ut.map_dict_vals(np.array, infr.user_feedback)
            p_bg = 0.0
            part1 = user_feedback['p_match'] * (1 - user_feedback['p_notcomp'])
            part2 = p_bg * user_feedback['p_notcomp']
            p_same_list = part1 + part2
            for aid1, aid2, p_same in zip(user_feedback['aid1'],
                                          user_feedback['aid2'], p_same_list):
                if p_same > .5:
                    if not graph.has_edge(aid1, aid2):
                        graph.add_edge(aid1, aid2)
                else:
                    if graph.has_edge(aid1, aid2):
                        graph.remove_edge(aid1, aid2)
        if show:
            import plottool as pt
            nx.set_node_attributes(graph, 'color',
                                   {aid: pt.LIGHT_PINK
                                    for aid in qreq_.daids})
            nx.set_node_attributes(graph, 'color',
                                   {aid: pt.TRUE_BLUE
                                    for aid in qreq_.qaids})
            nx.set_node_attributes(
                graph, 'color', {
                    aid: pt.LIGHT_PURPLE
                    for aid in np.intersect1d(qreq_.qaids, qreq_.daids)
                })
            nx.set_node_attributes(
                graph, 'label',
                {node: 'n%r' % (node[1], )
                 for node in name_nodes})
            nx.set_node_attributes(
                graph, 'color', {node: pt.LIGHT_GREEN
                                 for node in name_nodes})
        if show:
            import plottool as pt
            pt.show_nx(graph, layoutkw={'prog': 'neato'}, verbose=False)
        return graph
Beispiel #3
0
def test_em2(prob_names, prob_annots=None):
    """
    assert prob_names.shape == (nAnnots, nNames)
    """
    learn_rate = 0.05
    num_iters = 1

    # Matrix if unary probabilites, The probability that each node takes on a
    # given label, independent of its edges.
    num_annots, num_names = prob_names.shape

    # prevent zero probabilities
    prob_names_ = prob_names + 1E-9
    prob_names_ /= prob_names_.sum(axis=1)[:, None]

    if prob_annots is None:
        prob_annots_ = np.full((num_annots, num_annots), 1 / num_annots)
        prob_annots_[np.diag_indices(num_annots)] *= 1.01
        # perterb
        rng = np.random.RandomState(0)
        prob_annots_ += (rng.randn(*prob_annots_.shape)) / 100
        prob_annots_ /= prob_annots_.sum(axis=1)[:, None]
        prob_annots_ = (prob_annots_.T + prob_annots_) / 2
    else:
        prob_annots_ = prob_annots + 1E-9
        prob_annots_ /= prob_annots_.sum(axis=1)[:, None]

    # Stack everything into a single matrix
    prob_part = np.hstack([prob_names_, prob_annots_])
    zero_part = np.zeros((num_names, num_annots + num_names))
    prior = np.vstack([zero_part, prob_part])

    # Gamma will hold a probability distribution over the nodes
    # The labeled nodes must match themselves.
    # The unlabeld nodes are initialized with a uniform distribution.
    gam = np.hstack([np.eye(num_names), np.ones((num_names, num_annots)) / num_names])

    verbose = 1
    if verbose:
        print('Initialize')
        print('num_names = %r' % (num_names,))
        print(ut.hz_str('prior = ', ut.array2string2(prob_part[:, :], precision=2, max_line_width=140, suppress_small=True)))
        print(ut.hz_str('gamma = ', ut.array2string2(gam[:, :], max_line_width=140, precision=2, suppress_small=True)))
    #print(ut.hz_str(' gamma = ', ut.array_repr2(gam, max_line_width=140, precision=2)))

    delta_i = np.zeros(num_names)
    def dErr(i, gam, prior, delta_i=delta_i, num_names=num_names):
        # exepcted liklihood is cross entropy error
        delta_i[:] = 0
        # Compute the gradient of the cross entropy error
        # This is over both names and annotations
        jdxs = [j for j in range(prior.shape[0]) if j != i]
        prior_ij = prior[i, jdxs]
        np.log(prior_ij / (1 - prior_ij))
        gam[:, jdxs]

        for j in range(prior.shape[0]):
            if i != j:
                delta_i += gam[:, j] * np.log(prior[i, j] / (1 - prior[i, j]))
        # compute the projected gradient
        delta_i_hat = delta_i - delta_i.sum() / num_names
        return delta_i_hat

    # Build node for each annot and each name
    num_nodes = num_annots + num_names

    # Maximies the expected liklihood of gamma
    dGam = np.zeros(gam.shape)
    # for count in range(num_iters):
    for count in ut.ProgIter(range(num_iters), label='EM', bs=True):
        # Compute error gradient
        for i in range(num_names, num_nodes):
            dGam[:, i] = dErr(i, gam, prior)
        # Make a step in the gradient direction
        # print(ut.hz_str(' dGam = ', ut.array_repr2(dGam, max_line_width=140, precision=2)))
        gam = gam + learn_rate * dGam
        # Normalize
        gam = np.clip(gam, 0, 1)
        for i in range(num_names, num_nodes):
            gam[:, i] = gam[:, i] / np.sum(gam[:, i])
    # print(ut.hz_str(' gamma = ', ut.array_repr2(gam, max_line_width=140, precision=2)))
    if verbose:
        print(ut.hz_str(' gamma = ', ut.array2string2(gam[:, num_names:], max_line_width=140, precision=2, suppress_small=True)))
        print('Finished')
    return gam
Beispiel #4
0
    def make_graph(infr, show=False):
        import networkx as nx
        import itertools
        cm_list = infr.cm_list
        unique_nids, prob_names = infr.make_prob_names()
        thresh = infr.choose_thresh()

        # Simply cut any edge with a weight less than a threshold
        qaid_list = [cm.qaid for cm in cm_list]
        postcut = prob_names > thresh
        qxs, nxs = np.where(postcut)
        if False:
            kw = dict(precision=2, max_line_width=140, suppress_small=True)
            print(ut.hz_str('prob_names = ', ut.array2string2((prob_names), **kw)))
            print(ut.hz_str('postcut = ', ut.array2string2((postcut).astype(np.int), **kw)))
        matching_qaids = ut.take(qaid_list, qxs)
        matched_nids = ut.take(unique_nids, nxs)

        qreq_ = infr.qreq_

        nodes = ut.unique(qreq_.qaids.tolist() + qreq_.daids.tolist())
        if not hasattr(qreq_, 'dnids'):
            qreq_.dnids = qreq_.ibs.get_annot_nids(qreq_.daids)
            qreq_.qnids = qreq_.ibs.get_annot_nids(qreq_.qaids)
        dnid2_daids = ut.group_items(qreq_.daids, qreq_.dnids)
        grouped_aids = dnid2_daids.values()
        matched_daids = ut.take(dnid2_daids, matched_nids)
        name_cliques = [list(itertools.combinations(aids, 2)) for aids in grouped_aids]
        aid_matches = [list(ut.product([qaid], daids)) for qaid, daids in
                       zip(matching_qaids, matched_daids)]

        graph = nx.Graph()
        graph.add_nodes_from(nodes)
        graph.add_edges_from(ut.flatten(name_cliques))
        graph.add_edges_from(ut.flatten(aid_matches))

        #matchless_quries = ut.take(qaid_list, ut.index_complement(qxs, len(qaid_list)))
        name_nodes = [('nid', l) for l in qreq_.dnids]
        db_aid_nid_edges = list(zip(qreq_.daids, name_nodes))
        #query_aid_nid_edges = list(zip(matching_qaids, [('nid', l) for l in matched_nids]))
        #G = nx.Graph()
        #G.add_nodes_from(matchless_quries)
        #G.add_edges_from(db_aid_nid_edges)
        #G.add_edges_from(query_aid_nid_edges)

        graph.add_edges_from(db_aid_nid_edges)

        if infr.user_feedback is not None:
            user_feedback = ut.map_dict_vals(np.array, infr.user_feedback)
            p_bg = 0.0
            part1 = user_feedback['p_match'] * (1 - user_feedback['p_notcomp'])
            part2 = p_bg * user_feedback['p_notcomp']
            p_same_list = part1 + part2
            for aid1, aid2, p_same in zip(user_feedback['aid1'],
                                          user_feedback['aid2'], p_same_list):
                if p_same > .5:
                    if not graph.has_edge(aid1, aid2):
                        graph.add_edge(aid1, aid2)
                else:
                    if graph.has_edge(aid1, aid2):
                        graph.remove_edge(aid1, aid2)
        if show:
            import plottool as pt
            nx.set_node_attributes(graph, 'color', {aid: pt.LIGHT_PINK
                                                    for aid in qreq_.daids})
            nx.set_node_attributes(graph, 'color', {aid: pt.TRUE_BLUE
                                                    for aid in qreq_.qaids})
            nx.set_node_attributes(graph, 'color', {
                aid: pt.LIGHT_PURPLE
                for aid in np.intersect1d(qreq_.qaids, qreq_.daids)})
            nx.set_node_attributes(graph, 'label', {node: 'n%r' % (node[1],)
                                                    for node in name_nodes})
            nx.set_node_attributes(graph, 'color', {node: pt.LIGHT_GREEN
                                                    for node in name_nodes})
        if show:
            import plottool as pt
            pt.show_nx(graph, layoutkw={'prog': 'neato'}, verbose=False)
        return graph
Beispiel #5
0
def setcover_example():
    """
    CommandLine:
        python -m ibeis.scripts.specialdraw setcover_example --show

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis.scripts.specialdraw import *  # NOQA
        >>> result = setcover_example()
        >>> print(result)
        >>> ut.quit_if_noshow()
        >>> import plottool as pt
        >>> ut.show_if_requested()
    """
    import ibeis
    import plottool as pt
    from ibeis.viz import viz_graph
    import networkx as nx
    pt.ensure_pylab_qt4()
    ibs = ibeis.opendb(defaultdb='testdb2')

    if False:
        # Select a good set
        aids = ibs.get_name_aids(ibs.get_valid_nids())
        # ibeis.testdata_aids('testdb2', a='default:mingt=2')
        aids = [a for a in aids if len(a) > 1]
        for a in aids:
            print(ut.repr3(ibs.get_annot_stats_dict(a)))
        print(aids[-2])
    #aids = [78, 79, 80, 81, 88, 91]
    aids = [78, 79, 81, 88, 91]
    qreq_ = ibs.depc.new_request('vsone', aids, aids, cfgdict={})
    cm_list = qreq_.execute()
    from ibeis.algo.hots import graph_iden
    infr = graph_iden.AnnotInference(cm_list)
    unique_aids, prob_annots = infr.make_prob_annots()
    import numpy as np
    print(ut.hz_str('prob_annots = ', ut.array2string2(prob_annots, precision=2, max_line_width=140, suppress_small=True)))
    # ut.setcover_greedy(candidate_sets_dict)
    max_weight = 3
    prob_annots[np.diag_indices(len(prob_annots))] = np.inf
    prob_annots = prob_annots
    thresh_points = np.sort(prob_annots[np.isfinite(prob_annots)])

    # probably not the best way to go about searching for these thresholds
    # but when you have a hammer...
    if False:
        quant = sorted(np.diff(thresh_points))[(len(thresh_points) - 1) // 2 ]
        candset = {point: thresh_points[np.abs(thresh_points - point) < quant] for point in thresh_points}
        check_thresholds = len(aids) * 2
        thresh_points2 = np.array(ut.setcover_greedy(candset, max_weight=check_thresholds).keys())
        thresh_points = thresh_points2

    # pt.plot(sorted(thresh_points), 'rx')
    # pt.plot(sorted(thresh_points2), 'o')

    # prob_annots = prob_annots.T

    # thresh_start = np.mean(thresh_points)
    current_idxs = []
    current_covers = []
    current_val = np.inf
    for thresh in thresh_points:
        covering_sets = [np.where(row >= thresh)[0] for row in (prob_annots)]
        candidate_sets_dict = {ax: others for ax, others in enumerate(covering_sets)}
        soln_cover = ut.setcover_ilp(candidate_sets_dict, max_weight=max_weight)
        exemplar_idxs = list(soln_cover.keys())
        soln_weight = len(exemplar_idxs)
        val = max_weight - soln_weight
        # print('val = %r' % (val,))
        # print('soln_weight = %r' % (soln_weight,))
        if val < current_val:
            current_val = val
            current_covers = covering_sets
            current_idxs = exemplar_idxs
    exemplars = ut.take(aids, current_idxs)
    ensure_edges = [(aids[ax], aids[ax2]) for ax, other_xs in enumerate(current_covers) for ax2 in other_xs]
    graph = viz_graph.make_netx_graph_from_aid_groups(
        ibs, [aids], allow_directed=True, ensure_edges=ensure_edges,
        temp_nids=[1] * len(aids))
    viz_graph.ensure_node_images(ibs, graph)

    nx.set_node_attributes(graph, 'framewidth', False)
    nx.set_node_attributes(graph, 'framewidth', {aid: 4.0 for aid in exemplars})
    nx.set_edge_attributes(graph, 'color', pt.ORANGE)
    nx.set_node_attributes(graph, 'color', pt.LIGHT_BLUE)
    nx.set_node_attributes(graph, 'shape', 'rect')

    layoutkw = {
        'sep' : 1 / 10,
        'prog': 'neato',
        'overlap': 'false',
        #'splines': 'ortho',
        'splines': 'spline',
    }
    pt.show_nx(graph, layout='agraph', layoutkw=layoutkw)
    pt.zoom_factory()