Пример #1
0
def load_gztest(ibs):
    r"""
    CommandLine:
        python -m ibeis.algo.hots.special_query --test-load_gztest

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis.algo.hots.devcases import *  # NOQA
        >>> import ibeis
        >>> ibs = ibeis.opendb('GZ_ALL')
    """
    from os.path import join
    from ibeis.algo.hots import match_chips4 as mc4
    dir_ = ut.get_module_dir(mc4)
    eval_text = ut.read_from(join(dir_, 'GZ_TESTTUP.txt'))
    testcases = eval(eval_text)
    count_dict = ut.count_dict_vals(testcases)
    print(ut.dict_str(count_dict))

    testtup_list = ut.flatten(
        ut.dict_take_list(testcases, [
            'vsone_wins', 'vsmany_outperformed', 'vsmany_dominates',
            'vsmany_wins'
        ]))
    qaid_list = [testtup.qaid_t for testtup in testtup_list]
    visual_uuids = ibs.get_annot_visual_uuids(qaid_list)
    visual_uuids
Пример #2
0
def load_gztest(ibs):
    r"""
    CommandLine:
        python -m ibeis.algo.hots.special_query --test-load_gztest

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis.algo.hots.devcases import *  # NOQA
        >>> import ibeis
        >>> ibs = ibeis.opendb('GZ_ALL')
    """
    from os.path import join
    from ibeis.algo.hots import match_chips4 as mc4
    dir_ = ut.get_module_dir(mc4)
    eval_text = ut.read_from(join(dir_,  'GZ_TESTTUP.txt'))
    testcases = eval(eval_text)
    count_dict = ut.count_dict_vals(testcases)
    print(ut.dict_str(count_dict))

    testtup_list = ut.flatten(ut.dict_take_list(testcases, ['vsone_wins',
                                                            'vsmany_outperformed',
                                                            'vsmany_dominates',
                                                            'vsmany_wins']))
    qaid_list = [testtup.qaid_t for testtup in testtup_list]
    visual_uuids = ibs.get_annot_visual_uuids(qaid_list)
    visual_uuids
Пример #3
0
def check_results(ibs_gt, ibs2, aid1_to_aid2, aids_list1_, incinfo):
    """
    reports how well the incremental query ran when the oracle was calling the
    shots.
    """
    print('--------- CHECKING RESULTS ------------')
    testcases = incinfo.get('testcases')
    if testcases is not None:
        count_dict = ut.count_dict_vals(testcases)
        print('+--')
        #print(ut.dict_str(testcases))
        print('---')
        print(ut.dict_str(count_dict))
        print('L__')
    # TODO: dont include initially added aids in the result reporting
    aid_list1 = aids_list1_  # ibs_gt.get_valid_aids()
    #aid_list1 = ibs_gt.get_aids_with_groundtruth()
    aid_list2 = ibs2.get_valid_aids()

    nid_list1 = ibs_gt.get_annot_nids(aid_list1)
    nid_list2 = ibs2.get_annot_nids(aid_list2)

    # Group annotations from test and gt database by their respective names
    grouped_dict1 = ut.group_items(aid_list1, nid_list1)
    grouped_dict2 = ut.group_items(aid_list2, nid_list2)
    grouped_aids1 = list(six.itervalues(grouped_dict1))
    grouped_aids2 = list(map(tuple, six.itervalues(grouped_dict2)))
    #group_nids1 = list(six.iterkeys(grouped_dict1))
    #group_nids2 = list(six.iterkeys(grouped_dict2))

    # Transform annotation ids from database1 space to database2 space
    grouped_aids1_t = [
        tuple(ut.dict_take_list(aid1_to_aid2, aids1))
        for aids1 in grouped_aids1
    ]

    set_grouped_aids1_t = set(grouped_aids1_t)
    set_grouped_aids2 = set(grouped_aids2)

    # Find names we got right. (correct groupings of annotations)
    # these are the annotation groups that are intersecting between
    # the test database and groundtruth database
    perfect_groups = set_grouped_aids2.intersection(set_grouped_aids1_t)
    # Find names we got wrong. (incorrect groupings of annotations)
    # The test database sets that were not perfect
    nonperfect_groups = set_grouped_aids2.difference(perfect_groups)
    # What we should have got
    # The ground truth database sets that were not fully identified
    missed_groups = set_grouped_aids1_t.difference(perfect_groups)

    # Mark non perfect groups by their error type
    false_negative_groups = []  # failed to link enough
    false_positive_groups = []  # linked too much
    for nonperfect_group in nonperfect_groups:
        if ut.is_subset_of_any(nonperfect_group, missed_groups):
            false_negative_groups.append(nonperfect_group)
        else:
            false_positive_groups.append(nonperfect_group)

    # Get some more info on the nonperfect groups
    # find which groups should have been linked
    aid2_to_aid1 = ut.invert_dict(aid1_to_aid2)
    false_negative_groups_t = [
        tuple(ut.dict_take_list(aid2_to_aid1, aids2))
        for aids2 in false_negative_groups
    ]
    false_negative_group_nids_t = ibs_gt.unflat_map(ibs_gt.get_annot_nids,
                                                    false_negative_groups_t)
    assert all(map(ut.allsame,
                   false_negative_group_nids_t)), 'inconsistent nids'
    false_negative_group_nid_t = ut.get_list_column(
        false_negative_group_nids_t, 0)
    # These are the links that should have been made
    missed_links = ut.group_items(false_negative_groups,
                                  false_negative_group_nid_t)

    print(ut.dict_str(missed_links))

    print('# Name with failed links (FN) = %r' % len(false_negative_groups))
    print('... should have reduced to %d names.' % (len(missed_links)))
    print('# Name with wrong links (FP)  = %r' % len(false_positive_groups))
    print('# Name correct names (TP)     = %r' % len(perfect_groups))
Пример #4
0
        elif vsmany_rank is not None and vsone_rank is None:
            if vsmany_rank < 5:
                append_case(VSMANY_DOMINATES, testtup)
            else:
                append_case(VSMANY_OUTPERFORMED, testtup)
        elif vsmany_rank is None:
            append_case(BOTH_FAIL, testtup)
        elif vsone_rank > vsmany_rank:
            append_case(VSMANY_WINS, testtup)
        elif vsone_rank < vsmany_rank:
            append_case(VSONE_WINS, testtup)
        elif vsone_rank == vsmany_rank:
            append_case(WASH, testtup)
        else:
            raise AssertionError('unenumerated case')
        count_dict = ut.count_dict_vals(testcases)
        print('+--')
        #print(ut.dict_str(testcases))
        print('---')
        print(ut.dict_str(count_dict))
        print('L__')
        #ut.embed()


@profile
def query_vsmany_initial(ibs,
                         qaids,
                         daids,
                         use_cache=False,
                         qreq_vsmany_=None,
                         save_qcache=False):
Пример #5
0
def check_results(ibs_gt, ibs2, aid1_to_aid2, aids_list1_, incinfo):
    """
    reports how well the incremental query ran when the oracle was calling the
    shots.
    """
    print('--------- CHECKING RESULTS ------------')
    testcases = incinfo.get('testcases')
    if testcases is not None:
        count_dict = ut.count_dict_vals(testcases)
        print('+--')
        #print(ut.dict_str(testcases))
        print('---')
        print(ut.dict_str(count_dict))
        print('L__')
    # TODO: dont include initially added aids in the result reporting
    aid_list1 = aids_list1_  # ibs_gt.get_valid_aids()
    #aid_list1 = ibs_gt.get_aids_with_groundtruth()
    aid_list2 = ibs2.get_valid_aids()

    nid_list1 = ibs_gt.get_annot_nids(aid_list1)
    nid_list2 = ibs2.get_annot_nids(aid_list2)

    # Group annotations from test and gt database by their respective names
    grouped_dict1 = ut.group_items(aid_list1, nid_list1)
    grouped_dict2 = ut.group_items(aid_list2, nid_list2)
    grouped_aids1 = list(six.itervalues(grouped_dict1))
    grouped_aids2 = list(map(tuple, six.itervalues(grouped_dict2)))
    #group_nids1 = list(six.iterkeys(grouped_dict1))
    #group_nids2 = list(six.iterkeys(grouped_dict2))

    # Transform annotation ids from database1 space to database2 space
    grouped_aids1_t = [tuple(ut.dict_take_list(aid1_to_aid2, aids1)) for aids1 in grouped_aids1]

    set_grouped_aids1_t = set(grouped_aids1_t)
    set_grouped_aids2   = set(grouped_aids2)

    # Find names we got right. (correct groupings of annotations)
    # these are the annotation groups that are intersecting between
    # the test database and groundtruth database
    perfect_groups = set_grouped_aids2.intersection(set_grouped_aids1_t)
    # Find names we got wrong. (incorrect groupings of annotations)
    # The test database sets that were not perfect
    nonperfect_groups = set_grouped_aids2.difference(perfect_groups)
    # What we should have got
    # The ground truth database sets that were not fully identified
    missed_groups = set_grouped_aids1_t.difference(perfect_groups)

    # Mark non perfect groups by their error type
    false_negative_groups = []  # failed to link enough
    false_positive_groups = []  # linked too much
    for nonperfect_group in nonperfect_groups:
        if ut.is_subset_of_any(nonperfect_group, missed_groups):
            false_negative_groups.append(nonperfect_group)
        else:
            false_positive_groups.append(nonperfect_group)

    # Get some more info on the nonperfect groups
    # find which groups should have been linked
    aid2_to_aid1 = ut.invert_dict(aid1_to_aid2)
    false_negative_groups_t = [tuple(ut.dict_take_list(aid2_to_aid1, aids2)) for aids2 in false_negative_groups]
    false_negative_group_nids_t = ibs_gt.unflat_map(ibs_gt.get_annot_nids, false_negative_groups_t)
    assert all(map(ut.allsame, false_negative_group_nids_t)), 'inconsistent nids'
    false_negative_group_nid_t = ut.get_list_column(false_negative_group_nids_t, 0)
    # These are the links that should have been made
    missed_links = ut.group_items(false_negative_groups, false_negative_group_nid_t)

    print(ut.dict_str(missed_links))

    print('# Name with failed links (FN) = %r' % len(false_negative_groups))
    print('... should have reduced to %d names.' % (len(missed_links)))
    print('# Name with wrong links (FP)  = %r' % len(false_positive_groups))
    print('# Name correct names (TP)     = %r' % len(perfect_groups))
Пример #6
0
def test_vsone_errors(ibs, daids, qaid2_qres_vsmany, qaid2_qres_vsone, incinfo):
    """
    ibs1 = ibs_gt
    ibs2 = ibs (the current test database, sorry for the backwardness)
    aid1_to_aid2 - maps annots from ibs1 to ibs2
    """
    WASH                = 'wash'
    BOTH_FAIL           = 'both_fail'
    SINGLETON           = 'singleton'
    VSMANY_OUTPERFORMED = 'vsmany_outperformed'
    VSMANY_DOMINATES    = 'vsmany_dominates'
    VSMANY_WINS         = 'vsmany_wins'
    VSONE_WINS          = 'vsone_wins'
    if 'testcases' not in incinfo:
        testcases = {}
        for case in [WASH, BOTH_FAIL, SINGLETON, VSMANY_OUTPERFORMED,
                     VSMANY_DOMINATES, VSMANY_WINS, VSONE_WINS]:
            testcases[case] = []
        incinfo['testcases'] = testcases
    testcases = incinfo['testcases']

    def append_case(case, testtup):
        print('APPENDED NEW TESTCASE: case=%r' % (case,))
        print('* testup = %r' % (testtup,))
        print('* vuuid = %r' % (ibs_gt.get_annot_visual_uuids(testtup.qaid_t),))
        if ut.get_argflag('--interupt-case') and case in [VSMANY_WINS, VSMANY_DOMINATES]:
            incinfo['interactive'] = True
            incinfo['use_oracle'] = False
            incinfo['STOP'] = True
            if ut.is_developer():
                import plottool as pt  # NOQA
                IPYTHON_COMMANDS = """
                >>> %pylab qt4
                >>> from ibeis.viz.interact import interact_matches  # NOQA
                >>> #qres_vsmany = ut.search_stack_for_localvar('qres_vsmany')
                >>> ibs        = ut.search_stack_for_localvar('ibs')
                >>> daids      = ut.search_stack_for_localvar('daids')
                >>> qnid_t     = ut.search_stack_for_localvar('qnid_t')
                >>> qres_vsone = ut.search_stack_for_localvar('qres_vsone')
                >>> all_nids_t = ut.search_stack_for_localvar('all_nids_t')
                >>> # Find index in daids of correct matches
                >>> cm = qres_vsone
                >>> correct_indices = np.where(np.array(all_nids_t) == qnid_t)[0]
                >>> correct_aids2 = ut.take(daids, correct_indices)
                >>> qaid = cm.qaid
                >>> aid = correct_aids2[0]
                >>> # Report visual uuid for inclusion or exclusion in script
                >>> print(ibs.get_annot_visual_uuids([qaid, aid]))

                >>> # Feature match things
                >>> print('cm.filtkey_list = %r' % (cm.filtkey_list,))
                >>> fm  = cm.aid2_fm[aid]
                >>> fs  = cm.aid2_fs[aid]
                >>> fsv = cm.aid2_fsv[aid]
                >>> mx = 2
                >>> qfx, dfx = fm[mx]
                >>> fsv_single = fsv[mx]
                >>> fs_single = fs[mx]
                >>> # check featweights
                >>> data_featweights = ibs.get_annot_fgweights([aid])[0]
                >>> data_featweights[dfx]
                >>> fnum = pt.next_fnum()
                >>> bad_aid = cm.get_top_aids()[0]
                >>> #match_interaction_good = interact_matches.MatchInteraction(ibs, cm, aid, annot_mode=1)
                >>> #match_interaction_bad = interact_matches.MatchInteraction(ibs, cm, bad_aid)
                >>> match_interaction_good = cm.ishow_matches(ibs, aid, annot_mode=1, fnum=1)
                >>> match_interaction_bad = cm.ishow_matches(ibs, bad_aid, annot_mode=1, fnum=2)
                >>> match_interaction = match_interaction_good
                >>> self = match_interaction
                >>> self.select_ith_match(mx)
                >>> #impossible_to_match = len(correct_indices) > 0
                """
                y = """
                >>> from os.path import exists
                >>> import vtool as vt
                >>> import vtool.patch as vtpatch
                >>> import vtool.image as vtimage  # NOQA
                >>> chip_list = ibs.get_annot_chips([aid])
                >>> kpts_list = ibs.get_annot_kpts([aid])
                >>> probchip_fpath_list = ibs.get_probchip_fpath(aid)
                >>> probchip_list = [vt.imread(fpath, grayscale=True) if exists(fpath) else None for fpath in probchip_fpath_list]
                >>> kpts  = kpts_list[0]
                >>> probchip = probchip_list[0]
                >>> kp = kpts[dfx]
                >>> patch  = vt.get_warped_patch(probchip, kp)[0].astype(np.float32) / 255.0
                >>> fnum2 = pt.next_fnum()
                >>> pt.figure(fnum2, pnum=(1, 2, 1), doclf=True, docla=True)
                >>> pt.imshow(probchip)
                >>> pt.draw_kpts2([kp])
                >>> pt.figure(fnum2, pnum=(1, 2, 2))
                >>> pt.imshow(patch * 255)
                >>> pt.update()
                >>> vt.gaussian_average_patch(patch)
                >>> cm.ishow_top(ibs, annot_mode=1)
                """
                y
                ut.set_clipboard(IPYTHON_COMMANDS)
                #ut.spawn_delayed_ipython_paste()
                ut.embed(remove_pyqt_hook=False)
                IPYTHON_COMMANDS

        testcases[case].append(testtup)

    for qaid in six.iterkeys(qaid2_qres_vsmany):
        qres_vsmany = qaid2_qres_vsmany[qaid]
        qres_vsone  = qaid2_qres_vsone[qaid]
        nscoretup_vsone  = qres_vsone.get_nscoretup()
        nscoretup_vsmany = qres_vsmany.get_nscoretup()
        metatup = incinfo['metatup']
        ibs_gt, aid1_to_aid2 = metatup
        aid2_to_aid1 = ut.invert_dict(aid1_to_aid2)

        top_aids_vsone  = ut.get_list_column(nscoretup_vsone.sorted_aids, 0)
        top_aids_vsmany = ut.get_list_column(nscoretup_vsmany.sorted_aids, 0)
        # tranform to groundtruth database coordinates
        all_daids_t = ut.dict_take_list(aid2_to_aid1, daids)
        top_aids_vsone_t  = ut.dict_take_list(aid2_to_aid1, top_aids_vsone)
        top_aids_vsmany_t = ut.dict_take_list(aid2_to_aid1, top_aids_vsmany)
        qaid_t = aid2_to_aid1[qaid]

        aids_tup = (all_daids_t, top_aids_vsone_t, top_aids_vsmany_t, (qaid_t,),)
        nids_tup = ibs_gt.unflat_map(ibs_gt.get_annot_nids, aids_tup)
        (all_nids_t, top_nids_vsone_t, top_nids_vsmany_t, (qnid_t,),) = nids_tup

        vsmany_rank  = ut.listfind(top_nids_vsmany_t, qnid_t)
        vsone_rank   = ut.listfind(top_nids_vsone_t, qnid_t)
        impossible_to_match = ut.listfind(all_nids_t, qnid_t) is None

        # Sort the test case into a category
        testtup = TestTup(qaid_t, qaid, vsmany_rank, vsone_rank)
        if vsmany_rank is None and vsone_rank is None and impossible_to_match:
            append_case(SINGLETON, testtup)
        elif vsmany_rank is not None and vsone_rank is None:
            if vsmany_rank < 5:
                append_case(VSMANY_DOMINATES, testtup)
            else:
                append_case(VSMANY_OUTPERFORMED, testtup)
        elif vsmany_rank is None:
            append_case(BOTH_FAIL, testtup)
        elif vsone_rank > vsmany_rank:
            append_case(VSMANY_WINS, testtup)
        elif vsone_rank < vsmany_rank:
            append_case(VSONE_WINS, testtup)
        elif vsone_rank == vsmany_rank:
            append_case(WASH, testtup)
        else:
            raise AssertionError('unenumerated case')
        count_dict = ut.count_dict_vals(testcases)
        print('+--')
        #print(ut.dict_str(testcases))
        print('---')
        print(ut.dict_str(count_dict))
        print('L__')