def make_cm_very_old_tuple(qres_copy): assert ut.listfind(qres_copy.filtkey_list, filtkey) is None weight_filters = hstypes.WEIGHT_FILTERS weight_filtxs, nonweight_filtxs = special_query.index_partition( qres_copy.filtkey_list, weight_filters) aid2_fsv = {} aid2_fs = {} aid2_score = {} for new_fsv_vsone, daid in zip(newfsv_list, newscore_aids): pass break #scorex_vsone = ut.listfind(qres_copy.filtkey_list, filtkey) #if scorex_vsone is None: # TODO: add spatial verification as a filter score # augment the vsone scores # TODO: paramaterize weighted_ave_score = True if weighted_ave_score: # weighted average scoring new_fs_vsone = special_query.weighted_average_scoring( new_fsv_vsone, weight_filtxs, nonweight_filtxs) else: # product scoring new_fs_vsone = special_query.product_scoring(new_fsv_vsone) new_score_vsone = new_fs_vsone.sum() aid2_fsv[daid] = new_fsv_vsone aid2_fs[daid] = new_fs_vsone aid2_score[daid] = new_score_vsone return aid2_fsv, aid2_fs, aid2_score
def make_cm_very_old_tuple(qres_copy): assert ut.listfind(qres_copy.filtkey_list, filtkey) is None weight_filters = hstypes.WEIGHT_FILTERS weight_filtxs, nonweight_filtxs = special_query.index_partition(qres_copy.filtkey_list, weight_filters) aid2_fsv = {} aid2_fs = {} aid2_score = {} for new_fsv_vsone, daid in zip(newfsv_list, newscore_aids): pass break #scorex_vsone = ut.listfind(qres_copy.filtkey_list, filtkey) #if scorex_vsone is None: # TODO: add spatial verification as a filter score # augment the vsone scores # TODO: paramaterize weighted_ave_score = True if weighted_ave_score: # weighted average scoring new_fs_vsone = special_query.weighted_average_scoring(new_fsv_vsone, weight_filtxs, nonweight_filtxs) else: # product scoring new_fs_vsone = special_query.product_scoring(new_fsv_vsone) new_score_vsone = new_fs_vsone.sum() aid2_fsv[daid] = new_fsv_vsone aid2_fs[daid] = new_fs_vsone aid2_score[daid] = new_score_vsone return aid2_fsv, aid2_fs, aid2_score
def myquery(): r""" BUG:: THERE IS A BUG SOMEWHERE: HOW IS THIS POSSIBLE? if everything is weightd ) how di the true positive even get a score while the true negative did not qres_copy.filtkey_list = ['ratio', 'fg', 'homogerr', 'distinctiveness'] CORRECT STATS { 'max' : [0.832, 0.968, 0.604, 0.000], 'min' : [0.376, 0.524, 0.000, 0.000], 'mean' : [0.561, 0.924, 0.217, 0.000], 'std' : [0.114, 0.072, 0.205, 0.000], 'nMin' : [1, 1, 1, 51], 'nMax' : [1, 1, 1, 1], 'shape': (52, 4), } INCORRECT STATS { 'max' : [0.759, 0.963, 0.264, 0.000], 'min' : [0.379, 0.823, 0.000, 0.000], 'mean' : [0.506, 0.915, 0.056, 0.000], 'std' : [0.125, 0.039, 0.078, 0.000], 'nMin' : [1, 1, 1, 24], 'nMax' : [1, 1, 1, 1], 'shape': (26, 4), # score_diff, tp_score, tn_score, p, K, dcvs_clip_max, fg_power, homogerr_power 0.494, 0.494, 0.000, 73.000, 2, 0.500, 0.100, 10.000 see how seperability changes as we very things CommandLine: python -m ibeis.algo.hots.devcases --test-myquery python -m ibeis.algo.hots.devcases --test-myquery --show --index 0 python -m ibeis.algo.hots.devcases --test-myquery --show --index 1 python -m ibeis.algo.hots.devcases --test-myquery --show --index 2 References: http://en.wikipedia.org/wiki/Pareto_distribution <- look into Example: >>> # DISABLE_DOCTEST >>> from ibeis.all_imports import * # NOQA >>> from ibeis.algo.hots.devcases import * # NOQA >>> ut.dev_ipython_copypaster(myquery) if ut.inIPython() else myquery() >>> pt.show_if_requested() """ from ibeis.algo.hots import special_query # NOQA from ibeis.algo.hots import distinctiveness_normalizer # NOQA from ibeis import viz # NOQA import plottool as pt index = ut.get_argval('--index', int, 0) ibs, aid1, aid2, tn_aid = testdata_my_exmaples(index) qaids = [aid1] daids = [aid2] + [tn_aid] qvuuid = ibs.get_annot_visual_uuids(aid1) cfgdict_vsone = dict( sv_on=True, #sv_on=False, #codename='vsone_unnorm_dist_ratio_extern_distinctiveness', codename='vsone_unnorm_ratio_extern_distinctiveness', sver_output_weighting=True, ) use_cache = False save_qcache = False qres_list, qreq_ = ibs.query_chips(qaids, daids, cfgdict=cfgdict_vsone, return_request=True, use_cache=use_cache, save_qcache=save_qcache, verbose=True) qreq_.load_distinctiveness_normalizer() qres = qres_list[0] top_aids = qres.get_top_aids() # NOQA qres_orig = qres # NOQA def test_config(qreq_, qres_orig, cfgdict): """ function to grid search over """ qres_copy = copy.deepcopy(qres_orig) qreq_vsone_ = qreq_ qres_vsone = qres_copy filtkey = hstypes.FiltKeys.DISTINCTIVENESS newfsv_list, newscore_aids = special_query.get_extern_distinctiveness( qreq_, qres_copy, **cfgdict) special_query.apply_new_qres_filter_scores(qreq_vsone_, qres_vsone, newfsv_list, newscore_aids, filtkey) tp_score = qres_copy.aid2_score[aid2] tn_score = qres_copy.aid2_score[tn_aid] return qres_copy, tp_score, tn_score #[.01, .1, .2, .5, .6, .7, .8, .9, 1.0]), #FiltKeys = hstypes.FiltKeys # FIXME: Use other way of doing gridsearch grid_basis = distinctiveness_normalizer.DCVS_DEFAULT.get_grid_basis() gridsearch = ut.GridSearch(grid_basis, label='qvuuid=%r' % (qvuuid, )) print('Begin Grid Search') for cfgdict in ut.ProgressIter(gridsearch, lbl='GridSearch'): qres_copy, tp_score, tn_score = test_config(qreq_, qres_orig, cfgdict) gridsearch.append_result(tp_score, tn_score) print('Finish Grid Search') # Get best result best_cfgdict = gridsearch.get_rank_cfgdict() qres_copy, tp_score, tn_score = test_config(qreq_, qres_orig, best_cfgdict) # Examine closely what you can do with scores if False: qres_copy = copy.deepcopy(qres_orig) qreq_vsone_ = qreq_ filtkey = hstypes.FiltKeys.DISTINCTIVENESS newfsv_list, newscore_aids = special_query.get_extern_distinctiveness( qreq_, qres_copy, **cfgdict) ut.embed() def make_cm_very_old_tuple(qres_copy): assert ut.listfind(qres_copy.filtkey_list, filtkey) is None weight_filters = hstypes.WEIGHT_FILTERS weight_filtxs, nonweight_filtxs = special_query.index_partition( qres_copy.filtkey_list, weight_filters) aid2_fsv = {} aid2_fs = {} aid2_score = {} for new_fsv_vsone, daid in zip(newfsv_list, newscore_aids): pass break #scorex_vsone = ut.listfind(qres_copy.filtkey_list, filtkey) #if scorex_vsone is None: # TODO: add spatial verification as a filter score # augment the vsone scores # TODO: paramaterize weighted_ave_score = True if weighted_ave_score: # weighted average scoring new_fs_vsone = special_query.weighted_average_scoring( new_fsv_vsone, weight_filtxs, nonweight_filtxs) else: # product scoring new_fs_vsone = special_query.product_scoring(new_fsv_vsone) new_score_vsone = new_fs_vsone.sum() aid2_fsv[daid] = new_fsv_vsone aid2_fs[daid] = new_fs_vsone aid2_score[daid] = new_score_vsone return aid2_fsv, aid2_fs, aid2_score # Look at plot of query products for new_fsv_vsone, daid in zip(newfsv_list, newscore_aids): new_fs_vsone = special_query.product_scoring(new_fsv_vsone) scores_list = np.array(new_fs_vsone)[:, None].T pt.plot_sorted_scores(scores_list, logscale=False, figtitle=str(daid)) pt.iup() special_query.apply_new_qres_filter_scores(qreq_vsone_, qres_copy, newfsv_list, newscore_aids, filtkey) # PRINT INFO import functools #ut.rrrr() get_stats_str = functools.partial(ut.get_stats_str, axis=0, newlines=True, precision=3) tp_stats_str = ut.align(get_stats_str(qres_copy.aid2_fsv[aid2]), ':') tn_stats_str = ut.align(get_stats_str(qres_copy.aid2_fsv[tn_aid]), ':') info_str_list = [] info_str_list.append('qres_copy.filtkey_list = %r' % (qres_copy.filtkey_list, )) info_str_list.append('CORRECT STATS') info_str_list.append(tp_stats_str) info_str_list.append('INCORRECT STATS') info_str_list.append(tn_stats_str) info_str = '\n'.join(info_str_list) print(info_str) # SHOW BEST RESULT #qres_copy.ishow_top(ibs, fnum=pt.next_fnum()) #qres_orig.ishow_top(ibs, fnum=pt.next_fnum()) # Text Informatio param_lbl = 'dcvs_power' param_stats_str = gridsearch.get_dimension_stats_str(param_lbl) print(param_stats_str) csvtext = gridsearch.get_csv_results(10) print(csvtext) # Paramter visuzliation fnum = pt.next_fnum() # plot paramter influence param_label_list = gridsearch.get_param_lbls() pnum_ = pt.get_pnum_func(2, len(param_label_list)) for px, param_label in enumerate(param_label_list): gridsearch.plot_dimension(param_label, fnum=fnum, pnum=pnum_(px)) # plot match figure pnum2_ = pt.get_pnum_func(2, 2) qres_copy.show_matches(ibs, aid2, fnum=fnum, pnum=pnum2_(2)) qres_copy.show_matches(ibs, tn_aid, fnum=fnum, pnum=pnum2_(3)) # Add figure labels figtitle = 'Effect of parameters on vsone separation for a single case' subtitle = 'qvuuid = %r' % (qvuuid) figtitle += '\n' + subtitle pt.set_figtitle(figtitle) # Save Figure #fig_fpath = pt.save_figure(usetitle=True) #print(fig_fpath) # Write CSV Results #csv_fpath = fig_fpath + '.csv.txt' #ut.write_to(csv_fpath, csvtext) #qres_copy.ishow_top(ibs) #from matplotlib import pyplot as plt #plt.show() #print(ut.list_str())) # TODO: plot max variation dims #import plottool as pt #pt.plot(p_list, diff_list) """
def myquery(): r""" BUG:: THERE IS A BUG SOMEWHERE: HOW IS THIS POSSIBLE? if everything is weightd ) how di the true positive even get a score while the true negative did not qres_copy.filtkey_list = ['ratio', 'fg', 'homogerr', 'distinctiveness'] CORRECT STATS { 'max' : [0.832, 0.968, 0.604, 0.000], 'min' : [0.376, 0.524, 0.000, 0.000], 'mean' : [0.561, 0.924, 0.217, 0.000], 'std' : [0.114, 0.072, 0.205, 0.000], 'nMin' : [1, 1, 1, 51], 'nMax' : [1, 1, 1, 1], 'shape': (52, 4), } INCORRECT STATS { 'max' : [0.759, 0.963, 0.264, 0.000], 'min' : [0.379, 0.823, 0.000, 0.000], 'mean' : [0.506, 0.915, 0.056, 0.000], 'std' : [0.125, 0.039, 0.078, 0.000], 'nMin' : [1, 1, 1, 24], 'nMax' : [1, 1, 1, 1], 'shape': (26, 4), # score_diff, tp_score, tn_score, p, K, dcvs_clip_max, fg_power, homogerr_power 0.494, 0.494, 0.000, 73.000, 2, 0.500, 0.100, 10.000 see how seperability changes as we very things CommandLine: python -m ibeis.algo.hots.devcases --test-myquery python -m ibeis.algo.hots.devcases --test-myquery --show --index 0 python -m ibeis.algo.hots.devcases --test-myquery --show --index 1 python -m ibeis.algo.hots.devcases --test-myquery --show --index 2 References: http://en.wikipedia.org/wiki/Pareto_distribution <- look into Example: >>> # DISABLE_DOCTEST >>> from ibeis.all_imports import * # NOQA >>> from ibeis.algo.hots.devcases import * # NOQA >>> ut.dev_ipython_copypaster(myquery) if ut.inIPython() else myquery() >>> pt.show_if_requested() """ from ibeis.algo.hots import special_query # NOQA from ibeis.algo.hots import distinctiveness_normalizer # NOQA from ibeis import viz # NOQA import plottool as pt index = ut.get_argval('--index', int, 0) ibs, aid1, aid2, tn_aid = testdata_my_exmaples(index) qaids = [aid1] daids = [aid2] + [tn_aid] qvuuid = ibs.get_annot_visual_uuids(aid1) cfgdict_vsone = dict( sv_on=True, #sv_on=False, #codename='vsone_unnorm_dist_ratio_extern_distinctiveness', codename='vsone_unnorm_ratio_extern_distinctiveness', sver_output_weighting=True, ) use_cache = False save_qcache = False qres_list, qreq_ = ibs.query_chips(qaids, daids, cfgdict=cfgdict_vsone, return_request=True, use_cache=use_cache, save_qcache=save_qcache, verbose=True) qreq_.load_distinctiveness_normalizer() qres = qres_list[0] top_aids = qres.get_top_aids() # NOQA qres_orig = qres # NOQA def test_config(qreq_, qres_orig, cfgdict): """ function to grid search over """ qres_copy = copy.deepcopy(qres_orig) qreq_vsone_ = qreq_ qres_vsone = qres_copy filtkey = hstypes.FiltKeys.DISTINCTIVENESS newfsv_list, newscore_aids = special_query.get_extern_distinctiveness(qreq_, qres_copy, **cfgdict) special_query.apply_new_qres_filter_scores(qreq_vsone_, qres_vsone, newfsv_list, newscore_aids, filtkey) tp_score = qres_copy.aid2_score[aid2] tn_score = qres_copy.aid2_score[tn_aid] return qres_copy, tp_score, tn_score #[.01, .1, .2, .5, .6, .7, .8, .9, 1.0]), #FiltKeys = hstypes.FiltKeys # FIXME: Use other way of doing gridsearch grid_basis = distinctiveness_normalizer.DCVS_DEFAULT.get_grid_basis() gridsearch = ut.GridSearch(grid_basis, label='qvuuid=%r' % (qvuuid,)) print('Begin Grid Search') for cfgdict in ut.ProgressIter(gridsearch, lbl='GridSearch'): qres_copy, tp_score, tn_score = test_config(qreq_, qres_orig, cfgdict) gridsearch.append_result(tp_score, tn_score) print('Finish Grid Search') # Get best result best_cfgdict = gridsearch.get_rank_cfgdict() qres_copy, tp_score, tn_score = test_config(qreq_, qres_orig, best_cfgdict) # Examine closely what you can do with scores if False: qres_copy = copy.deepcopy(qres_orig) qreq_vsone_ = qreq_ filtkey = hstypes.FiltKeys.DISTINCTIVENESS newfsv_list, newscore_aids = special_query.get_extern_distinctiveness(qreq_, qres_copy, **cfgdict) ut.embed() def make_cm_very_old_tuple(qres_copy): assert ut.listfind(qres_copy.filtkey_list, filtkey) is None weight_filters = hstypes.WEIGHT_FILTERS weight_filtxs, nonweight_filtxs = special_query.index_partition(qres_copy.filtkey_list, weight_filters) aid2_fsv = {} aid2_fs = {} aid2_score = {} for new_fsv_vsone, daid in zip(newfsv_list, newscore_aids): pass break #scorex_vsone = ut.listfind(qres_copy.filtkey_list, filtkey) #if scorex_vsone is None: # TODO: add spatial verification as a filter score # augment the vsone scores # TODO: paramaterize weighted_ave_score = True if weighted_ave_score: # weighted average scoring new_fs_vsone = special_query.weighted_average_scoring(new_fsv_vsone, weight_filtxs, nonweight_filtxs) else: # product scoring new_fs_vsone = special_query.product_scoring(new_fsv_vsone) new_score_vsone = new_fs_vsone.sum() aid2_fsv[daid] = new_fsv_vsone aid2_fs[daid] = new_fs_vsone aid2_score[daid] = new_score_vsone return aid2_fsv, aid2_fs, aid2_score # Look at plot of query products for new_fsv_vsone, daid in zip(newfsv_list, newscore_aids): new_fs_vsone = special_query.product_scoring(new_fsv_vsone) scores_list = np.array(new_fs_vsone)[:, None].T pt.plot_sorted_scores(scores_list, logscale=False, figtitle=str(daid)) pt.iup() special_query.apply_new_qres_filter_scores(qreq_vsone_, qres_copy, newfsv_list, newscore_aids, filtkey) # PRINT INFO import functools #ut.rrrr() get_stats_str = functools.partial(ut.get_stats_str, axis=0, newlines=True, precision=3) tp_stats_str = ut.align(get_stats_str(qres_copy.aid2_fsv[aid2]), ':') tn_stats_str = ut.align(get_stats_str(qres_copy.aid2_fsv[tn_aid]), ':') info_str_list = [] info_str_list.append('qres_copy.filtkey_list = %r' % (qres_copy.filtkey_list,)) info_str_list.append('CORRECT STATS') info_str_list.append(tp_stats_str) info_str_list.append('INCORRECT STATS') info_str_list.append(tn_stats_str) info_str = '\n'.join(info_str_list) print(info_str) # SHOW BEST RESULT #qres_copy.ishow_top(ibs, fnum=pt.next_fnum()) #qres_orig.ishow_top(ibs, fnum=pt.next_fnum()) # Text Informatio param_lbl = 'dcvs_power' param_stats_str = gridsearch.get_dimension_stats_str(param_lbl) print(param_stats_str) csvtext = gridsearch.get_csv_results(10) print(csvtext) # Paramter visuzliation fnum = pt.next_fnum() # plot paramter influence param_label_list = gridsearch.get_param_lbls() pnum_ = pt.get_pnum_func(2, len(param_label_list)) for px, param_label in enumerate(param_label_list): gridsearch.plot_dimension(param_label, fnum=fnum, pnum=pnum_(px)) # plot match figure pnum2_ = pt.get_pnum_func(2, 2) qres_copy.show_matches(ibs, aid2, fnum=fnum, pnum=pnum2_(2)) qres_copy.show_matches(ibs, tn_aid, fnum=fnum, pnum=pnum2_(3)) # Add figure labels figtitle = 'Effect of parameters on vsone separation for a single case' subtitle = 'qvuuid = %r' % (qvuuid) figtitle += '\n' + subtitle pt.set_figtitle(figtitle) # Save Figure #fig_fpath = pt.save_figure(usetitle=True) #print(fig_fpath) # Write CSV Results #csv_fpath = fig_fpath + '.csv.txt' #ut.write_to(csv_fpath, csvtext) #qres_copy.ishow_top(ibs) #from matplotlib import pyplot as plt #plt.show() #print(ut.list_str())) # TODO: plot max variation dims #import plottool as pt #pt.plot(p_list, diff_list) """