def trytest_incremental_add(ibs): r""" Args: ibs (IBEISController): CommandLine: python -m wbia.algo.hots._neighbor_experiment --test-test_incremental_add Example: >>> # DISABLE_DOCTEST >>> from wbia.algo.hots.neighbor_index_cache import * # NOQA >>> import wbia >>> ibs = wbia.opendb('PZ_MTEST') >>> result = test_incremental_add(ibs) >>> print(result) """ import wbia sample_aids = wbia.testdata_aids(a='default:pername=1,mingt=2') aids1 = sample_aids[::2] aids2 = sample_aids[0:5] aids3 = sample_aids[:-1] # NOQA daid_list = aids1 # NOQA qreq_ = ibs.new_query_request(aids1, aids1) nnindexer1 = neighbor_index_cache.request_wbia_nnindexer( # NOQA ibs.new_query_request(aids1, aids1)) nnindexer2 = neighbor_index_cache.request_wbia_nnindexer( # NOQA ibs.new_query_request(aids2, aids2)) # TODO: SYSTEM use visual uuids items = ibs.get_annot_visual_uuids(aids3) uuid_map_fpath = neighbor_index_cache.get_nnindexer_uuid_map_fpath(qreq_) candidate_uuids = neighbor_index_cache.read_uuid_map(uuid_map_fpath, 0) candidate_sets = candidate_uuids covertup = ut.greedy_max_inden_setcover(candidate_sets, items) uncovered_items, covered_items_list, accepted_keys = covertup covered_items = ut.flatten(covered_items_list) covered_aids = sorted(ibs.get_annot_aids_from_visual_uuid(covered_items)) uncovered_aids = sorted( ibs.get_annot_aids_from_visual_uuid(uncovered_items)) nnindexer3 = neighbor_index_cache.request_wbia_nnindexer( # NOQA ibs.new_query_request(uncovered_aids, uncovered_aids)) # TODO: SYSTEM use visual uuids items = ibs.get_annot_visual_uuids(sample_aids) uuid_map_fpath = neighbor_index_cache.get_nnindexer_uuid_map_fpath(qreq_) # contextlib.closing(shelve.open(uuid_map_fpath)) as uuid_map: candidate_uuids = neighbor_index_cache.read_uuid_map(uuid_map_fpath, 0) candidate_sets = candidate_uuids covertup = ut.greedy_max_inden_setcover(candidate_sets, items) uncovered_items, covered_items_list, accepted_keys = covertup covered_items = ut.flatten(covered_items_list) covered_aids = sorted( ibs.get_annot_aids_from_visual_uuid(covered_items)) # NOQA uncovered_aids = sorted( ibs.get_annot_aids_from_visual_uuid(uncovered_items))
def test_incremental_add(ibs): r""" Args: ibs (IBEISController): CommandLine: python -m ibeis.algo.hots._neighbor_experiment --test-test_incremental_add Example: >>> # DISABLE_DOCTEST >>> from ibeis.algo.hots.neighbor_index_cache import * # NOQA >>> import ibeis >>> ibs = ibeis.opendb('PZ_MTEST') >>> result = test_incremental_add(ibs) >>> print(result) """ import ibeis sample_aids = ibeis.testdata_aids(a='default:pername=1,mingt=2') aids1 = sample_aids[::2] aids2 = sample_aids[0:5] aids3 = sample_aids[:-1] # NOQA daid_list = aids1 # NOQA qreq_ = ibs.new_query_request(aids1, aids1) nnindexer1 = neighbor_index_cache.request_ibeis_nnindexer(ibs.new_query_request(aids1, aids1)) # NOQA nnindexer2 = neighbor_index_cache.request_ibeis_nnindexer(ibs.new_query_request(aids2, aids2)) # NOQA # TODO: SYSTEM use visual uuids #daids_hashid = qreq_.ibs.get_annot_hashid_visual_uuid(daid_list) # get_internal_data_hashid() items = ibs.get_annot_visual_uuids(aids3) uuid_map_fpath = neighbor_index_cache.get_nnindexer_uuid_map_fpath(qreq_) candidate_uuids = neighbor_index_cache.read_uuid_map(uuid_map_fpath, 0) candidate_sets = candidate_uuids covertup = ut.greedy_max_inden_setcover(candidate_sets, items) uncovered_items, covered_items_list, accepted_keys = covertup covered_items = ut.flatten(covered_items_list) covered_aids = sorted(ibs.get_annot_aids_from_visual_uuid(covered_items)) uncovered_aids = sorted(ibs.get_annot_aids_from_visual_uuid(uncovered_items)) nnindexer3 = neighbor_index_cache.request_ibeis_nnindexer(ibs.new_query_request(uncovered_aids, uncovered_aids)) # NOQA # TODO: SYSTEM use visual uuids #daids_hashid = qreq_.ibs.get_annot_hashid_visual_uuid(daid_list) # get_internal_data_hashid() items = ibs.get_annot_visual_uuids(sample_aids) uuid_map_fpath = neighbor_index_cache.get_nnindexer_uuid_map_fpath(qreq_) #contextlib.closing(shelve.open(uuid_map_fpath)) as uuid_map: candidate_uuids = neighbor_index_cache.read_uuid_map(uuid_map_fpath, 0) candidate_sets = candidate_uuids covertup = ut.greedy_max_inden_setcover(candidate_sets, items) uncovered_items, covered_items_list, accepted_keys = covertup covered_items = ut.flatten(covered_items_list) covered_aids = sorted(ibs.get_annot_aids_from_visual_uuid(covered_items)) # NOQA uncovered_aids = sorted(ibs.get_annot_aids_from_visual_uuid(uncovered_items))
def group_daids_by_cached_nnindexer( qreq_, daid_list, min_reindex_thresh, max_covers=None ): r""" CommandLine: python -m wbia.algo.hots.neighbor_index_cache --test-group_daids_by_cached_nnindexer Example: >>> # ENABLE_DOCTEST >>> from wbia.algo.hots.neighbor_index_cache import * # NOQA >>> import wbia >>> ibs = wbia.opendb('testdb1') >>> ZEB_PLAIN = wbia.const.TEST_SPECIES.ZEB_PLAIN >>> daid_list = ibs.get_valid_aids(species=ZEB_PLAIN) >>> qreq_ = ibs.new_query_request(daid_list, daid_list) >>> # Set the params a bit lower >>> max_covers = None >>> qreq_.qparams.min_reindex_thresh = 1 >>> min_reindex_thresh = qreq_.qparams.min_reindex_thresh >>> # STEP 0: CLEAR THE CACHE >>> clear_uuid_cache(qreq_) >>> # STEP 1: ASSERT EMPTY INDEX >>> daid_list = ibs.get_valid_aids(species=ZEB_PLAIN)[0:3] >>> uncovered_aids, covered_aids_list = group_daids_by_cached_nnindexer( ... qreq_, daid_list, min_reindex_thresh, max_covers) >>> result1 = uncovered_aids, covered_aids_list >>> ut.assert_eq(result1, ([], [[1, 2, 3]]), 'pre request') >>> # TEST 2: SHOULD MAKE 123 COVERED >>> nnindexer = request_memcached_wbia_nnindexer(qreq_, daid_list) >>> uncovered_aids, covered_aids_list = group_daids_by_cached_nnindexer( ... qreq_, daid_list, min_reindex_thresh, max_covers) >>> result2 = uncovered_aids, covered_aids_list >>> ut.assert_eq(result2, ([], [[1, 2, 3]]), 'post request') """ ibs = qreq_.ibs # read which annotations have prebuilt caches uuid_map_fpath = get_nnindexer_uuid_map_fpath(qreq_) candidate_uuids = UUID_MAP_CACHE.read_uuid_map_dict( uuid_map_fpath, min_reindex_thresh ) # find a maximum independent set cover of the requested annotations annot_vuuid_list = ibs.get_annot_visual_uuids(daid_list) # 3.2 % covertup = ut.greedy_max_inden_setcover( candidate_uuids, annot_vuuid_list, max_covers ) # 0.2 % uncovered_vuuids, covered_vuuids_list, accepted_keys = covertup # return the grouped covered items (so they can be loaded) and # the remaining uuids which need to have an index computed. # uncovered_aids_ = ibs.get_annot_aids_from_visual_uuid(uncovered_vuuids) # 28.0% covered_aids_list_ = ibs.unflat_map( ibs.get_annot_aids_from_visual_uuid, covered_vuuids_list ) # 68% # FIXME: uncovered_aids = sorted(uncovered_aids_) # covered_aids_list = list(map(sorted, covered_aids_list_)) covered_aids_list = covered_aids_list_ return uncovered_aids, covered_aids_list
def group_daids_by_cached_nnindexer(qreq_, daid_list, min_reindex_thresh, max_covers=None): r""" CommandLine: python -m ibeis.algo.hots.neighbor_index_cache --test-group_daids_by_cached_nnindexer Example: >>> # ENABLE_DOCTEST >>> from ibeis.algo.hots.neighbor_index_cache import * # NOQA >>> import ibeis >>> ibs = ibeis.opendb('testdb1') >>> ZEB_PLAIN = ibeis.const.TEST_SPECIES.ZEB_PLAIN >>> daid_list = ibs.get_valid_aids(species=ZEB_PLAIN) >>> qreq_ = ibs.new_query_request(daid_list, daid_list) >>> # Set the params a bit lower >>> max_covers = None >>> qreq_.qparams.min_reindex_thresh = 1 >>> min_reindex_thresh = qreq_.qparams.min_reindex_thresh >>> # STEP 0: CLEAR THE CACHE >>> clear_uuid_cache(qreq_) >>> # STEP 1: ASSERT EMPTY INDEX >>> daid_list = ibs.get_valid_aids(species=ZEB_PLAIN)[0:3] >>> uncovered_aids, covered_aids_list = group_daids_by_cached_nnindexer( ... qreq_, daid_list, min_reindex_thresh, max_covers) >>> result1 = uncovered_aids, covered_aids_list >>> ut.assert_eq(result1, ([1, 2, 3], []), 'pre request') >>> # TEST 2: SHOULD MAKE 123 COVERED >>> nnindexer = request_memcached_ibeis_nnindexer(qreq_, daid_list) >>> uncovered_aids, covered_aids_list = group_daids_by_cached_nnindexer( ... qreq_, daid_list, min_reindex_thresh, max_covers) >>> result2 = uncovered_aids, covered_aids_list >>> ut.assert_eq(result2, ([], [[1, 2, 3]]), 'post request') """ ibs = qreq_.ibs # read which annotations have prebuilt caches uuid_map_fpath = get_nnindexer_uuid_map_fpath(qreq_) candidate_uuids = UUID_MAP_CACHE.read_uuid_map_dict(uuid_map_fpath, min_reindex_thresh) # find a maximum independent set cover of the requested annotations annot_vuuid_list = ibs.get_annot_visual_uuids(daid_list) # 3.2 % covertup = ut.greedy_max_inden_setcover( candidate_uuids, annot_vuuid_list, max_covers) # 0.2 % uncovered_vuuids, covered_vuuids_list, accepted_keys = covertup # return the grouped covered items (so they can be loaded) and # the remaining uuids which need to have an index computed. # uncovered_aids_ = ibs.get_annot_aids_from_visual_uuid(uncovered_vuuids) # 28.0% covered_aids_list_ = ibs.unflat_map( ibs.get_annot_aids_from_visual_uuid, covered_vuuids_list) # 68% # FIXME: uncovered_aids = sorted(uncovered_aids_) #covered_aids_list = list(map(sorted, covered_aids_list_)) covered_aids_list = covered_aids_list_ return uncovered_aids, covered_aids_list