def add_ibeis_support(mxer, qreq_, new_aid_list): """ Chooses indexer with smallest number of annotations and reindexes it. Args: qreq_ (QueryRequest): query request object with hyper-parameters new_aid_list (list): CommandLine: python -m ibeis.algo.hots.multi_index --test-add_ibeis_support Example: >>> # DISABLE_DOCTEST >>> from ibeis.algo.hots.multi_index import * # NOQA >>> mxer, qreq_, ibs = testdata_mindexer() >>> new_aid_list = ibs.get_valid_aids()[70:80] >>> # execute function >>> result = mxer.add_ibeis_support(qreq_, new_aid_list) >>> # verify results >>> print(result) """ print('adding multi-indexer support') # Assert that the aids are indeed new mxer.assert_can_add_aids(new_aid_list) # Find the indexer to add to num_indexed_list = mxer.get_multi_num_indexed_annots() min_argx = num_indexed_list.argmin() nnindexer_old = mxer.nn_indexer_list[min_argx] # Combine old and new aids prev_aids = nnindexer_old.get_indexed_aids() new_aid_list_ = np.append(prev_aids, new_aid_list) # Reindexed combined aids nnindexer_new = neighbor_index_cache.request_memcached_ibeis_nnindexer( qreq_, new_aid_list_) # Replace the old nnindexer with the new nnindexer mxer.nn_indexer_list[min_argx] = nnindexer_new mxer.min_reindex_thresh = qreq_.qparams.min_reindex_thresh if neighbor_index_cache.can_request_background_nnindexer(): # Check if background process needs to be spawned # FIXME: this does not belong in method code num_indexed_list_new = mxer.get_multi_num_indexed_annots() new_smalled_size = min(num_indexed_list_new) need_reindex = (new_smalled_size > mxer.min_reindex_thresh or len(num_indexed_list_new) > mxer.max_subindexers) if need_reindex: if USE_FORGROUND_REINDEX: raise NotImplementedError( 'no foreground reindex in stateful query') else: # Reindex the multi-indexed trees in the background aid_list = mxer.get_indexed_aids() neighbor_index_cache.request_background_nnindexer( qreq_, aid_list)
def add_ibeis_support(mxer, qreq_, new_aid_list): """ Chooses indexer with smallest number of annotations and reindexes it. Args: qreq_ (QueryRequest): query request object with hyper-parameters new_aid_list (list): CommandLine: python -m ibeis.algo.hots.multi_index --test-add_ibeis_support Example: >>> # DISABLE_DOCTEST >>> from ibeis.algo.hots.multi_index import * # NOQA >>> mxer, qreq_, ibs = testdata_mindexer() >>> new_aid_list = ibs.get_valid_aids()[70:80] >>> # execute function >>> result = mxer.add_ibeis_support(qreq_, new_aid_list) >>> # verify results >>> print(result) """ print('adding multi-indexer support') # Assert that the aids are indeed new mxer.assert_can_add_aids(new_aid_list) # Find the indexer to add to num_indexed_list = mxer.get_multi_num_indexed_annots() min_argx = num_indexed_list.argmin() nnindexer_old = mxer.nn_indexer_list[min_argx] # Combine old and new aids prev_aids = nnindexer_old.get_indexed_aids() new_aid_list_ = np.append(prev_aids, new_aid_list) # Reindexed combined aids nnindexer_new = neighbor_index_cache.request_memcached_ibeis_nnindexer(qreq_, new_aid_list_) # Replace the old nnindexer with the new nnindexer mxer.nn_indexer_list[min_argx] = nnindexer_new mxer.min_reindex_thresh = qreq_.qparams.min_reindex_thresh if neighbor_index_cache.can_request_background_nnindexer(): # Check if background process needs to be spawned # FIXME: this does not belong in method code num_indexed_list_new = mxer.get_multi_num_indexed_annots() new_smalled_size = min(num_indexed_list_new) need_reindex = (new_smalled_size > mxer.min_reindex_thresh or len(num_indexed_list_new) > mxer.max_subindexers) if need_reindex: if USE_FORGROUND_REINDEX: raise NotImplementedError('no foreground reindex in stateful query') else: # Reindex the multi-indexed trees in the background aid_list = mxer.get_indexed_aids() neighbor_index_cache.request_background_nnindexer(qreq_, aid_list)
def request_ibeis_mindexer(qreq_, index_method='multi', verbose=True): """ CommandLine: python -m ibeis.algo.hots.multi_index --test-request_ibeis_mindexer:2 Example0: >>> # SLOW_DOCTEST >>> from ibeis.algo.hots.multi_index import * # NOQA >>> import ibeis >>> ibs = ibeis.opendb(db='PZ_MTEST') >>> valid_aids = ibs.get_valid_aids() >>> daid_list = valid_aids[1:60] >>> cfgdict = dict(fg_on=False) >>> qreq_ = ibs.new_query_request(daid_list, daid_list, cfgdict=cfgdict) >>> index_method = 'multi' >>> mxer = request_ibeis_mindexer(qreq_, index_method) Example1: >>> # DISABLE_DOCTEST >>> from ibeis.algo.hots.multi_index import * # NOQA >>> import ibeis >>> ibs = ibeis.opendb(db='PZ_Master0') >>> valid_aids = ibs.get_valid_aids() >>> daid_list = valid_aids[1:60] >>> cfgdict = dict(fg_on=False) >>> qreq_ = ibs.new_query_request(daid_list, daid_list, cfgdict=cfgdict) >>> index_method = 'multi' >>> mxer = request_ibeis_mindexer(qreq_, index_method) Example2: >>> # DISABLE_DOCTEST >>> # Test background reindex >>> from ibeis.algo.hots.multi_index import * # NOQA >>> import ibeis >>> import time >>> ibs = ibeis.opendb(db='PZ_MTEST') >>> valid_aids = ibs.get_valid_aids() >>> # Remove all cached nnindexers >>> ibs.delete_flann_cachedir() >>> # This request should build a new nnindexer >>> daid_list = valid_aids[1:30] >>> cfgdict = dict(fg_on=False) >>> qreq_ = ibs.new_query_request(daid_list, daid_list, cfgdict=cfgdict) >>> index_method = 'multi' >>> mxer = request_ibeis_mindexer(qreq_, index_method) >>> ut.assert_eq(len(mxer.nn_indexer_list), 1, 'one subindexer') >>> # The next request should trigger a background process >>> # and build two subindexer >>> daid_list = valid_aids[1:60] >>> qreq_ = ibs.new_query_request(daid_list, daid_list, cfgdict=cfgdict) >>> index_method = 'multi' >>> mxer = request_ibeis_mindexer(qreq_, index_method) >>> # Do some work in the foreground to ensure that it doesnt block >>> # the background job >>> print('[FG] sleeping or doing bit compute') >>> # Takes about 15 seconds >>> with ut.Timer(): ... ut.enumerate_primes(int(9E4)) >>> #time.sleep(10) >>> print('[FG] done sleeping') >>> ut.assert_eq(len(mxer.nn_indexer_list), 2, 'two subindexer') >>> # And this shoud build just one subindexer >>> daid_list = valid_aids[1:60] >>> qreq_ = ibs.new_query_request(daid_list, daid_list, cfgdict=cfgdict) >>> index_method = 'multi' >>> mxer = request_ibeis_mindexer(qreq_, index_method) >>> ut.assert_eq(len(mxer.nn_indexer_list), 1, 'one big subindexer') """ min_reindex_thresh = qreq_.qparams.min_reindex_thresh max_subindexers = qreq_.qparams.max_subindexers daid_list = qreq_.get_internal_daids() print('[mindex] make MultiNeighborIndex over %d annots' % (len(daid_list),)) print('[mindex] index_method=%r' % index_method) # Split annotations into groups accorindg to index_method ibs = qreq_.ibs if index_method == 'name': # each group are annotations of the same name num_indexers = 8 aids_list, overflow_aids, num_bins = group_daids_for_indexing_by_name(ibs, daid_list, num_indexers, verbose) elif index_method == 'multi': neighbor_index_cache.check_background_process() # Use greedy set cover to get a list of nnindxers that are already built tup = neighbor_index_cache.group_daids_by_cached_nnindexer( qreq_, daid_list, min_reindex_thresh) uncovered_aids, covered_aids_list = tup # If the number of bins gets too big do a reindex # in the background num_subindexers = len(covered_aids_list) + (len(uncovered_aids) > 1) if num_subindexers > max_subindexers: print('need to reindex something') if USE_FORGROUND_REINDEX: aids_list = [sorted(ut.flatten(covered_aids_list))] #ut.embed() else: neighbor_index_cache.request_background_nnindexer(qreq_, daid_list) aids_list = covered_aids_list else: aids_list = covered_aids_list if len(uncovered_aids) > 0: aids_list.append(uncovered_aids) num_bins = len(aids_list) else: raise AssertionError('unknown index_method=%r' % (index_method,)) # Build a neighbor indexer for each nn_indexer_list = [] #extra_indexes = [] for tx, aids in enumerate(aids_list): print('[mindex] building forest %d/%d with %d aids' % (tx + 1, num_bins, len(aids))) if len(aids) > 0: # Dont bother shallow copying qreq_ here. # just passing aids is enough nnindexer = neighbor_index_cache.request_memcached_ibeis_nnindexer(qreq_, aids) nn_indexer_list.append(nnindexer) #if len(unknown_aids) > 0: # print('[mindex] building unknown forest') # unknown_vecs_list = ibs.get_annot_vecs(overflow_aids, config2_=qreq_.get_internal_data_config2()) # unknown_index = NeighborIndex(overflow_aids, unknown_vecs_list) # extra_indexes.append(unknown_index) ##print('[mindex] building normalizer forest') # TODO #mxer.nn_indexer_list = nn_indexer_list #mxer.extra_indexes = extra_indexes #mxer.overflow_index = overflow_index #mxer.unknown_index = unknown_index mxer = MultiNeighborIndex(nn_indexer_list, min_reindex_thresh, max_subindexers) return mxer