def add_ibeis_support(mxer, qreq_, new_aid_list):
"""
Chooses indexer with smallest number of annotations and reindexes it.
Args:
qreq_ (QueryRequest): query request object with hyper-parameters
new_aid_list (list):
CommandLine:
python -m ibeis.algo.hots.multi_index --test-add_ibeis_support
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.algo.hots.multi_index import * # NOQA
>>> mxer, qreq_, ibs = testdata_mindexer()
>>> new_aid_list = ibs.get_valid_aids()[70:80]
>>> # execute function
>>> result = mxer.add_ibeis_support(qreq_, new_aid_list)
>>> # verify results
>>> print(result)
"""
print('adding multi-indexer support')
# Assert that the aids are indeed new
mxer.assert_can_add_aids(new_aid_list)
# Find the indexer to add to
num_indexed_list = mxer.get_multi_num_indexed_annots()
min_argx = num_indexed_list.argmin()
nnindexer_old = mxer.nn_indexer_list[min_argx]
# Combine old and new aids
prev_aids = nnindexer_old.get_indexed_aids()
new_aid_list_ = np.append(prev_aids, new_aid_list)
# Reindexed combined aids
nnindexer_new = neighbor_index_cache.request_memcached_ibeis_nnindexer(
qreq_, new_aid_list_)
# Replace the old nnindexer with the new nnindexer
mxer.nn_indexer_list[min_argx] = nnindexer_new
mxer.min_reindex_thresh = qreq_.qparams.min_reindex_thresh
if neighbor_index_cache.can_request_background_nnindexer():
# Check if background process needs to be spawned
# FIXME: this does not belong in method code
num_indexed_list_new = mxer.get_multi_num_indexed_annots()
new_smalled_size = min(num_indexed_list_new)
need_reindex = (new_smalled_size > mxer.min_reindex_thresh or
len(num_indexed_list_new) > mxer.max_subindexers)
if need_reindex:
if USE_FORGROUND_REINDEX:
raise NotImplementedError(
'no foreground reindex in stateful query')
else:
# Reindex the multi-indexed trees in the background
aid_list = mxer.get_indexed_aids()
neighbor_index_cache.request_background_nnindexer(
qreq_, aid_list)
def add_ibeis_support(mxer, qreq_, new_aid_list):
"""
Chooses indexer with smallest number of annotations and reindexes it.
Args:
qreq_ (QueryRequest): query request object with hyper-parameters
new_aid_list (list):
CommandLine:
python -m ibeis.algo.hots.multi_index --test-add_ibeis_support
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.algo.hots.multi_index import * # NOQA
>>> mxer, qreq_, ibs = testdata_mindexer()
>>> new_aid_list = ibs.get_valid_aids()[70:80]
>>> # execute function
>>> result = mxer.add_ibeis_support(qreq_, new_aid_list)
>>> # verify results
>>> print(result)
"""
print('adding multi-indexer support')
# Assert that the aids are indeed new
mxer.assert_can_add_aids(new_aid_list)
# Find the indexer to add to
num_indexed_list = mxer.get_multi_num_indexed_annots()
min_argx = num_indexed_list.argmin()
nnindexer_old = mxer.nn_indexer_list[min_argx]
# Combine old and new aids
prev_aids = nnindexer_old.get_indexed_aids()
new_aid_list_ = np.append(prev_aids, new_aid_list)
# Reindexed combined aids
nnindexer_new = neighbor_index_cache.request_memcached_ibeis_nnindexer(qreq_, new_aid_list_)
# Replace the old nnindexer with the new nnindexer
mxer.nn_indexer_list[min_argx] = nnindexer_new
mxer.min_reindex_thresh = qreq_.qparams.min_reindex_thresh
if neighbor_index_cache.can_request_background_nnindexer():
# Check if background process needs to be spawned
# FIXME: this does not belong in method code
num_indexed_list_new = mxer.get_multi_num_indexed_annots()
new_smalled_size = min(num_indexed_list_new)
need_reindex = (new_smalled_size > mxer.min_reindex_thresh or
len(num_indexed_list_new) > mxer.max_subindexers)
if need_reindex:
if USE_FORGROUND_REINDEX:
raise NotImplementedError('no foreground reindex in stateful query')
else:
# Reindex the multi-indexed trees in the background
aid_list = mxer.get_indexed_aids()
neighbor_index_cache.request_background_nnindexer(qreq_, aid_list)
def request_ibeis_mindexer(qreq_, index_method='multi', verbose=True):
"""
CommandLine:
python -m ibeis.algo.hots.multi_index --test-request_ibeis_mindexer:2
Example0:
>>> # SLOW_DOCTEST
>>> from ibeis.algo.hots.multi_index import * # NOQA
>>> import ibeis
>>> ibs = ibeis.opendb(db='PZ_MTEST')
>>> valid_aids = ibs.get_valid_aids()
>>> daid_list = valid_aids[1:60]
>>> cfgdict = dict(fg_on=False)
>>> qreq_ = ibs.new_query_request(daid_list, daid_list, cfgdict=cfgdict)
>>> index_method = 'multi'
>>> mxer = request_ibeis_mindexer(qreq_, index_method)
Example1:
>>> # DISABLE_DOCTEST
>>> from ibeis.algo.hots.multi_index import * # NOQA
>>> import ibeis
>>> ibs = ibeis.opendb(db='PZ_Master0')
>>> valid_aids = ibs.get_valid_aids()
>>> daid_list = valid_aids[1:60]
>>> cfgdict = dict(fg_on=False)
>>> qreq_ = ibs.new_query_request(daid_list, daid_list, cfgdict=cfgdict)
>>> index_method = 'multi'
>>> mxer = request_ibeis_mindexer(qreq_, index_method)
Example2:
>>> # DISABLE_DOCTEST
>>> # Test background reindex
>>> from ibeis.algo.hots.multi_index import * # NOQA
>>> import ibeis
>>> import time
>>> ibs = ibeis.opendb(db='PZ_MTEST')
>>> valid_aids = ibs.get_valid_aids()
>>> # Remove all cached nnindexers
>>> ibs.delete_flann_cachedir()
>>> # This request should build a new nnindexer
>>> daid_list = valid_aids[1:30]
>>> cfgdict = dict(fg_on=False)
>>> qreq_ = ibs.new_query_request(daid_list, daid_list, cfgdict=cfgdict)
>>> index_method = 'multi'
>>> mxer = request_ibeis_mindexer(qreq_, index_method)
>>> ut.assert_eq(len(mxer.nn_indexer_list), 1, 'one subindexer')
>>> # The next request should trigger a background process
>>> # and build two subindexer
>>> daid_list = valid_aids[1:60]
>>> qreq_ = ibs.new_query_request(daid_list, daid_list, cfgdict=cfgdict)
>>> index_method = 'multi'
>>> mxer = request_ibeis_mindexer(qreq_, index_method)
>>> # Do some work in the foreground to ensure that it doesnt block
>>> # the background job
>>> print('[FG] sleeping or doing bit compute')
>>> # Takes about 15 seconds
>>> with ut.Timer():
... ut.enumerate_primes(int(9E4))
>>> #time.sleep(10)
>>> print('[FG] done sleeping')
>>> ut.assert_eq(len(mxer.nn_indexer_list), 2, 'two subindexer')
>>> # And this shoud build just one subindexer
>>> daid_list = valid_aids[1:60]
>>> qreq_ = ibs.new_query_request(daid_list, daid_list, cfgdict=cfgdict)
>>> index_method = 'multi'
>>> mxer = request_ibeis_mindexer(qreq_, index_method)
>>> ut.assert_eq(len(mxer.nn_indexer_list), 1, 'one big subindexer')
"""
min_reindex_thresh = qreq_.qparams.min_reindex_thresh
max_subindexers = qreq_.qparams.max_subindexers
daid_list = qreq_.get_internal_daids()
print('[mindex] make MultiNeighborIndex over %d annots' % (len(daid_list),))
print('[mindex] index_method=%r' % index_method)
# Split annotations into groups accorindg to index_method
ibs = qreq_.ibs
if index_method == 'name':
# each group are annotations of the same name
num_indexers = 8
aids_list, overflow_aids, num_bins = group_daids_for_indexing_by_name(ibs, daid_list, num_indexers, verbose)
elif index_method == 'multi':
neighbor_index_cache.check_background_process()
# Use greedy set cover to get a list of nnindxers that are already built
tup = neighbor_index_cache.group_daids_by_cached_nnindexer(
qreq_, daid_list, min_reindex_thresh)
uncovered_aids, covered_aids_list = tup
# If the number of bins gets too big do a reindex
# in the background
num_subindexers = len(covered_aids_list) + (len(uncovered_aids) > 1)
if num_subindexers > max_subindexers:
print('need to reindex something')
if USE_FORGROUND_REINDEX:
aids_list = [sorted(ut.flatten(covered_aids_list))]
#ut.embed()
else:
neighbor_index_cache.request_background_nnindexer(qreq_, daid_list)
aids_list = covered_aids_list
else:
aids_list = covered_aids_list
if len(uncovered_aids) > 0:
aids_list.append(uncovered_aids)
num_bins = len(aids_list)
else:
raise AssertionError('unknown index_method=%r' % (index_method,))
# Build a neighbor indexer for each
nn_indexer_list = []
#extra_indexes = []
for tx, aids in enumerate(aids_list):
print('[mindex] building forest %d/%d with %d aids' %
(tx + 1, num_bins, len(aids)))
if len(aids) > 0:
# Dont bother shallow copying qreq_ here.
# just passing aids is enough
nnindexer = neighbor_index_cache.request_memcached_ibeis_nnindexer(qreq_, aids)
nn_indexer_list.append(nnindexer)
#if len(unknown_aids) > 0:
# print('[mindex] building unknown forest')
# unknown_vecs_list = ibs.get_annot_vecs(overflow_aids, config2_=qreq_.get_internal_data_config2())
# unknown_index = NeighborIndex(overflow_aids, unknown_vecs_list)
# extra_indexes.append(unknown_index)
##print('[mindex] building normalizer forest') # TODO
#mxer.nn_indexer_list = nn_indexer_list
#mxer.extra_indexes = extra_indexes
#mxer.overflow_index = overflow_index
#mxer.unknown_index = unknown_index
mxer = MultiNeighborIndex(nn_indexer_list, min_reindex_thresh, max_subindexers)
return mxer
def request_ibeis_mindexer(qreq_, index_method='multi', verbose=True):
"""
CommandLine:
python -m ibeis.algo.hots.multi_index --test-request_ibeis_mindexer:2
Example0:
>>> # SLOW_DOCTEST
>>> from ibeis.algo.hots.multi_index import * # NOQA
>>> import ibeis
>>> ibs = ibeis.opendb(db='PZ_MTEST')
>>> valid_aids = ibs.get_valid_aids()
>>> daid_list = valid_aids[1:60]
>>> cfgdict = dict(fg_on=False)
>>> qreq_ = ibs.new_query_request(daid_list, daid_list, cfgdict=cfgdict)
>>> index_method = 'multi'
>>> mxer = request_ibeis_mindexer(qreq_, index_method)
Example1:
>>> # DISABLE_DOCTEST
>>> from ibeis.algo.hots.multi_index import * # NOQA
>>> import ibeis
>>> ibs = ibeis.opendb(db='PZ_Master0')
>>> valid_aids = ibs.get_valid_aids()
>>> daid_list = valid_aids[1:60]
>>> cfgdict = dict(fg_on=False)
>>> qreq_ = ibs.new_query_request(daid_list, daid_list, cfgdict=cfgdict)
>>> index_method = 'multi'
>>> mxer = request_ibeis_mindexer(qreq_, index_method)
Example2:
>>> # DISABLE_DOCTEST
>>> # Test background reindex
>>> from ibeis.algo.hots.multi_index import * # NOQA
>>> import ibeis
>>> import time
>>> ibs = ibeis.opendb(db='PZ_MTEST')
>>> valid_aids = ibs.get_valid_aids()
>>> # Remove all cached nnindexers
>>> ibs.delete_flann_cachedir()
>>> # This request should build a new nnindexer
>>> daid_list = valid_aids[1:30]
>>> cfgdict = dict(fg_on=False)
>>> qreq_ = ibs.new_query_request(daid_list, daid_list, cfgdict=cfgdict)
>>> index_method = 'multi'
>>> mxer = request_ibeis_mindexer(qreq_, index_method)
>>> ut.assert_eq(len(mxer.nn_indexer_list), 1, 'one subindexer')
>>> # The next request should trigger a background process
>>> # and build two subindexer
>>> daid_list = valid_aids[1:60]
>>> qreq_ = ibs.new_query_request(daid_list, daid_list, cfgdict=cfgdict)
>>> index_method = 'multi'
>>> mxer = request_ibeis_mindexer(qreq_, index_method)
>>> # Do some work in the foreground to ensure that it doesnt block
>>> # the background job
>>> print('[FG] sleeping or doing bit compute')
>>> # Takes about 15 seconds
>>> with ut.Timer():
... ut.enumerate_primes(int(9E4))
>>> #time.sleep(10)
>>> print('[FG] done sleeping')
>>> ut.assert_eq(len(mxer.nn_indexer_list), 2, 'two subindexer')
>>> # And this shoud build just one subindexer
>>> daid_list = valid_aids[1:60]
>>> qreq_ = ibs.new_query_request(daid_list, daid_list, cfgdict=cfgdict)
>>> index_method = 'multi'
>>> mxer = request_ibeis_mindexer(qreq_, index_method)
>>> ut.assert_eq(len(mxer.nn_indexer_list), 1, 'one big subindexer')
"""
min_reindex_thresh = qreq_.qparams.min_reindex_thresh
max_subindexers = qreq_.qparams.max_subindexers
daid_list = qreq_.get_internal_daids()
print('[mindex] make MultiNeighborIndex over %d annots' % (len(daid_list),))
print('[mindex] index_method=%r' % index_method)
# Split annotations into groups accorindg to index_method
ibs = qreq_.ibs
if index_method == 'name':
# each group are annotations of the same name
num_indexers = 8
aids_list, overflow_aids, num_bins = group_daids_for_indexing_by_name(ibs, daid_list, num_indexers, verbose)
elif index_method == 'multi':
neighbor_index_cache.check_background_process()
# Use greedy set cover to get a list of nnindxers that are already built
tup = neighbor_index_cache.group_daids_by_cached_nnindexer(
qreq_, daid_list, min_reindex_thresh)
uncovered_aids, covered_aids_list = tup
# If the number of bins gets too big do a reindex
# in the background
num_subindexers = len(covered_aids_list) + (len(uncovered_aids) > 1)
if num_subindexers > max_subindexers:
print('need to reindex something')
if USE_FORGROUND_REINDEX:
aids_list = [sorted(ut.flatten(covered_aids_list))]
#ut.embed()
else:
neighbor_index_cache.request_background_nnindexer(qreq_, daid_list)
aids_list = covered_aids_list
else:
aids_list = covered_aids_list
if len(uncovered_aids) > 0:
aids_list.append(uncovered_aids)
num_bins = len(aids_list)
else:
raise AssertionError('unknown index_method=%r' % (index_method,))
# Build a neighbor indexer for each
nn_indexer_list = []
#extra_indexes = []
for tx, aids in enumerate(aids_list):
print('[mindex] building forest %d/%d with %d aids' %
(tx + 1, num_bins, len(aids)))
if len(aids) > 0:
# Dont bother shallow copying qreq_ here.
# just passing aids is enough
nnindexer = neighbor_index_cache.request_memcached_ibeis_nnindexer(qreq_, aids)
nn_indexer_list.append(nnindexer)
#if len(unknown_aids) > 0:
# print('[mindex] building unknown forest')
# unknown_vecs_list = ibs.get_annot_vecs(overflow_aids, config2_=qreq_.get_internal_data_config2())
# unknown_index = NeighborIndex(overflow_aids, unknown_vecs_list)
# extra_indexes.append(unknown_index)
##print('[mindex] building normalizer forest') # TODO
#mxer.nn_indexer_list = nn_indexer_list
#mxer.extra_indexes = extra_indexes
#mxer.overflow_index = overflow_index
#mxer.unknown_index = unknown_index
mxer = MultiNeighborIndex(nn_indexer_list, min_reindex_thresh, max_subindexers)
return mxer