def test_nnindexer(*args, **kwargs):
from ibeis.algo.hots.neighbor_index_cache import test_nnindexer
return test_nnindexer(*args, **kwargs)
def test_multiple_add_removes():
r"""
CommandLine:
python -m ibeis.algo.hots._neighbor_experiment --exec-test_multiple_add_removes
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.algo.hots._neighbor_experiment import * # NOQA
>>> result = test_multiple_add_removes()
>>> print(result)
"""
from ibeis.algo.hots.neighbor_index_cache import test_nnindexer
K = 4
nnindexer, qreq_, ibs = test_nnindexer('PZ_MTEST', use_memcache=False)
assert len(nnindexer.get_removed_idxs()) == 0
print('\n\n --- got nnindex testdata --- ')
print('')
@ut.tracefunc_xml
def print_nnindexer(nnindexer):
print('nnindexer.get_indexed_aids() = %r' % (nnindexer.get_indexed_aids(),))
print('nnindexer.num_indexed_vecs() = %r' % (nnindexer.num_indexed_vecs(),))
print('nnindexer.get_removed_idxs().shape = %r' % (nnindexer.get_removed_idxs().shape,))
print('INITIALIZE TEST')
print_nnindexer(nnindexer)
config2_ = qreq_.get_internal_query_config2()
qaid = 1
qfx2_vec = ibs.get_annot_vecs(qaid, config2_=config2_)
(qfx2_idx1, qfx2_dist1) = nnindexer.knn(qfx2_vec, K)
aids1 = set(nnindexer.get_nn_aids(qfx2_idx1).ravel())
print('aids1 = %r' % (aids1,))
print('')
print('TESTING ADD')
add_first_daids = [17, 22]
nnindexer.add_ibeis_support(qreq_, add_first_daids)
print_nnindexer(nnindexer)
(qfx2_idx0, qfx2_dist0) = nnindexer.knn(qfx2_vec, K)
assert np.any(qfx2_idx0 != qfx2_idx1), 'some should change'
aids0 = set(nnindexer.get_nn_aids(qfx2_idx0).ravel())
print('aids0 = %r' % (aids0,))
# execute test function
print('')
print('TESTING REMOVE')
remove_daid_list = [8, 10, 11]
nnindexer.remove_ibeis_support(qreq_, remove_daid_list)
print_nnindexer(nnindexer)
# test after modification
(qfx2_idx2, qfx2_dist2) = nnindexer.knn(qfx2_vec, K)
aids2 = set(nnindexer.get_nn_aids(qfx2_idx2).ravel())
print('aids2 = %r' % (aids2,))
assert len(aids2.intersection(remove_daid_list)) == 0
__removed_ids = nnindexer.flann._FLANN__removed_ids
invalid_idxs = nnindexer.get_removed_idxs()
assert len(np.intersect1d(invalid_idxs, __removed_ids)) == len(__removed_ids)
print('')
print('TESTING DUPLICATE REMOVE')
nnindexer.remove_ibeis_support(qreq_, remove_daid_list)
print_nnindexer(nnindexer)
# test after modification
(qfx2_idx2_, qfx2_dist2_) = nnindexer.knn(qfx2_vec, K)
assert np.all(qfx2_idx2_ == qfx2_idx2)
assert np.all(qfx2_dist2_ == qfx2_dist2)
print('')
print('TESTING ADD AFTER REMOVE')
# Is the error here happening because added points seem to
# get the ids of the removed points?
new_daid_list = [8, 10]
nnindexer.add_ibeis_support(qreq_, new_daid_list)
print_nnindexer(nnindexer)
# test after modification
(qfx2_idx3, qfx2_dist3) = nnindexer.knn(qfx2_vec, K)
qfx2_aid3 = nnindexer.get_nn_aids(qfx2_idx3)
found_removed_idxs = np.intersect1d(qfx2_idx3, nnindexer.get_removed_idxs())
if len(found_removed_idxs) != 0:
print('found_removed_idxs.max() = %r' % (found_removed_idxs.max(),))
print('found_removed_idxs.min() = %r' % (found_removed_idxs.min(),))
raise AssertionError('found_removed_idxs.shape = %r' % (found_removed_idxs.shape,))
aids3 = set(qfx2_aid3.ravel())
assert aids3.intersection(remove_daid_list) == set(new_daid_list).intersection(remove_daid_list)
print('TESTING DUPLICATE ADD')
new_daid_list = [8, 10]
nnindexer.add_ibeis_support(qreq_, new_daid_list)
# test after modification
print_nnindexer(nnindexer)
(qfx2_idx3_, qfx2_dist3_) = nnindexer.knn(qfx2_vec, K)
qfx2_aid3_ = nnindexer.get_nn_aids(qfx2_idx3_)
assert np.all(qfx2_aid3 == qfx2_aid3_)
print('TESTING ADD QUERY TO DATABASE')
add_daid_list1 = [qaid]
nnindexer.add_ibeis_support(qreq_, add_daid_list1)
print_nnindexer(nnindexer)
(qfx2_idx4_, qfx2_dist4_) = nnindexer.knn(qfx2_vec, K)
qfx2_aid4_ = nnindexer.get_nn_aids(qfx2_idx4_)
qfx2_fx4_ = nnindexer.get_nn_featxs(qfx2_idx4_)
assert np.all(qfx2_aid4_.T[0] == qaid), 'should find self'
assert ut.issorted(qfx2_fx4_.T[0]), 'should be in order'
print('TESTING REMOVE QUERY POINTS')
add_daid_list1 = [qaid]
nnindexer.remove_ibeis_support(qreq_, add_daid_list1)
print_nnindexer(nnindexer)
(qfx2_idx5_, qfx2_dist5_) = nnindexer.knn(qfx2_vec, K)
issame = (qfx2_idx5_ == qfx2_idx3_)
percentsame = issame.sum() / issame.size
print('percentsame = %r' % (percentsame,))
assert percentsame > .85, 'a large majority of the feature idxs should remain the same'
print_nnindexer(nnindexer)
# Do this multiple times
for _ in range(10):
add_daid_list1 = [qaid]
nnindexer.add_ibeis_support(qreq_, add_daid_list1, verbose=False)
nnindexer.remove_ibeis_support(qreq_, add_daid_list1, verbose=False)
(qfx2_idxX_, qfx2_distX_) = nnindexer.knn(qfx2_vec, K)
issame = (qfx2_idxX_ == qfx2_idx3_)
percentsame = issame.sum() / issame.size
print('percentsame = %r' % (percentsame,))
assert percentsame > .85, 'a large majority of the feature idxs should remain the same'
# Test again with more data
print('testing remove query points with more data')
nnindexer.add_ibeis_support(qreq_, ibs.get_valid_aids())
(qfx2_idx6_, qfx2_dist6_) = nnindexer.knn(qfx2_vec, K)
qfx2_aid6_ = nnindexer.get_nn_aids(qfx2_idx6_)
assert np.all(qfx2_aid6_.T[0] == qaid), 'should be same'
nnindexer.remove_ibeis_support(qreq_, add_daid_list1)
print_nnindexer(nnindexer)
(qfx2_idx7_, qfx2_dist6_) = nnindexer.knn(qfx2_vec, K)
qfx2_aid7_ = nnindexer.get_nn_aids(qfx2_idx7_)
assert np.all(qfx2_aid7_.T[0] != qaid), 'should not be same'
# Do this multiple times
for _ in range(10):
add_daid_list1 = [qaid]
nnindexer.add_ibeis_support(qreq_, add_daid_list1, verbose=True)
nnindexer.remove_ibeis_support(qreq_, add_daid_list1, verbose=True)
# weird that all seem to work here
(qfx2_idxX_, qfx2_distX_) = nnindexer.knn(qfx2_vec, K)
issame = (qfx2_idxX_ == qfx2_idx7_)
percentsame = issame.sum() / issame.size
print('percentsame = %r' % (percentsame,))
print_nnindexer(nnindexer)
assert percentsame > .85, 'a large majority of the feature idxs should remain the same'
nnindexer, qreq_, ibs = test_nnindexer('PZ_MTEST', use_memcache=False)
big_set = ibs.get_valid_aids()[5:]
remove_later = big_set[10:14]
nnindexer.add_ibeis_support(qreq_, big_set)
# Try again where remove is not the last operation
print('testing remove query points with more op')
extra_data = np.setdiff1d(ibs.get_valid_aids()[0:5], add_daid_list1)
nnindexer.remove_ibeis_support(qreq_, extra_data)
nnindexer.add_ibeis_support(qreq_, add_daid_list1)
nnindexer.add_ibeis_support(qreq_, extra_data)
(qfx2_idx8_, qfx2_dist8_) = nnindexer.knn(qfx2_vec, K)
qfx2_aid8_ = nnindexer.get_nn_aids(qfx2_idx8_)
assert np.all(qfx2_aid8_.T[0] == qaid), 'should be same'
nnindexer.remove_ibeis_support(qreq_, extra_data)
(qfx2_idx9_, qfx2_dist9_) = nnindexer.knn(qfx2_vec, K)
qfx2_aid9_ = nnindexer.get_nn_aids(qfx2_idx9_)
assert np.all(qfx2_aid9_.T[0] == qaid), 'should be same'
nnindexer.remove_ibeis_support(qreq_, add_daid_list1)
nnindexer.add_ibeis_support(qreq_, add_daid_list1)
nnindexer.add_ibeis_support(qreq_, extra_data)
nnindexer.remove_ibeis_support(qreq_, remove_later)
print(nnindexer.ax2_aid)
aid_list = nnindexer.get_indexed_aids() # NOQA
nnindexer.flann.save_index('test.flann')
idx2_vec_masked = nnindexer.idx2_vec
idx2_vec_compressed = nnindexer.get_indexed_vecs()
import pyflann
flann1 = pyflann.FLANN()
flann1.load_index('test.flann', idx2_vec_masked)
import pyflann
flann2 = pyflann.FLANN()
flann2.load_index('test.flann', idx2_vec_compressed)
# NOW WE NEED TO TEST THAT WE CAN SAVE AND LOAD THIS DATA
#
#ax2_nvecs = ut.dict_take(ut.dict_hist(nnindexer.idx2_ax), range(len(nnindexer.ax2_aid)))
pass
def test_nnindexer(*args, **kwargs):
from ibeis.algo.hots.neighbor_index_cache import test_nnindexer
return test_nnindexer(*args, **kwargs)
