def test_annoy_wrap_indexer(self):
a = NumpyIndexer(index_filename='wrap-npidx.gz')
a.name = 'wrap-npidx'
a.add(vec_idx, vec)
a.save()
a.close()
b = BaseIndexer.load_config(os.path.join(cur_dir, 'annoy-wrap.yml'))
idx, dist = b.query(query, top_k=4)
print(idx, dist)
global retr_idx
if retr_idx is None:
retr_idx = idx
else:
np.testing.assert_almost_equal(retr_idx, idx)
self.assertEqual(idx.shape, dist.shape)
self.assertEqual(idx.shape, (10, 4))
c = BaseIndexer.load_config(os.path.join(cur_dir, 'nmslib-wrap.yml'))
idx, dist = c.query(query, top_k=4)
print(idx, dist)
if retr_idx is None:
retr_idx = idx
else:
np.testing.assert_almost_equal(retr_idx, idx)
self.assertEqual(idx.shape, dist.shape)
self.assertEqual(idx.shape, (10, 4))
self.add_tmpfile(a.index_abspath, a.save_abspath)
def test_annoy_wrap_indexer(self):
with NumpyIndexer(index_filename='wrap-npidx.gz') as a:
a.name = 'wrap-npidx'
a.add(vec_idx, vec)
a.save()
index_abspath = a.index_abspath
save_abspath = a.save_abspath
with BaseIndexer.load_config(os.path.join(cur_dir, 'yaml/annoy-wrap.yml')) as b:
idx, dist = b.query(query, top_k=4)
global retr_idx
if retr_idx is None:
retr_idx = idx
else:
np.testing.assert_almost_equal(retr_idx, idx)
self.assertEqual(idx.shape, dist.shape)
self.assertEqual(idx.shape, (10, 4))
with BaseIndexer.load_config(os.path.join(cur_dir, 'yaml/nmslib-wrap.yml')) as c:
idx, dist = c.query(query, top_k=4)
if retr_idx is None:
retr_idx = idx
else:
np.testing.assert_almost_equal(retr_idx, idx)
self.assertEqual(idx.shape, dist.shape)
self.assertEqual(idx.shape, (10, 4))
self.add_tmpfile(index_abspath, save_abspath)
def get_numpy_indexer(params):
name = params['name']
return NumpyIndexer(index_filename=f'{name}.tgz',
compress_level=1,
metas={
'workspace': WORKSPACE_DIR,
'warn_unnamed': False,
'separated_workspace': False,
'is_trained': False,
'max_snapshot': 0
})
def test_scipy_indexer(self):
a = NumpyIndexer(index_filename='np.test.gz', backend='scipy')
a.add(vec_idx, vec)
a.save()
a.close()
self.assertTrue(os.path.exists(a.index_abspath))
# a.query(np.array(np.random.random([10, 5]), dtype=np.float32), top_k=4)
b = BaseIndexer.load(a.save_abspath)
idx, dist = b.query(query, top_k=4)
print(idx, dist)
global retr_idx
if retr_idx is None:
retr_idx = idx
else:
np.testing.assert_almost_equal(retr_idx, idx)
self.assertEqual(idx.shape, dist.shape)
self.assertEqual(idx.shape, (10, 4))
self.add_tmpfile(a.index_abspath, a.save_abspath)
def test_scipy_indexer(self):
with NumpyIndexer(index_filename='np.test.gz', backend='scipy') as a:
a.add(vec_idx, vec)
a.save()
self.assertTrue(os.path.exists(a.index_abspath))
index_abspath = a.index_abspath
save_abspath = a.save_abspath
with BaseIndexer.load(save_abspath) as b:
idx, dist = b.query(query, top_k=4)
global retr_idx
if retr_idx is None:
retr_idx = idx
else:
np.testing.assert_almost_equal(retr_idx, idx)
self.assertEqual(idx.shape, dist.shape)
self.assertEqual(idx.shape, (10, 4))
self.add_tmpfile(index_abspath, save_abspath)
def test_np_indexer_known(self):
vectors = np.array([[1, 1, 1], [10, 10, 10], [100, 100, 100],
[1000, 1000, 1000]])
keys = np.array([0, 1, 2, 3]).reshape(-1, 1)
with NumpyIndexer(index_filename='np.test.gz') as a:
a.add(keys, vectors)
a.save()
self.assertTrue(os.path.exists(a.index_abspath))
index_abspath = a.index_abspath
save_abspath = a.save_abspath
queries = np.array([[1, 1, 1], [10, 10, 10], [100, 100, 100],
[1000, 1000, 1000]])
with BaseIndexer.load(save_abspath) as b:
idx, dist = b.query(queries, top_k=2)
np.testing.assert_equal(idx,
np.array([[0, 1], [1, 0], [2, 1], [3, 2]]))
self.assertEqual(idx.shape, dist.shape)
self.assertEqual(idx.shape, (4, 2))
self.add_tmpfile(index_abspath, save_abspath)
def test_load_yaml1(self):
from jina.executors.indexers.vector.numpy import NumpyIndexer
NumpyIndexer.load_config(os.path.join(cur_dir, 'yaml/dummy_exec1.yml'))
self.add_tmpfile('test.gzip')