def test_L2(self):
swig_ptr = faiss.swig_ptr
x = faiss.rand((100, 10), seed=1)
y = faiss.rand((200, 10), seed=2)
ix = faiss.randint(50, vmax=100)
iy = faiss.randint(50, vmax=200)
dis = np.empty(50, dtype='float32')
faiss.pairwise_indexed_L2sqr(10, 50, swig_ptr(x), swig_ptr(ix),
swig_ptr(y), swig_ptr(iy), swig_ptr(dis))
for i in range(50):
assert np.allclose(dis[i], ((x[ix[i]] - y[iy[i]])**2).sum())
def test_IP(self):
swig_ptr = faiss.swig_ptr
x = faiss.rand((100, 10), seed=1)
y = faiss.rand((200, 10), seed=2)
ix = faiss.randint(50, vmax=100)
iy = faiss.randint(50, vmax=200)
dis = np.empty(50, dtype='float32')
faiss.pairwise_indexed_inner_product(10, 50, swig_ptr(x), swig_ptr(ix),
swig_ptr(y), swig_ptr(iy),
swig_ptr(dis))
for i in range(50):
assert np.allclose(dis[i], np.dot(x[ix[i]], y[iy[i]]))
def test_clone(self):
index = faiss.index_factory(16, 'IVF10,PQ4np')
xb = faiss.rand((1000, 16))
index.train(xb)
index.add(xb)
index2 = faiss.clone_index(index)
assert index2.ntotal == index.ntotal
def test_clone_size(self):
index = faiss.index_factory(20, 'PCA10,Flat')
xb = faiss.rand((100, 20))
index.train(xb)
index.add(xb)
index2 = faiss.clone_index(index)
assert index2.ntotal == 100
def simulate_mee_runtime(n_videos=1000000, d=256, n_query=100, max_neighbors=100, n_runs=5, n_warmup_runs=10):
""" Search over a database of shape [n_videos, d] with query of shape [n_query, d].
For each query, return max_neighbors results.
"""
import faiss
torch.cuda.synchronize()
st_time = time.time()
fake_database = faiss.rand((n_videos, d))
fake_query = faiss.rand((n_query, d))
torch.cuda.synchronize()
logger.info("Construct fake database + query time {}".format(time.time() - st_time))
torch.cuda.synchronize()
st_time = time.time()
index = faiss.index_factory(d, "IVF4096,Flat", faiss.METRIC_L2)
index_ivf = faiss.extract_index_ivf(index)
clustering_index = faiss.index_cpu_to_all_gpus(faiss.IndexFlatL2(d))
index_ivf.clustering_index = clustering_index
torch.cuda.synchronize()
logger.info("Build/Move to GPU? index time {}".format(time.time() - st_time))
st_time = time.time()
torch.cuda.synchronize()
index_ivf.train(fake_database)
torch.cuda.synchronize()
logger.info("Train index time {}".format(time.time() - st_time))
times = []
for _ in range(n_warmup_runs+n_runs):
torch.cuda.synchronize()
st_time = time.time()
D, I = index_ivf.search(fake_query, max_neighbors)
torch.cuda.synchronize()
times.append(time.time() - st_time)
avg_time = np.mean(times[n_warmup_runs:]) * 2 # video + sub
logger.info("Avg searching time ({} runs) {}".format(n_runs, avg_time))
return avg_time
def run_test(self, keep_max):
nq = 100
nb = 1000
restab = faiss.rand((nq, nb), 123)
ids = faiss.randint((nq, nb), 1324, 10000)
all_rh = {}
for nstep in 1, 3:
rh = faiss.ResultHeap(nq, 10, keep_max=keep_max)
for i in range(nstep):
i0, i1 = i * nb // nstep, (i + 1) * nb // nstep
D = restab[:, i0:i1].copy()
I = ids[:, i0:i1].copy()
rh.add_result(D, I)
rh.finalize()
if keep_max:
assert np.all(rh.D[:, :-1] >= rh.D[:, 1:])
else:
assert np.all(rh.D[:, :-1] <= rh.D[:, 1:])
all_rh[nstep] = rh
np.testing.assert_equal(all_rh[1].D, all_rh[3].D)
np.testing.assert_equal(all_rh[1].I, all_rh[3].I)
def test_rand(self):
x = faiss.rand(2000)
assert np.all(x >= 0) and np.all(x < 1)
h, _ = np.histogram(x, np.arange(0, 1, 0.1))
assert h.min() > 160 and h.max() < 240
1 / 0
xd = 100
yd = 1000000
np.random.seed(1234)
faiss.omp_set_num_threads(1)
print('xd=%d yd=%d' % (xd, yd))
print('Running inner products test..')
for d in 3, 4, 12, 36, 64:
x = faiss.rand(xd * d).reshape(xd, d)
y = faiss.rand(yd * d).reshape(yd, d)
distances = np.empty((xd, yd), dtype='float32')
t0 = time.time()
for i in xrange(xd):
faiss.fvec_inner_products_ny(swig_ptr(distances[i]), swig_ptr(x[i]),
swig_ptr(y), d, yd)
t1 = time.time()
# sparse verification
ntry = 100
num, denom = 0, 0
for t in range(ntry):
xi = np.random.randint(xd)
def test_issue_2019(self):
index = faiss.index_factory(
32, "PCAR16,IVF200(IVF10,PQ2x4fs,RFlat),PQ4x4fsr")
des = faiss.rand((1000, 32))
index.train(des)
