def test_L2(self): swig_ptr = faiss.swig_ptr x = faiss.rand((100, 10), seed=1) y = faiss.rand((200, 10), seed=2) ix = faiss.randint(50, vmax=100) iy = faiss.randint(50, vmax=200) dis = np.empty(50, dtype='float32') faiss.pairwise_indexed_L2sqr(10, 50, swig_ptr(x), swig_ptr(ix), swig_ptr(y), swig_ptr(iy), swig_ptr(dis)) for i in range(50): assert np.allclose(dis[i], ((x[ix[i]] - y[iy[i]])**2).sum())
def test_IP(self): swig_ptr = faiss.swig_ptr x = faiss.rand((100, 10), seed=1) y = faiss.rand((200, 10), seed=2) ix = faiss.randint(50, vmax=100) iy = faiss.randint(50, vmax=200) dis = np.empty(50, dtype='float32') faiss.pairwise_indexed_inner_product(10, 50, swig_ptr(x), swig_ptr(ix), swig_ptr(y), swig_ptr(iy), swig_ptr(dis)) for i in range(50): assert np.allclose(dis[i], np.dot(x[ix[i]], y[iy[i]]))
def test_clone(self): index = faiss.index_factory(16, 'IVF10,PQ4np') xb = faiss.rand((1000, 16)) index.train(xb) index.add(xb) index2 = faiss.clone_index(index) assert index2.ntotal == index.ntotal
def test_clone_size(self): index = faiss.index_factory(20, 'PCA10,Flat') xb = faiss.rand((100, 20)) index.train(xb) index.add(xb) index2 = faiss.clone_index(index) assert index2.ntotal == 100
def simulate_mee_runtime(n_videos=1000000, d=256, n_query=100, max_neighbors=100, n_runs=5, n_warmup_runs=10): """ Search over a database of shape [n_videos, d] with query of shape [n_query, d]. For each query, return max_neighbors results. """ import faiss torch.cuda.synchronize() st_time = time.time() fake_database = faiss.rand((n_videos, d)) fake_query = faiss.rand((n_query, d)) torch.cuda.synchronize() logger.info("Construct fake database + query time {}".format(time.time() - st_time)) torch.cuda.synchronize() st_time = time.time() index = faiss.index_factory(d, "IVF4096,Flat", faiss.METRIC_L2) index_ivf = faiss.extract_index_ivf(index) clustering_index = faiss.index_cpu_to_all_gpus(faiss.IndexFlatL2(d)) index_ivf.clustering_index = clustering_index torch.cuda.synchronize() logger.info("Build/Move to GPU? index time {}".format(time.time() - st_time)) st_time = time.time() torch.cuda.synchronize() index_ivf.train(fake_database) torch.cuda.synchronize() logger.info("Train index time {}".format(time.time() - st_time)) times = [] for _ in range(n_warmup_runs+n_runs): torch.cuda.synchronize() st_time = time.time() D, I = index_ivf.search(fake_query, max_neighbors) torch.cuda.synchronize() times.append(time.time() - st_time) avg_time = np.mean(times[n_warmup_runs:]) * 2 # video + sub logger.info("Avg searching time ({} runs) {}".format(n_runs, avg_time)) return avg_time
def run_test(self, keep_max): nq = 100 nb = 1000 restab = faiss.rand((nq, nb), 123) ids = faiss.randint((nq, nb), 1324, 10000) all_rh = {} for nstep in 1, 3: rh = faiss.ResultHeap(nq, 10, keep_max=keep_max) for i in range(nstep): i0, i1 = i * nb // nstep, (i + 1) * nb // nstep D = restab[:, i0:i1].copy() I = ids[:, i0:i1].copy() rh.add_result(D, I) rh.finalize() if keep_max: assert np.all(rh.D[:, :-1] >= rh.D[:, 1:]) else: assert np.all(rh.D[:, :-1] <= rh.D[:, 1:]) all_rh[nstep] = rh np.testing.assert_equal(all_rh[1].D, all_rh[3].D) np.testing.assert_equal(all_rh[1].I, all_rh[3].I)
def test_rand(self): x = faiss.rand(2000) assert np.all(x >= 0) and np.all(x < 1) h, _ = np.histogram(x, np.arange(0, 1, 0.1)) assert h.min() > 160 and h.max() < 240
1 / 0 xd = 100 yd = 1000000 np.random.seed(1234) faiss.omp_set_num_threads(1) print('xd=%d yd=%d' % (xd, yd)) print('Running inner products test..') for d in 3, 4, 12, 36, 64: x = faiss.rand(xd * d).reshape(xd, d) y = faiss.rand(yd * d).reshape(yd, d) distances = np.empty((xd, yd), dtype='float32') t0 = time.time() for i in xrange(xd): faiss.fvec_inner_products_ny(swig_ptr(distances[i]), swig_ptr(x[i]), swig_ptr(y), d, yd) t1 = time.time() # sparse verification ntry = 100 num, denom = 0, 0 for t in range(ntry): xi = np.random.randint(xd)
def test_issue_2019(self): index = faiss.index_factory( 32, "PCAR16,IVF200(IVF10,PQ2x4fs,RFlat),PQ4x4fsr") des = faiss.rand((1000, 32)) index.train(des)