index = faiss.GpuIndexFlatL2(res, d, flat_config)
print("add vectors to index")
index.add(xb)
print("warmup")
index.search(xq, 123)
print("benchmark")
for lk in range(11):
k = 1 << lk
t, r = evaluate(index, xq, gt, k)
# the recall should be 1 at all times
print("k=%d %.3f ms, R@1 %.4f" % (k, t, r[1]))
#################################################################
# Approximate search experiment
#################################################################
print("============ Approximate search")
index = faiss.index_factory(d, "IVF4096,PQ64")
# faster, uses more memory
# index = faiss.index_factory(d, "IVF16384,Flat")
#############################################################
# Index is ready
#############################################################
xq = sanitize(xq)
if args.searchthreads != -1:
print "Setting nb of threads to", args.searchthreads
faiss.omp_set_num_threads(args.searchthreads)
if gt is None:
print "no valid groundtruth -- exit"
sys.exit()
k_reorders = [int(x) for x in args.k_reorder.split(',')]
efSearchs = [int(x) for x in args.efSearch.split(',')]
for k_reorder in k_reorders:
if index_hnsw.reconstruct_from_neighbors:
print "setting k_reorder=%d" % k_reorder
index_hnsw.reconstruct_from_neighbors.k_reorder = k_reorder
for efSearch in efSearchs:
print "efSearch=%-4d" % efSearch,
hnsw.efSearch = efSearch
hnsw_stats.reset()
datasets.evaluate(xq, gt, index, k=args.k, endl=False)
print "ndis %d nreorder %d" % (hnsw_stats.ndis, hnsw_stats.nreorder)
print("load data")
xb, xq, xt, gt = load_sift1M()
nq, d = xq.shape
# index with 16 subquantizers, 8 bit each
index = faiss.IndexPQ(d, 16, 8)
index.do_polysemous_training = True
index.verbose = True
print("train")
index.train(xt)
print("add vectors to index")
index.add(xb)
nt = 1
faiss.omp_set_num_threads(1)
print("PQ baseline", end=' ')
index.search_type = faiss.IndexPQ.ST_PQ
evaluate()
for ht in 64, 62, 58, 54, 50, 46, 42, 38, 34, 30:
print("Polysemous", ht, end=' ')
index.search_type = faiss.IndexPQ.ST_polysemous
index.polysemous_ht = ht
t, r = evaluate(index, xq, gt, 1)
print("\t %7.3f ms per query, R@1 %.4f" % (t, r[1]))
