index = faiss.read_index(filename)
if save_numpy_index:
print("Saving index to numpy array...")
chunk = faiss.serialize_index(index)
np.save("{}.npy".format(filename), chunk)
print("Finish saving numpy index")
return index
#################################################################
# Perform searches
#################################################################
index = get_populated_index()
ps = faiss.ParameterSpace()
ps.initialize(index)
# make sure queries are in RAM
xq = xq.astype('float32').copy()
# a static C++ object that collects statistics about searches
ivfpq_stats = faiss.cvar.indexIVFPQ_stats
ivf_stats = faiss.cvar.indexIVF_stats
if parametersets == ['autotune'] or parametersets == ['autotuneMT']:
if parametersets == ['autotune']:
faiss.omp_set_num_threads(1)
basename = '/tmp/base%s%s.index' % (pf, key)
if os.path.exists(basename):
print('load', basename)
index_1 = faiss.read_index(basename)
else:
print('train + write', basename)
index_1 = faiss.index_factory(d, key)
index_1.train(xt)
faiss.write_index(index_1, basename)
print('add')
index_1.add(xb)
print('set nprobe=', nprobe)
faiss.ParameterSpace().set_index_parameter(index_1, 'nprobe', nprobe)
class ResultHeap:
""" Combine query results from a sliced dataset """
def __init__(self, nq, k):
" nq: number of query vectors, k: number of results per query "
self.I = np.zeros((nq, k), dtype='int64')
self.D = np.zeros((nq, k), dtype='float32')
self.nq, self.k = nq, k
heaps = faiss.float_maxheap_array_t()
heaps.k = k
heaps.nh = nq
heaps.val = faiss.swig_ptr(self.D)
heaps.ids = faiss.swig_ptr(self.I)
heaps.heapify()