def knngraph_to_CSRMatrix(I):
""" converts the neighbor indices I to a CSRMatrix, by:
- compute W0 + W0.T
- L1-normalize the W0 rows"""
print(' knngraph_to_CSRMatrix in: %.2f GiB' %
(faiss.get_mem_usage_kb() / float(1 << 20)))
N, k = I.shape
CSRMatrix = graph_utils.CSRMatrix
swig_ptr = graph_utils.swig_ptr
indptr = np.arange(N + 1, dtype='uint64')
indptr *= k
I = np.ascontiguousarray(I, dtype='int32')
vals = np.ones(I.size, dtype='float32')
W1 = CSRMatrix(N, N, swig_ptr(indptr), swig_ptr(I), swig_ptr(vals))
print(' A: %.2f GiB' % (faiss.get_mem_usage_kb() / float(1 << 20)))
W2 = W1.transpose()
print(' B: %.2f GiB' % (faiss.get_mem_usage_kb() / float(1 << 20)))
# changes I!
W1.sort_rows()
W2.sort_rows()
W = W1.point_op(W2, CSRMatrix.Pop_add)
W.rows_normalize_L1()
print(' C: %.2f GiB' % (faiss.get_mem_usage_kb() / float(1 << 20)))
del W1, W2, vals
print(' D: %.2f GiB, matrix nnz=%d' %
(faiss.get_mem_usage_kb() / float(1 << 20), W.count_nz()))
return W
print " add in %.3f s" % (time.time() - t0)
if args.indexfile:
print "storing", args.indexfile
faiss.write_index(index, args.indexfile)
if args.no_precomputed_tables:
if isinstance(index_ivf, faiss.IndexIVFPQ):
print "disabling precomputed table"
index_ivf.use_precomputed_table = -1
index_ivf.precomputed_table.clear()
if args.indexfile:
print "index size on disk: ", os.stat(args.indexfile).st_size
print "current RSS:", faiss.get_mem_usage_kb() * 1024
precomputed_table_size = 0
if hasattr(index_ivf, 'precomputed_table'):
precomputed_table_size = index_ivf.precomputed_table.size() * 4
print "precomputed tables size:", precomputed_table_size
#############################################################
# Index is ready
#############################################################
xq = sanitize(xq)
if args.searchthreads != -1:
print "Setting nb of threads to", args.searchthreads
def pt():
ts.append(time.time())
return "[%.3f s, %.2f GiB]" % (
ts[-1] - ts[-2], faiss.get_mem_usage_kb() / float(1 << 20))
import cPickle
import argparse
from scipy import sparse as SM
import faiss
import build_graph
import diffusion_dataset
def do_diffusion(Ytr, I, (indexes2, Yte), clstep,
niter, storeLiters=None):
print('[%.2f GiB] begin diffusion ' % (
faiss.get_mem_usage_kb() / float(1<<20)))
t0 = time.time()
nclasses = Ytr.max() + 1
if clstep == 0:
clstep = nclasses
N, k = I.shape
assert N == Ytr.size
nl1 = (Ytr >= 0).sum()
W = build_graph.knngraph_to_CSRMatrix(I)
# build the graph that links test points to the diffusion graph
N2 = indexes2.shape[0]
indptr = np.arange(N2 + 1) * k