class ApproximateIndexer(object):
def __init__(self,index_name,model_path,lmdb_path,V=16, M=16):
self.model = LOPQModel(V,M)
self.index_name = index_name
self.searcher = None
self.model_path = model_path
self.lmdb_path = lmdb_path
def load(self):
self.model.load_proto(self.model_path)
def fit(self,train):
print train.shape
self.pca_reduction = PCA(n_components=256)
self.pca_reduction.fit(train)
train = self.pca_reduction.transform(train)
self.P, self.mu = pca(train)
train = np.dot(train, self.P)
print train.shape
self.model.fit(train, n_init=1)
def transform(self,test):
print test.shape
test = self.pca_reduction.transform(test)
test = test - self.mu
test = np.dot(test,self.P)
print test.shape
return test
def fit_model(self,train):
self.fit(train)
self.model.export_proto(self.model_path)
self.searcher = LOPQSearcher(self.model) # LOPQSearcherLMDB(self.model,self.lmdb_path)
def experiment(self,data):
train, test = train_test_split(data, test_size=0.1)
print data.shape,train.shape,test.shape
nns = compute_all_neighbors(test, train)
self.fit_model(train)
self.searcher.add_data(self.transform(train))
recall, _ = get_recall(self.searcher, self.transform(test), nns)
print 'Recall (V={}, M={}, subquants={}): {}'.format(self.model.V, self.model.M, self.model.subquantizer_clusters, str(recall))
def add_data(self,data):
self.searcher.add_data(data)
def search(self,x):
return self.searcher.search(x,quota=100)
def load(self):
self.model = LOPQModel.load_proto(self.model_proto_filename)
self.pca_reduction = pickle.load(file(self.pca_filename))
self.P = np.load(file(self.P_filename))
self.mu = np.load(file(self.mu_filename))
self.permuted_inds = np.load(file(self.permuted_inds_filename))
self.searcher = LOPQSearcherLMDB(model=self.model,lmdb_path=self.model_lmdb_filename)
def load(self):
logging.info("Loading LOPQ indexer model {}".format(self.name))
self.model = LOPQModel.load_proto(self.model_proto_filename)
self.pca_reduction = pickle.load(file(self.pca_filename))
self.P = np.load(file(self.P_filename))
self.mu = np.load(file(self.mu_filename))
self.permuted_inds = np.load(file(self.permuted_inds_filename))
def load(self):
self.model = LOPQModel.load_proto(self.model_proto_filename)
self.pca_reduction = pickle.load(file(self.pca_filename))
self.P = np.load(file(self.P_filename))
self.mu = np.load(file(self.mu_filename))
self.permuted_inds = np.load(file(self.permuted_inds_filename))
self.searcher = LOPQSearcherLMDB(model=self.model,
lmdb_path=self.model_lmdb_filename)
def load(self):
self.model = LOPQModel.load_proto(self.model_proto_filename)