def test_searcher_lmdb():
import shutil
data = pkl.load(open(relpath('./testdata/test_searcher_data.pkl')))
m = LOPQModel.load_proto(relpath('./testdata/random_test_model.lopq'))
lmbd_test_path = './test_lopq_lmbd'
q = np.ones(8)
# Test add_data
searcher = LOPQSearcherLMDB(m, lmbd_test_path)
searcher.add_data(data)
searcher_instance_battery(searcher, q)
# Clean up
shutil.rmtree(lmbd_test_path)
# Test add_codes
searcher = LOPQSearcherLMDB(m, lmbd_test_path)
codes = [m.predict(x) for x in data]
searcher.add_codes(codes)
searcher_instance_battery(searcher, q)
# Clean up
shutil.rmtree(lmbd_test_path)
def test_searcher_lmdb():
import shutil
data = pkl.load(open(relpath('./testdata/test_searcher_data.pkl')))
m = LOPQModel.load_proto(relpath('./testdata/random_test_model.lopq'))
lmbd_test_path = './test_lopq_lmbd'
q = np.ones(8)
# Test add_data
searcher = LOPQSearcherLMDB(m, lmbd_test_path)
searcher.add_data(data)
searcher_instance_battery(searcher, q)
# Clean up
shutil.rmtree(lmbd_test_path)
# Test add_codes
searcher = LOPQSearcherLMDB(m, lmbd_test_path)
codes = [m.predict(x) for x in data]
searcher.add_codes(codes)
searcher_instance_battery(searcher, q)
# Clean up
shutil.rmtree(lmbd_test_path)
class Clustering(object):
def __init__(self,
fnames,
n_components,
model_proto_filename,
m,
v,
sub,
test_mode=False,
dc=None):
"""
Simplify this mess haivng a seperate create vs load/init
"""
data = []
self.dc = dc
self.fnames = fnames
self.entries = []
for fname in fnames:
nmat = np.load(fname)
if nmat.ndim > 2:
nmat = nmat.squeeze()
data.append(nmat)
for e in json.load(file(fname.replace('npy', 'json'))):
self.entries.append(e)
if data:
if len(data) > 1:
self.data = np.concatenate(data)
else:
self.data = data[0]
logging.info(self.data.shape)
self.test_mode = test_mode
self.n_components = n_components
self.m = m
self.v = v
self.sub = sub
self.model = None
self.searcher = None
self.pca_reduction = None
self.P = None
self.mu = None
self.model_proto_filename = model_proto_filename
self.P_filename = model_proto_filename.replace('.proto', '.P.npy')
self.mu_filename = model_proto_filename.replace('.proto', '.mu.npy')
self.pca_filename = model_proto_filename.replace('.proto', '.pca.pkl')
self.model_lmdb_filename = model_proto_filename.replace(
'.proto', '_lmdb')
self.permuted_inds_filename = model_proto_filename.replace(
'.proto', '.permuted_inds.pkl')
self.permuted_inds = None
def pca(self):
"""
A simple PCA implementation that demonstrates how eigenvalue allocation
is used to permute dimensions in order to balance the variance across
subvectors. There are plenty of PCA implementations elsewhere. What is
important is that the eigenvalues can be used to compute a variance-balancing
dimension permutation.
"""
count, D = self.data.shape
mu = self.data.sum(axis=0) / float(count)
summed_covar = reduce(lambda acc, x: acc + np.outer(x, x), self.data,
np.zeros((D, D)))
A = summed_covar / (count - 1) - np.outer(mu, mu)
eigenvalues, P = np.linalg.eigh(A)
self.permuted_inds = eigenvalue_allocation(2, eigenvalues)
P = P[:, self.permuted_inds]
return P, mu
def cluster(self):
self.pca_reduction = PCA(n_components=self.n_components)
self.pca_reduction.fit(self.data)
self.data = self.pca_reduction.transform(self.data)
self.P, self.mu = self.pca()
self.data = self.data - self.mu
self.data = np.dot(self.data, self.P)
train, test = train_test_split(self.data, test_size=0.2)
self.model = LOPQModel(V=self.v,
M=self.m,
subquantizer_clusters=self.sub)
self.model.fit(train, n_init=1)
for i, e in enumerate(
self.entries): # avoid doing this twice again in searcher
r = self.model.predict(self.data[i])
e['coarse'] = r.coarse
e['fine'] = r.fine
e['index'] = i
self.searcher = LOPQSearcherLMDB(self.model, self.model_lmdb_filename)
if self.test_mode:
self.searcher.add_data(train)
nns = compute_all_neighbors(test, train)
recall, _ = get_recall(self.searcher, test, nns)
print 'Recall (V=%d, M=%d, subquants=%d): %s' % (
self.model.V, self.model.M, self.model.subquantizer_clusters,
str(recall))
else:
self.searcher.add_data(self.data)
def find(self):
i, selected = random.choice([k for k in enumerate(self.entries)])
print selected
for k in self.searcher.get_result_quota(self.data[i], 10):
print k
def save(self):
self.model.export_proto(self.model_proto_filename)
with open(self.pca_filename, 'w') as out:
pickle.dump(self.pca_reduction, out)
with open(self.P_filename, 'w') as out:
np.save(out, self.P)
with open(self.mu_filename, 'w') as out:
np.save(out, self.mu)
with open(self.permuted_inds_filename, 'w') as out:
pickle.dump(self.permuted_inds, out)
self.searcher.env.close()
def load(self):
self.model = LOPQModel.load_proto(self.model_proto_filename)
self.pca_reduction = pickle.load(file(self.pca_filename))
self.P = np.load(file(self.P_filename))
self.mu = np.load(file(self.mu_filename))
self.permuted_inds = np.load(file(self.permuted_inds_filename))
self.searcher = LOPQSearcherLMDB(model=self.model,
lmdb_path=self.model_lmdb_filename)
def apply(self, vector, count=None):
vector = np.dot((self.pca_reduction.transform(vector) - self.mu),
self.P).transpose().squeeze()
codes = self.model.predict(vector)
if count:
results = self.searcher.search(vector, quota=count)
else:
results = None
return codes.coarse, codes.fine, results
class Clustering(object):
def __init__(self,fnames,n_components,model_proto_filename,m,v,sub,test_mode=False,dc=None):
"""
Simplify this mess haivng a seperate create vs load/init
"""
data = []
self.dc = dc
self.fnames = fnames
self.entries = []
for fname in fnames:
nmat = np.load(fname)
if nmat.ndim > 2:
nmat = nmat.squeeze()
data.append(nmat)
for e in json.load(file(fname.replace('npy','json'))):
self.entries.append(e)
if data:
if len(data) > 1:
self.data = np.concatenate(data)
else:
self.data = data[0]
logging.info(self.data.shape)
self.test_mode = test_mode
self.n_components = n_components
self.m = m
self.v = v
self.sub = sub
self.model = None
self.searcher = None
self.pca_reduction = None
self.P = None
self.mu = None
self.model_proto_filename = model_proto_filename
self.P_filename = model_proto_filename.replace('.proto','.P.npy')
self.mu_filename = model_proto_filename.replace('.proto','.mu.npy')
self.pca_filename = model_proto_filename.replace('.proto', '.pca.pkl')
self.model_lmdb_filename = model_proto_filename.replace('.proto', '_lmdb')
self.permuted_inds_filename = model_proto_filename.replace('.proto', '.permuted_inds.pkl')
self.permuted_inds = None
def pca(self):
"""
A simple PCA implementation that demonstrates how eigenvalue allocation
is used to permute dimensions in order to balance the variance across
subvectors. There are plenty of PCA implementations elsewhere. What is
important is that the eigenvalues can be used to compute a variance-balancing
dimension permutation.
"""
count, D = self.data.shape
mu = self.data.sum(axis=0) / float(count)
summed_covar = reduce(lambda acc, x: acc + np.outer(x, x), self.data, np.zeros((D, D)))
A = summed_covar / (count - 1) - np.outer(mu, mu)
eigenvalues, P = np.linalg.eigh(A)
self.permuted_inds = eigenvalue_allocation(2, eigenvalues)
P = P[:, self.permuted_inds]
return P, mu
def cluster(self):
self.pca_reduction = PCA(n_components=self.n_components)
self.pca_reduction.fit(self.data)
self.data = self.pca_reduction.transform(self.data)
self.P, self.mu = self.pca()
self.data = self.data - self.mu
self.data = np.dot(self.data,self.P)
train, test = train_test_split(self.data, test_size=0.2)
self.model = LOPQModel(V=self.v, M=self.m, subquantizer_clusters=self.sub)
self.model.fit(train, n_init=1)
for i,e in enumerate(self.entries): # avoid doing this twice again in searcher
r = self.model.predict(self.data[i])
e['coarse'] = r.coarse
e['fine'] = r.fine
e['index'] = i
self.searcher = LOPQSearcherLMDB(self.model,self.model_lmdb_filename)
if self.test_mode:
self.searcher.add_data(train)
nns = compute_all_neighbors(test, train)
recall, _ = get_recall(self.searcher, test, nns)
print 'Recall (V=%d, M=%d, subquants=%d): %s' % (self.model.V, self.model.M, self.model.subquantizer_clusters, str(recall))
else:
self.searcher.add_data(self.data)
def find(self):
i,selected = random.choice([k for k in enumerate(self.entries)])
print selected
for k in self.searcher.get_result_quota(self.data[i],10):
print k
def save(self):
self.model.export_proto(self.model_proto_filename)
with open(self.pca_filename,'w') as out:
pickle.dump(self.pca_reduction,out)
with open(self.P_filename, 'w') as out:
np.save(out,self.P)
with open(self.mu_filename, 'w') as out:
np.save(out,self.mu)
with open(self.permuted_inds_filename, 'w') as out:
pickle.dump(self.permuted_inds,out)
self.searcher.env.close()
def load(self):
self.model = LOPQModel.load_proto(self.model_proto_filename)
self.pca_reduction = pickle.load(file(self.pca_filename))
self.P = np.load(file(self.P_filename))
self.mu = np.load(file(self.mu_filename))
self.permuted_inds = np.load(file(self.permuted_inds_filename))
self.searcher = LOPQSearcherLMDB(model=self.model,lmdb_path=self.model_lmdb_filename)
def apply(self,vector,count=None):
vector = np.dot((self.pca_reduction.transform(vector) - self.mu), self.P).transpose().squeeze()
codes = self.model.predict(vector)
if count:
results = self.searcher.search(vector,quota=count)
else:
results = None
return codes.coarse,codes.fine,results
class LOPQRetriever(BaseRetriever):
def __init__(self, name, args):
data = []
self.name = name
self.fnames = args.get('fnames', [])
self.entries = []
for fname in self.fnames:
nmat = np.load(fname)
if nmat.ndim > 2:
logging.info("squeezing shape {} with dimensions {}".format(
nmat.shape, nmat.ndim))
nmat = nmat.squeeze(axis=1)
elif nmat.ndim == 1:
logging.info("expanding shape {} with dimensions {}".format(
nmat.shape, nmat.ndim))
nmat = np.expand_dims(nmat, axis=0)
else:
logging.info(
"keeping same shape {} with dimensions {}".format(
nmat.shape, nmat.ndim))
data.append(nmat)
for e in json.load(file(fname.replace('npy', 'json'))):
self.entries.append(e)
if data:
if len(data) > 1:
self.data = np.concatenate(data)
else:
self.data = data[0]
logging.info(self.data.shape)
self.test_mode = args.get('test_mode', False)
self.n_components = int(args['components'])
self.m = int(args['m'])
self.v = int(args['v'])
self.sub = int(args['sub'])
self.model = None
self.searcher = None
self.pca_reduction = None
self.P = None
self.mu = None
self.model_proto_filename = args['proto_filename']
self.P_filename = args['proto_filename'].replace('.proto', '.P.npy')
self.mu_filename = args['proto_filename'].replace('.proto', '.mu.npy')
self.pca_filename = args['proto_filename'].replace(
'.proto', '.pca.pkl')
self.model_lmdb_filename = args['proto_filename'].replace(
'.proto', '_lmdb')
self.permuted_inds_filename = args['proto_filename'].replace(
'.proto', '.permuted_inds.pkl')
self.permuted_inds = None
def pca(self):
"""
A simple PCA implementation that demonstrates how eigenvalue allocation
is used to permute dimensions in order to balance the variance across
subvectors. There are plenty of PCA implementations elsewhere. What is
important is that the eigenvalues can be used to compute a variance-balancing
dimension permutation.
"""
count, D = self.data.shape
mu = self.data.sum(axis=0) / float(count)
summed_covar = reduce(lambda acc, x: acc + np.outer(x, x), self.data,
np.zeros((D, D)))
A = summed_covar / (count - 1) - np.outer(mu, mu)
eigenvalues, P = np.linalg.eigh(A)
self.permuted_inds = eigenvalue_allocation(2, eigenvalues)
P = P[:, self.permuted_inds]
return P, mu
def cluster(self):
self.pca_reduction = PCA(n_components=self.n_components)
self.pca_reduction.fit(self.data)
self.data = self.pca_reduction.transform(self.data)
self.P, self.mu = self.pca()
self.data = self.data - self.mu
self.data = np.dot(self.data, self.P)
# train, test = train_test_split(self.data, test_size=0.2)
self.model = LOPQModel(V=self.v,
M=self.m,
subquantizer_clusters=self.sub)
self.model.fit(self.data, n_init=1) # replace self.data by train
for i, e in enumerate(
self.entries): # avoid doing this twice again in searcher
r = self.model.predict(self.data[i])
e['coarse'] = r.coarse
e['fine'] = r.fine
e['index'] = i
self.searcher = LOPQSearcherLMDB(self.model, self.model_lmdb_filename)
# if self.test_mode:
# self.searcher.add_data(train)
# nns = compute_all_neighbors(test, train)
# recall, _ = get_recall(self.searcher, test, nns)
# print 'Recall (V=%d, M=%d, subquants=%d): %s' % (self.model.V, self.model.M, self.model.subquantizer_clusters, str(recall))
# else:
self.searcher.add_data(self.data)
def find(self):
i, selected = random.choice([k for k in enumerate(self.entries)])
print selected
for k in self.searcher.get_result_quota(self.data[i], 10):
print k
def save(self):
with open(self.model_proto_filename, 'w') as f:
self.model.export_proto(f)
with open(self.pca_filename, 'w') as out:
pickle.dump(self.pca_reduction, out)
with open(self.P_filename, 'w') as out:
np.save(out, self.P)
with open(self.mu_filename, 'w') as out:
np.save(out, self.mu)
with open(self.permuted_inds_filename, 'w') as out:
pickle.dump(self.permuted_inds, out)
self.searcher.env.close()
def load(self):
self.model = LOPQModel.load_proto(self.model_proto_filename)
self.pca_reduction = pickle.load(file(self.pca_filename))
self.P = np.load(file(self.P_filename))
self.mu = np.load(file(self.mu_filename))
self.permuted_inds = np.load(file(self.permuted_inds_filename))
self.searcher = LOPQSearcherLMDB(model=self.model,
lmdb_path=self.model_lmdb_filename)
def apply(self, vector, count=None):
vector = np.dot((self.pca_reduction.transform(vector) - self.mu),
self.P).transpose().squeeze()
codes = self.model.predict(vector)
if count:
results = self.searcher.search(vector, quota=count)
else:
results = None
return codes.coarse, codes.fine, results
def nearest(self, vector=None, n=12, retriever_pk=None, entry_getter=None):
results = []
coarse, fine, results_indexes = self.apply(vector, n)
for i, k in enumerate(results_indexes[0]):
e = entry_getter(k.id, retriever_pk)
if e.detection_id:
results.append({
'rank': i + 1,
'dist': i,
'detection_primary_key': e.detection_id,
'frame_index': e.frame.frame_index,
'frame_primary_key': e.frame_id,
'video_primary_key': e.video_id,
'type': 'detection',
})
else:
results.append({
'rank': i + 1,
'dist': i,
'frame_index': e.frame.frame_index,
'frame_primary_key': e.frame_id,
'video_primary_key': e.video_id,
'type': 'frame',
})
return results
class LOPQRetriever(BaseRetriever):
def __init__(self, name, proto_filename, args, test_mode=False):
super(BaseRetriever, self).__init__()
self.name = name
self.proto_filename = proto_filename
self.entries = []
self.test_mode = test_mode
self.n_components = int(args['components'])
self.m = int(args['m'])
self.v = int(args['v'])
self.sub = int(args['sub'])
self.model = None
self.searcher = None
self.pca_reduction = None
self.P = None
self.mu = None
self.permuted_inds = None
self.model_proto_filename = proto_filename
self.P_filename = proto_filename.replace('.proto', '.P.npy')
self.entries_filename = proto_filename.replace('.proto', '.json')
self.mu_filename = proto_filename.replace('.proto', '.mu.npy')
self.pca_filename = proto_filename.replace('.proto', '.pca.pkl')
self.model_lmdb_filename = proto_filename.replace('.proto', '_lmdb')
self.permuted_inds_filename = proto_filename.replace(
'.proto', '.permuted_inds.pkl')
def pca(self):
"""
A simple PCA implementation that demonstrates how eigenvalue allocation
is used to permute dimensions in order to balance the variance across
sub vectors. There are plenty of PCA implementations elsewhere. What is
important is that the eigenvalues can be used to compute a variance-balancing
dimension permutation.
"""
count, D = self.data.shape
mu = self.data.sum(axis=0) / float(count)
summed_covar = reduce(lambda acc, x: acc + np.outer(x, x), self.data,
np.zeros((D, D)))
A = summed_covar / (count - 1) - np.outer(mu, mu)
eigenvalues, P = np.linalg.eigh(A)
self.permuted_inds = eigenvalue_allocation(2, eigenvalues)
P = P[:, self.permuted_inds]
return P, mu
def cluster(self):
self.data = self.index
self.pca_reduction = PCA(n_components=self.n_components)
self.pca_reduction.fit(self.data)
self.data = self.pca_reduction.transform(self.data)
self.P, self.mu = self.pca()
self.data = self.data - self.mu
self.data = np.dot(self.data, self.P)
self.model = LOPQModel(V=self.v,
M=self.m,
subquantizer_clusters=self.sub)
self.model.fit(self.data, n_init=1) # replace self.data by train
for i, e in enumerate(
self.entries): # avoid doing this twice again in searcher
r = self.model.predict(self.data[i])
e['coarse'] = r.coarse
e['fine'] = r.fine
e['index'] = i
self.searcher = LOPQSearcherLMDB(self.model, self.model_lmdb_filename)
self.searcher.add_data(self.data)
# cluster_codes = []
# for e in c.entries:
# cc.video_id = e['video_primary_key']
# if 'detection_primary_key' in e:
# cc.detection_id = e['detection_primary_key']
# cc.frame_id = Region.objects.get(pk=cc.detection_id).frame_id
# else:
# cc.frame_id = e['frame_primary_key']
# cc.clusters = dc
# cc.coarse = e['coarse']
# cc.fine = e['fine']
# cc.coarse_text = " ".join(map(str, e['coarse']))
# cc.fine_text = " ".join(map(str, e['fine']))
# cc.searcher_index = e['index']
# cluster_codes.append(cc)
def find(self):
i, selected = random.choice([k for k in enumerate(self.entries)])
print selected
for k in self.searcher.get_result_quota(self.data[i], 10):
print k
def save(self):
with open(self.model_proto_filename, 'w') as f:
self.model.export_proto(f)
with open(self.pca_filename, 'w') as out:
pickle.dump(self.pca_reduction, out)
with open(self.P_filename, 'w') as out:
np.save(out, self.P)
with open(self.mu_filename, 'w') as out:
np.save(out, self.mu)
with open(self.entries_filename, 'w') as out:
json.dump(out, self.entries)
with open(self.permuted_inds_filename, 'w') as out:
pickle.dump(self.permuted_inds, out)
self.searcher.env.close()
def load(self):
self.model = LOPQModel.load_proto(self.model_proto_filename)
self.pca_reduction = pickle.load(file(self.pca_filename))
self.P = np.load(file(self.P_filename))
self.mu = np.load(file(self.mu_filename))
self.permuted_inds = np.load(file(self.permuted_inds_filename))
self.searcher = LOPQSearcherLMDB(model=self.model,
lmdb_path=self.model_lmdb_filename)
def apply(self, vector, count=None):
vector = np.dot((self.pca_reduction.transform(vector) - self.mu),
self.P).transpose().squeeze()
codes = self.model.predict(vector)
if count:
results = self.searcher.search(vector, quota=count)
else:
results = None
return codes.coarse, codes.fine, results
def nearest(self, vector=None, n=12, retriever_pk=None, entry_getter=None):
results = []
coarse, fine, results_indexes = self.apply(vector, n)
for i, k in enumerate(results_indexes[0]):
e = entry_getter(k.id, retriever_pk)
if e.detection_id:
results.append({
'rank': i + 1,
'dist': i,
'detection_primary_key': e.detection_id,
'frame_index': e.frame.frame_index,
'frame_primary_key': e.frame_id,
'video_primary_key': e.video_id,
'type': 'detection',
})
else:
results.append({
'rank': i + 1,
'dist': i,
'frame_index': e.frame.frame_index,
'frame_primary_key': e.frame_id,
'video_primary_key': e.video_id,
'type': 'frame',
})
return results