def build_external_products_index(input_dir='/Users/aub3/temptest/gtin/', output_dir="/Users/aub3/temptest/products"):
"""
Build external index for products
:param input_dir:
:param output_dir:
:return:
"""
sys.path.append(os.path.dirname(__file__))
from dvalib import external_indexed
products = external_indexed.ProductsIndex(path=output_dir)
products.prepare(input_dir)
products.build_approximate()
def main(input_dir='/Users/aub3/temptest/gtin/',
output_dir="/Users/aub3/temptest/products"):
products = external_indexed.ProductsIndex(path=output_dir)
# products.prepare(input_dir)
products.build_approximate()
data = products.data
# data = load_oxford_data()
print data.shape
pca_reduction = PCA(n_components=32)
pca_reduction.fit(data)
data = pca_reduction.transform(data)
print data.shape
P, mu = pca(data)
data = data - mu
data = np.dot(data, P)
train, test = train_test_split(data, test_size=0.2)
print train.shape, test.shape
nns = compute_all_neighbors(test, train)
m = LOPQModel(V=16, M=8)
m.fit(train, n_init=1)
print "fitted"
searcher = LOPQSearcher(m)
print "adding data"
searcher.add_data(train)
recall, _ = get_recall(searcher, test, nns)
print 'Recall (V=%d, M=%d, subquants=%d): %s' % (
m.V, m.M, m.subquantizer_clusters, str(recall))
m2 = LOPQModel(V=16, M=16, parameters=(m.Cs, None, None, None))
m2.fit(train, n_init=1)
searcher = LOPQSearcher(m2)
searcher.add_data(train)
recall, _ = get_recall(searcher, test, nns)
print 'Recall (V=%d, M=%d, subquants=%d): %s' % (
m2.V, m2.M, m2.subquantizer_clusters, str(recall))
m3 = LOPQModel(V=16,
M=8,
subquantizer_clusters=512,
parameters=(m.Cs, m.Rs, m.mus, None))
m3.fit(train, n_init=1)
searcher = LOPQSearcher(m3)
searcher.add_data(train)
recall, _ = get_recall(searcher, test, nns)
print 'Recall (V=%d, M=%d, subquants=%d): %s' % (
m3.V, m3.M, m3.subquantizer_clusters, str(recall))
def push_external_products_index(path='/Users/aub3/temptest/products/'):
sys.path.append(os.path.dirname(__file__))
from dvalib import external_indexed
products = external_indexed.ProductsIndex(path=path)
products.push_to_s3()
'retriever_task':"inception_query_by_image",
'retriever_queue':Q_RETRIEVER,
'detection_specific':False
},
'facenet':
{
'indexer_task': "perform_face_detection_indexing_by_id",
'indexer_queue': Q_FACE_DETECTOR,
'retriever_task':"facenet_query_by_image",
'retriever_queue': Q_FACE_RETRIEVER,
'detection_specific': True
},
}
EXTERNAL_DATASETS = {
'products':external_indexed.ProductsIndex(path="{}/external/{}".format(MEDIA_ROOT,'products')),
'visual_genome':external_indexed.VisualGenomeIndex(path="{}/external/{}".format(MEDIA_ROOT,'products')),
}
for create_dirname in ['queries','external']:
if not os.path.isdir("{}/{}".format(MEDIA_ROOT,create_dirname)):
try:
os.mkdir("{}/{}".format(MEDIA_ROOT,create_dirname))
except:
pass
if 'ALEX_ENABLE' in os.environ:
POST_OPERATION_TASKS['extract_frames_by_id'].append('alexnet_index_by_id')
VISUAL_INDEXES['alexnet'] = {
'indexer_task': "alexnet_index_by_id",
'indexer_queue': Q_INDEXER,
