def build_external_products_index(input_dir='/Users/aub3/temptest/gtin/', output_dir="/Users/aub3/temptest/products"): """ Build external index for products :param input_dir: :param output_dir: :return: """ sys.path.append(os.path.dirname(__file__)) from dvalib import external_indexed products = external_indexed.ProductsIndex(path=output_dir) products.prepare(input_dir) products.build_approximate()
def main(input_dir='/Users/aub3/temptest/gtin/', output_dir="/Users/aub3/temptest/products"): products = external_indexed.ProductsIndex(path=output_dir) # products.prepare(input_dir) products.build_approximate() data = products.data # data = load_oxford_data() print data.shape pca_reduction = PCA(n_components=32) pca_reduction.fit(data) data = pca_reduction.transform(data) print data.shape P, mu = pca(data) data = data - mu data = np.dot(data, P) train, test = train_test_split(data, test_size=0.2) print train.shape, test.shape nns = compute_all_neighbors(test, train) m = LOPQModel(V=16, M=8) m.fit(train, n_init=1) print "fitted" searcher = LOPQSearcher(m) print "adding data" searcher.add_data(train) recall, _ = get_recall(searcher, test, nns) print 'Recall (V=%d, M=%d, subquants=%d): %s' % ( m.V, m.M, m.subquantizer_clusters, str(recall)) m2 = LOPQModel(V=16, M=16, parameters=(m.Cs, None, None, None)) m2.fit(train, n_init=1) searcher = LOPQSearcher(m2) searcher.add_data(train) recall, _ = get_recall(searcher, test, nns) print 'Recall (V=%d, M=%d, subquants=%d): %s' % ( m2.V, m2.M, m2.subquantizer_clusters, str(recall)) m3 = LOPQModel(V=16, M=8, subquantizer_clusters=512, parameters=(m.Cs, m.Rs, m.mus, None)) m3.fit(train, n_init=1) searcher = LOPQSearcher(m3) searcher.add_data(train) recall, _ = get_recall(searcher, test, nns) print 'Recall (V=%d, M=%d, subquants=%d): %s' % ( m3.V, m3.M, m3.subquantizer_clusters, str(recall))
def push_external_products_index(path='/Users/aub3/temptest/products/'): sys.path.append(os.path.dirname(__file__)) from dvalib import external_indexed products = external_indexed.ProductsIndex(path=path) products.push_to_s3()
'retriever_task':"inception_query_by_image", 'retriever_queue':Q_RETRIEVER, 'detection_specific':False }, 'facenet': { 'indexer_task': "perform_face_detection_indexing_by_id", 'indexer_queue': Q_FACE_DETECTOR, 'retriever_task':"facenet_query_by_image", 'retriever_queue': Q_FACE_RETRIEVER, 'detection_specific': True }, } EXTERNAL_DATASETS = { 'products':external_indexed.ProductsIndex(path="{}/external/{}".format(MEDIA_ROOT,'products')), 'visual_genome':external_indexed.VisualGenomeIndex(path="{}/external/{}".format(MEDIA_ROOT,'products')), } for create_dirname in ['queries','external']: if not os.path.isdir("{}/{}".format(MEDIA_ROOT,create_dirname)): try: os.mkdir("{}/{}".format(MEDIA_ROOT,create_dirname)) except: pass if 'ALEX_ENABLE' in os.environ: POST_OPERATION_TASKS['extract_frames_by_id'].append('alexnet_index_by_id') VISUAL_INDEXES['alexnet'] = { 'indexer_task': "alexnet_index_by_id", 'indexer_queue': Q_INDEXER,