def add_descriptors_smallcodes():
log = logging.getLogger(__name__)
log.info("Loading descriptor UUIDs")
with open(UUIDS_FILEPATH) as f:
descriptor_uuids = [l.strip() for l in f]
log.info("Loading ITQ components")
r = np.load(ITQ_ROTATION)
mv = np.load(ITQ_MEAN_VEC)
log.info("Making small-codes")
sc_d_pairs = async_compute_smallcodes(
r, mv, make_elements_from_uuids(descriptor_uuids))
log.info("Loading ITQ model")
itq_index = load_algo()
log.info("Adding small codes")
itq_index._code_index.add_many_descriptors(sc_d_pairs)
return descriptor_uuids, itq_index
def add_descriptors_smallcodes():
log = logging.getLogger(__name__)
log.info("Loading descriptor UUIDs")
with open(UUIDS_FILEPATH) as f:
descriptor_uuids = cPickle.load(f)
log.info("Loading ITQ components")
r = np.load("/data/shared/memex/ht_image_cnn/itq_model/16-bit/rotation.npy")
mv = np.load("/data/shared/memex/ht_image_cnn/itq_model/16-bit/mean_vec.npy")
log.info("Making small-codes")
sc_d_pairs = async_compute_smallcodes(
r, mv, make_elements_from_uuids(descriptor_uuids)
)
log.info("Loading ITQ model")
itq_index = load_algo()
log.info("Adding small codes")
itq_index._code_index.add_many_descriptors(sc_d_pairs)
return descriptor_uuids, itq_index
def add_descriptors_smallcodes():
log = logging.getLogger(__name__)
log.info("Loading descriptor UUIDs")
with open(UUIDS_FILEPATH) as f:
descriptor_uuids = [l.strip() for l in f]
log.info("Loading ITQ components")
r = np.load(ITQ_ROTATION)
mv = np.load(ITQ_MEAN_VEC)
log.info("Making small-codes")
sc_d_pairs = async_compute_smallcodes(
r, mv, make_elements_from_uuids(descriptor_uuids)
)
log.info("Loading ITQ model")
itq_index = load_algo()
log.info("Adding small codes")
itq_index._code_index.add_many_descriptors(sc_d_pairs)
return descriptor_uuids, itq_index
#
# def add_non_nan_uuid(uuid):
# d = factory.new_descriptor(d_type_str, uuid)
# if d.vector().sum() > 0:
# return uuid
# return None
#
# import multiprocessing
# p = multiprocessing.Pool()
# non_nan_uuids = \
# p.map(add_non_nan_uuid,
# (l.strip() for l in open('descriptor_uuids.txt')))
d_elements = []
with open("descriptor_uuids.train.txt") as f:
for uuid in (l.strip() for l in f):
d_elements.append(factory(d_type_str, uuid))
# log.info("Sorting descriptors by UUID")
# d_elements.sort(key=lambda e: e.uuid())
log.info("Loading ITQ index algo")
itq_index = load_algo()
# Assuming ITQ functor, which needs fitting
itq_index.lsh_functor.fit(d_elements)
log.info("Building index")
# includes adding to adding to configured descriptor and hash indexes
itq_index.build_index(d_elements)
from smqtk.utils.bin_utils import logging, initialize_logging
if not logging.getLogger().handlers:
initialize_logging(logging.getLogger(), logging.DEBUG)
log = logging.getLogger(__name__)
log.info("Loading descriptor elements")
import cPickle
import json
from smqtk.representation.descriptor_element.postgres_element import PostgresDescriptorElement
d_config = json.load(open('/data/kitware/smqtk/image_cache_cnn_compute/psql_descriptor_config.localhost.json'))
d_type_str = open("/data/kitware/smqtk/image_cache_cnn_compute/descriptor_type_name.txt").read().strip()
d_elements = []
with open("/data/kitware/smqtk/image_cache_cnn_compute/descriptor_uuid_set.1m_train_sample.pickle") as f:
for uuid in cPickle.load(f):
d_elements.append(
PostgresDescriptorElement.from_config(d_config, d_type_str, uuid)
)
log.info("Loading ITQ index algo")
from load_algo import load_algo
itq_index = load_algo()
log.info("Building index")
itq_index.build_index(d_elements)
