def get_current_iqr_session(self):
"""
Get the current IQR Session instance.
:rtype: smqtk.IQR.iqr_session.IqrSession
"""
with self._iqr_controller:
sid = flask.session.sid
if not self._iqr_controller.has_session_uuid(sid):
sid_work_dir = osp.join(self.work_dir, sid)
descriptor = ContentDescriptorConfiguration.new_inst(self._fd_type_str)
indexer = IndexerConfiguration.new_inst(self._idxr_type_str)
iqr_sess = IqrSession(sid_work_dir, descriptor, indexer, sid)
self._iqr_controller.add_session(iqr_sess, sid)
# If there are things already in our extension ingest, extend
# the base indexer
feat_map = \
descriptor.compute_descriptor_async(iqr_sess.extension_ds)
indexer.extend_model(feat_map)
return self._iqr_controller.get_session(sid)
def main():
import optparse
description = \
"Generate the model for the given indexer type, using features " \
"from the given feature descriptor type. We use configured valued in " \
"the smqtk_config module and from the system configuration JSON file " \
"(etc/system_config.json) unless otherwise specified by options to " \
"this script. Specific ingest used is determined by the ingest type " \
"provided (-t/--type)."
parser = bin_utils.SMQTKOptParser(description=description)
group_required = optparse.OptionGroup(parser, "Required Options")
group_optional = optparse.OptionGroup(parser, "Optional")
group_required.add_option('-d', '--data-set',
help="Data set to use for model generation.")
group_required.add_option('-c', '--content-descriptor',
help="Feature descriptor type for model and "
"feature generation.")
group_required.add_option('-i', '--indexer',
help="(Optional) Indexer type for model "
"generation.")
group_optional.add_option('--sys-json',
help="Custom system configuration JSON file to "
"use. Otherwise we use the one specified in "
"the smqtk_config module.")
group_optional.add_option('-l', '--list',
action='store_true', default=False,
help="List available ingest configurations. If "
"a valid ingest configuration has been "
"specified, we list available "
"FeatureDetector and Indexer configurations "
"available.")
group_optional.add_option('-t', '--threads', type=int, default=None,
help='Number of threads/processes to use for '
'processing. By default we use all '
'available cores/threads.')
group_optional.add_option('-v', '--verbose', action='store_true',
default=False,
help='Add debug messaged to output logging.')
parser.add_option_group(group_required)
parser.add_option_group(group_optional)
opts, args = parser.parse_args()
bin_utils.initialize_logging(logging.getLogger(),
logging.INFO - (10*opts.verbose))
log = logging.getLogger("main")
dset_label = opts.data_set
cd_label = opts.content_descriptor
idxr_label = opts.indexer
parallel = opts.threads
# Prep custom JSON configuration if one was given
if opts.sys_json:
with open(opts.sys_json) as json_file:
json_config = json.loads(jsmin(json_file.read()))
ConfigurationInterface.BASE_CONFIG = json_config['Ingests']
if opts.list:
log.info("")
log.info("Available Data Sets:")
log.info("")
for l in DataSetConfiguration.available_labels():
log.info("\t%s" % l)
log.info("")
log.info("Available ContentDescriptor types:")
log.info("")
for l in ContentDescriptorConfiguration.available_labels():
log.info("\t%s" % l)
log.info("")
log.info("Available Indexer types:")
log.info("")
for l in IndexerConfiguration.available_labels():
log.info("\t%s", l)
log.info("")
exit(0)
# Check given labels
fail = False
if dset_label and dset_label not in DataSetConfiguration.available_labels():
log.error("Given label '%s' is NOT associated to an existing "
"data set configuration!", dset_label)
fail = True
if cd_label and cd_label not in ContentDescriptorConfiguration.available_labels():
log.error("Given label '%s' is NOT associated to an existing "
"content descriptor configuration!", cd_label)
fail = True
if idxr_label and idxr_label not in IndexerConfiguration.available_labels():
log.error("Given label '%s' is NOT associated to an existing "
"indexer configuration!", idxr_label)
fail = True
if fail:
exit(1)
del fail
log.info("Loading data-set instance...")
#: :type: DataIngest or VideoIngest
dset = DataSetConfiguration.new_inst(dset_label)
log.info("Loading descriptor instance...")
#: :type: smqtk.content_description.ContentDescriptor
descriptor = ContentDescriptorConfiguration.new_inst(cd_label)
# Generate any model files needed by the chosen descriptor
descriptor.PARALLEL = parallel
descriptor.generate_model(dset)
# Don't do indexer model generation if a type was not provided
if idxr_label:
log.info("Loading indexer instance...")
#: :type: smqtk.indexing.Indexer
indexer = IndexerConfiguration.new_inst(idxr_label)
# It is not guaranteed that the feature computation method is doing
# anything in parallel, but if it is, request that it perform serially
# in order to allow multiple high-level feature computation jobs, else
# we could be overrun with threads.
descriptor.PARALLEL = 1
# Using NonDaemonicPool because content_description that might to
# parallel processing might use multiprocessing.Pool instances, too.
# Pools don't usually allow daemonic processes, so this custom top-level
# pool allows worker processes to spawn pools themselves.
fmap = descriptor.compute_descriptor_async(
dset,
parallel=parallel,
pool_type=NonDaemonicPool
)
indexer.generate_model(fmap, parallel=parallel)
