def from_config(cls, config_dict, merge_default=True): """ Instantiate a new instance of this class given the configuration JSON-compliant dictionary encapsulating initialization arguments. This method should not be called via super unless and instance of the class is desired. :param config_dict: JSON compliant dictionary encapsulating a configuration. :type config_dict: dict :param merge_default: Merge the given configuration on top of the default provided by ``get_default_config``. :type merge_default: bool :return: Constructed instance from the provided config. :rtype: LSHNearestNeighborIndex """ if merge_default: cfg = cls.get_default_config() merge_dict(cfg, config_dict) else: cfg = config_dict cfg['descriptor_set'] = plugin.from_plugin_config( cfg['descriptor_set'], get_descriptor_index_impls() ) cfg['uid2idx_kvs'] = plugin.from_plugin_config( cfg['uid2idx_kvs'], get_key_value_store_impls() ) cfg['idx2uid_kvs'] = plugin.from_plugin_config( cfg['idx2uid_kvs'], get_key_value_store_impls() ) if (cfg['index_element'] and cfg['index_element']['type']): index_element = plugin.from_plugin_config( cfg['index_element'], get_data_element_impls()) cfg['index_element'] = index_element else: cfg['index_element'] = None if (cfg['index_param_element'] and cfg['index_param_element']['type']): index_param_element = plugin.from_plugin_config( cfg['index_param_element'], get_data_element_impls()) cfg['index_param_element'] = index_param_element else: cfg['index_param_element'] = None return super(FaissNearestNeighborsIndex, cls).from_config(cfg, False)
def from_config(cls, config_dict, merge_default=True): """ Instantiate a new instance of this class given the configuration JSON-compliant dictionary encapsulating initialization arguments. This method should not be called via super unless an instance of the class is desired. :param config_dict: JSON compliant dictionary encapsulating a configuration. :type config_dict: dict :param merge_default: Merge the given configuration on top of the default provided by ``get_default_config``. :type merge_default: bool :return: Constructed instance from the provided config. :rtype: SkLearnBallTreeHashIndex """ if merge_default: config_dict = merge_dict(cls.get_default_config(), config_dict) # Parse ``cache_element`` configuration if set. cache_element = None if config_dict['cache_element'] and \ config_dict['cache_element']['type']: cache_element = \ plugin.from_plugin_config(config_dict['cache_element'], get_data_element_impls()) config_dict['cache_element'] = cache_element return super(SkLearnBallTreeHashIndex, cls).from_config(config_dict, False)
def from_config(cls, config_dict, merge_default=True): """ Instantiate a new instance of this class given the configuration JSON-compliant dictionary encapsulating initialization arguments. :param config_dict: JSON compliant dictionary encapsulating a configuration. :type config_dict: dict :param merge_default: Merge the given configuration on top of the default provided by ``get_default_config``. :type merge_default: bool :return: Constructed instance from the provided config. :rtype: MemoryDescriptorIndex """ if merge_default: config_dict = merge_dict(cls.get_default_config(), config_dict) # Optionally construct cache element from sub-config. if config_dict['cache_element'] \ and config_dict['cache_element']['type']: e = plugin.from_plugin_config(config_dict['cache_element'], get_data_element_impls()) config_dict['cache_element'] = e else: config_dict['cache_element'] = None return super(MemoryDescriptorIndex, cls).from_config(config_dict, False)
def from_config(cls, c, merge_default=True): """ Instantiate a new instance of this class given the configuration JSON-compliant dictionary encapsulating initialization arguments. This method should not be called via super unless an instance of the class is desired. :param c: JSON compliant dictionary encapsulating a configuration. :type c: dict :param merge_default: Merge the given configuration on top of the default provided by ``get_default_config``. :type merge_default: bool :return: Constructed instance from the provided config. :rtype: DataMemorySet """ if merge_default: c = merge_dict(cls.get_default_config(), c) cache_element = None if c['cache_element'] and c['cache_element']['type']: cache_element = plugin.from_plugin_config(c['cache_element'], get_data_element_impls()) c['cache_element'] = cache_element return super(DataMemorySet, cls).from_config(c, False)
def from_config(cls, config_dict, merge_default=True): """ Instantiate a new instance of this class given the configuration JSON-compliant dictionary encapsulating initialization arguments. This method should not be called via super unless an instance of the class is desired. :param config_dict: JSON compliant dictionary encapsulating a configuration. :type config_dict: dict :param merge_default: Merge the given configuration on top of the default provided by ``get_default_config``. :type merge_default: bool :return: Constructed instance from the provided config. :rtype: LinearHashIndex """ if merge_default: config_dict = merge_dict(cls.get_default_config(), config_dict) cache_element = None if config_dict['cache_element'] \ and config_dict['cache_element']['type']: cache_element = \ plugin.from_plugin_config(config_dict['cache_element'], get_data_element_impls()) config_dict['cache_element'] = cache_element return super(LinearHashIndex, cls).from_config(config_dict, False)
def get_default_config(cls): """ Generate and return a default configuration dictionary for this class. This will be primarily used for generating what the configuration dictionary would look like for this class without instantiating it. By default, we observe what this class's constructor takes as arguments, turning those argument names into configuration dictionary keys. If any of those arguments have defaults, we will add those values into the configuration dictionary appropriately. The dictionary returned should only contain JSON compliant value types. It is not be guaranteed that the configuration dictionary returned from this method is valid for construction of an instance of this class. :return: Default configuration dictionary for the class. :rtype: dict """ default = super(FaissNearestNeighborsIndex, cls).get_default_config() data_element_default_config = plugin.make_config( get_data_element_impls()) default['index_element'] = data_element_default_config default['index_param_element'] = deepcopy(data_element_default_config) di_default = plugin.make_config(get_descriptor_index_impls()) default['descriptor_set'] = di_default kvs_default = plugin.make_config(get_key_value_store_impls()) default['idx2uid_kvs'] = kvs_default default['uid2idx_kvs'] = deepcopy(kvs_default) return default
def get_default_config(cls): default = super(ItqFunctor, cls).get_default_config() # Cache element parameters need to be split out into sub-configurations data_element_default_config = \ plugin.make_config(get_data_element_impls()) default['mean_vec_cache'] = data_element_default_config # Need to deepcopy source to prevent modifications on one sub-config # from reflecting in the other. default['rotation_cache'] = deepcopy(data_element_default_config) return default
def get_default_config(cls): """ Generate and return a default configuration dictionary for this class. This will be primarily used for generating what the configuration dictionary would look like for this class without instantiating it. By default, we observe what this class's constructor takes as arguments, turning those argument names into configuration dictionary keys. If any of those arguments have defaults, we will add those values into the configuration dictionary appropriately. The dictionary returned should only contain JSON compliant value types. It is not be guaranteed that the configuration dictionary returned from this method is valid for construction of an instance of this class. :return: Default configuration dictionary for the class. :rtype: dict """ c = super(LinearHashIndex, cls).get_default_config() c['cache_element'] = plugin.make_config(get_data_element_impls()) return c
def from_config(cls, config_dict, merge_default=True): """ Instantiate a new instance of this class given the JSON-compliant configuration dictionary encapsulating initialization arguments. :param config_dict: JSON compliant dictionary encapsulating a configuration. :type config_dict: dict :param merge_default: Merge the given configuration on top of the default provided by ``get_default_config``. :type merge_default: bool :return: Constructed instance from the provided config. :rtype: ItqFunctor """ if merge_default: config_dict = merge_dict(cls.get_default_config(), config_dict) data_element_impls = get_data_element_impls() # Mean vector cache element. mean_vec_cache = None if config_dict['mean_vec_cache'] and \ config_dict['mean_vec_cache']['type']: mean_vec_cache = plugin.from_plugin_config( config_dict['mean_vec_cache'], data_element_impls) config_dict['mean_vec_cache'] = mean_vec_cache # Rotation matrix cache element. rotation_cache = None if config_dict['rotation_cache'] and \ config_dict['rotation_cache']['type']: rotation_cache = plugin.from_plugin_config( config_dict['rotation_cache'], data_element_impls) config_dict['rotation_cache'] = rotation_cache return super(ItqFunctor, cls).from_config(config_dict, False)
plugin.from_plugin_config(nn_index_config, algorithms.get_nn_index_impls) #: :type: algorithms.RelevancyIndex rel_index = \ plugin.from_plugin_config(rel_index_config, algorithms.get_relevancy_index_impls) # # Build models # # Perform the actual building or the models. # # Add data files to DataSet DataFileElement = representation.get_data_element_impls()["DataFileElement"] data_set.add_data(*[DataFileElement(fp) for fp in glob.iglob(input_image_file_glob)]) # Generate a mode if the generator defines a known generation method. if hasattr(descriptor_generator, "generate_model"): descriptor_generator.generate_model(data_set) # Add other if-else cases for other known implementation-specific generation # methods stubs # Generate descriptors of data for building NN index. data2descriptor = descriptor_generator.compute_descriptor_async( data_set, descriptor_elem_factory ) try:
def main(): parser = cli_parser() args = parser.parse_args() # # Setup logging # if not logging.getLogger().handlers: if args.verbose: bin_utils.initialize_logging(logging.getLogger(), logging.DEBUG) else: bin_utils.initialize_logging(logging.getLogger(), logging.INFO) log = logging.getLogger("smqtk.scripts.iqr_app_model_generation") search_app_config = json.loads(jsmin.jsmin(open(args.config).read())) # # Input parameters # # The following dictionaries are JSON configurations that are used to # configure the various data structures and algorithms needed for the IQR demo # application. Values here can be changed to suit your specific data and # algorithm needs. # # See algorithm implementation doc-strings for more information on configuration # parameters (see implementation class ``__init__`` method). # # base actions on a specific IQR tab configuration (choose index here) if args.tab < 0 or args.tab > (len(search_app_config["iqr_tabs"]) - 1): log.error("Invalid tab number provided.") exit(1) search_app_iqr_config = search_app_config["iqr_tabs"][args.tab] # Configure DataSet implementation and parameters data_set_config = search_app_iqr_config['data_set'] # Configure DescriptorGenerator algorithm implementation, parameters and # persistant model component locations (if implementation has any). descriptor_generator_config = search_app_iqr_config['descr_generator'] # Configure NearestNeighborIndex algorithm implementation, parameters and # persistant model component locations (if implementation has any). nn_index_config = search_app_iqr_config['nn_index'] # Configure RelevancyIndex algorithm implementation, parameters and # persistant model component locations (if implementation has any). # # The LibSvmHikRelevancyIndex implementation doesn't actually build a persistant # model (or doesn't have to that is), but we're leaving this block here in # anticipation of other potential implementations in the future. # rel_index_config = search_app_iqr_config['rel_index_config'] # Configure DescriptorElementFactory instance, which defines what implementation # of DescriptorElement to use for storing generated descriptor vectors below. descriptor_elem_factory_config = search_app_iqr_config[ 'descriptor_factory'] # # Initialize data/algorithms # # Constructing appropriate data structures and algorithms, needed for the IQR # demo application, in preparation for model training. # descriptor_elem_factory = \ representation.DescriptorElementFactory \ .from_config(descriptor_elem_factory_config) #: :type: representation.DataSet data_set = \ plugin.from_plugin_config(data_set_config, representation.get_data_set_impls()) #: :type: algorithms.DescriptorGenerator descriptor_generator = \ plugin.from_plugin_config(descriptor_generator_config, algorithms.get_descriptor_generator_impls()) #: :type: algorithms.NearestNeighborsIndex nn_index = \ plugin.from_plugin_config(nn_index_config, algorithms.get_nn_index_impls()) #: :type: algorithms.RelevancyIndex rel_index = \ plugin.from_plugin_config(rel_index_config, algorithms.get_relevancy_index_impls()) # # Build models # # Perform the actual building of the models. # # Add data files to DataSet DataFileElement = representation.get_data_element_impls( )["DataFileElement"] for fp in args.input_files: fp = osp.expanduser(fp) if osp.isfile(fp): data_set.add_data(DataFileElement(fp)) else: log.debug("Expanding glob: %s" % fp) for g in glob.iglob(fp): data_set.add_data(DataFileElement(g)) # Generate a mode if the generator defines a known generation method. if hasattr(descriptor_generator, "generate_model"): descriptor_generator.generate_model(data_set) # Add other if-else cases for other known implementation-specific generation # methods stubs # Generate descriptors of data for building NN index. data2descriptor = descriptor_generator.compute_descriptor_async( data_set, descriptor_elem_factory) try: nn_index.build_index(six.itervalues(data2descriptor)) except RuntimeError: # Already built model, so skipping this step pass rel_index.build_index(six.itervalues(data2descriptor))
def main(): parser = cli_parser() args = parser.parse_args() # # Setup logging # if not logging.getLogger().handlers: if args.verbose: bin_utils.initialize_logging(logging.getLogger(), logging.DEBUG) else: bin_utils.initialize_logging(logging.getLogger(), logging.INFO) log = logging.getLogger("smqtk.scripts.iqr_app_model_generation") search_app_config = json.loads(jsmin.jsmin(open(args.config).read())) # # Input parameters # # The following dictionaries are JSON configurations that are used to # configure the various data structures and algorithms needed for the IQR demo # application. Values here can be changed to suit your specific data and # algorithm needs. # # See algorithm implementation doc-strings for more information on configuration # parameters (see implementation class ``__init__`` method). # # base actions on a specific IQR tab configuration (choose index here) if args.tab < 0 or args.tab > (len(search_app_config["iqr_tabs"]) - 1): log.error("Invalid tab number provided.") exit(1) search_app_iqr_config = search_app_config["iqr_tabs"][args.tab] # Configure DataSet implementation and parameters data_set_config = search_app_iqr_config['data_set'] # Configure DescriptorGenerator algorithm implementation, parameters and # persistant model component locations (if implementation has any). descriptor_generator_config = search_app_iqr_config['descr_generator'] # Configure NearestNeighborIndex algorithm implementation, parameters and # persistant model component locations (if implementation has any). nn_index_config = search_app_iqr_config['nn_index'] # Configure RelevancyIndex algorithm implementation, parameters and # persistant model component locations (if implementation has any). # # The LibSvmHikRelevancyIndex implementation doesn't actually build a persistant # model (or doesn't have to that is), but we're leaving this block here in # anticipation of other potential implementations in the future. # rel_index_config = search_app_iqr_config['rel_index_config'] # Configure DescriptorElementFactory instance, which defines what implementation # of DescriptorElement to use for storing generated descriptor vectors below. descriptor_elem_factory_config = search_app_iqr_config['descriptor_factory'] # # Initialize data/algorithms # # Constructing appropriate data structures and algorithms, needed for the IQR # demo application, in preparation for model training. # descriptor_elem_factory = \ representation.DescriptorElementFactory \ .from_config(descriptor_elem_factory_config) #: :type: representation.DataSet data_set = \ plugin.from_plugin_config(data_set_config, representation.get_data_set_impls) #: :type: algorithms.DescriptorGenerator descriptor_generator = \ plugin.from_plugin_config(descriptor_generator_config, algorithms.get_descriptor_generator_impls) #: :type: algorithms.NearestNeighborsIndex nn_index = \ plugin.from_plugin_config(nn_index_config, algorithms.get_nn_index_impls) #: :type: algorithms.RelevancyIndex rel_index = \ plugin.from_plugin_config(rel_index_config, algorithms.get_relevancy_index_impls) # # Build models # # Perform the actual building of the models. # # Add data files to DataSet DataFileElement = representation.get_data_element_impls()["DataFileElement"] for fp in args.input_files: fp = osp.expanduser(fp) if osp.isfile(fp): data_set.add_data(DataFileElement(fp)) else: log.debug("Expanding glob: %s" % fp) for g in glob.iglob(fp): data_set.add_data(DataFileElement(g)) # Generate a mode if the generator defines a known generation method. if hasattr(descriptor_generator, "generate_model"): descriptor_generator.generate_model(data_set) # Add other if-else cases for other known implementation-specific generation # methods stubs # Generate descriptors of data for building NN index. data2descriptor = descriptor_generator.compute_descriptor_async( data_set, descriptor_elem_factory ) try: nn_index.build_index(data2descriptor.itervalues()) except RuntimeError: # Already built model, so skipping this step pass rel_index.build_index(data2descriptor.itervalues())