def main(): args = cli_parser().parse_args() config = bin_utils.utility_main_helper(default_config, args) log = logging.getLogger(__name__) uuids_list_filepath = config['uuids_list_filepath'] log.info("Initializing ITQ functor") #: :type: smqtk.algorithms.nn_index.lsh.functors.itq.ItqFunctor functor = ItqFunctor.from_config(config['itq_config']) log.info("Initializing DescriptorIndex [type=%s]", config['descriptor_index']['type']) #: :type: smqtk.representation.DescriptorIndex descriptor_index = plugin.from_plugin_config( config['descriptor_index'], get_descriptor_index_impls(), ) if uuids_list_filepath and os.path.isfile(uuids_list_filepath): def uuids_iter(): with open(uuids_list_filepath) as f: for l in f: yield l.strip() log.info("Loading UUIDs list from file: %s", uuids_list_filepath) d_iter = descriptor_index.get_many_descriptors(uuids_iter()) else: log.info("Using UUIDs from loaded DescriptorIndex (count=%d)", len(descriptor_index)) d_iter = descriptor_index log.info("Fitting ITQ model") functor.fit(d_iter) log.info("Done")
def test_configuration_with_caches(self): # This should run without error in both python # 2 and 3, as str/unicode are JSON compliant in both. expected_mean_vec = numpy.array([1, 2, 3]) expected_rotation = numpy.eye(3) expected_mean_vec_bytes = six.BytesIO() # noinspection PyTypeChecker numpy.save(expected_mean_vec_bytes, expected_mean_vec) expected_mean_vec_str = \ expected_mean_vec_bytes.getvalue().decode(BYTES_CONFIG_ENCODING) expected_rotation_bytes = six.BytesIO() # noinspection PyTypeChecker numpy.save(expected_rotation_bytes, expected_rotation) expected_rotation_str = \ expected_rotation_bytes.getvalue().decode(BYTES_CONFIG_ENCODING) new_parts = { 'mean_vec_cache': { 'smqtk.representation.data_element.memory_element.DataMemoryElement': { 'bytes': expected_mean_vec_str }, 'type': 'smqtk.representation.data_element.memory_element.DataMemoryElement' }, 'rotation_cache': { 'smqtk.representation.data_element.memory_element.DataMemoryElement': { 'bytes': expected_rotation_str }, 'type': 'smqtk.representation.data_element.memory_element.DataMemoryElement' }, 'bit_length': 153, 'itq_iterations': 7, 'normalize': 2, 'random_seed': 58, } c = merge_dict(ItqFunctor.get_default_config(), new_parts) itq = ItqFunctor.from_config(c) # Checking that loaded parameters were correctly set and cache elements # correctly return intended vector/matrix. numpy.testing.assert_equal(itq.mean_vec, [1, 2, 3]) numpy.testing.assert_equal(itq.rotation, [[1, 0, 0], [0, 1, 0], [0, 0, 1]]) self.assertEqual(itq.bit_length, 153) self.assertEqual(itq.itq_iterations, 7) self.assertEqual(itq.normalize, 2) self.assertEqual(itq.random_seed, 58)
def main(): description = """ Tool for training the ITQ functor algorithm's model on descriptors in an index. By default, we use all descriptors in the configured index (``uuids_list_filepath`` is not given a value). The ``uuids_list_filepath`` configuration property is optional and should be used to specify a sub-set of descriptors in the configured index to train on. This only works if the stored descriptors' UUID is a type of string. """ args, config = bin_utils.utility_main_helper(default_config, description) log = logging.getLogger(__name__) uuids_list_filepath = config['uuids_list_filepath'] log.info("Initializing ITQ functor") #: :type: smqtk.algorithms.nn_index.lsh.functors.itq.ItqFunctor functor = ItqFunctor.from_config(config['itq_config']) log.info("Initializing DescriptorIndex [type=%s]", config['descriptor_index']['type']) #: :type: smqtk.representation.DescriptorIndex descriptor_index = plugin.from_plugin_config( config['descriptor_index'], get_descriptor_index_impls(), ) if uuids_list_filepath and os.path.isfile(uuids_list_filepath): def uuids_iter(): with open(uuids_list_filepath) as f: for l in f: yield l.strip() log.info("Loading UUIDs list from file: %s", uuids_list_filepath) d_iter = descriptor_index.get_many_descriptors(uuids_iter()) else: log.info("Using UUIDs from loaded DescriptorIndex (count=%d)", len(descriptor_index)) d_iter = descriptor_index log.info("Fitting ITQ model") functor.fit(d_iter) log.info("Done")
def test_configuration_with_caches(self): expected_mean_vec = numpy.array([1, 2, 3]) expected_rotation = numpy.eye(3) expected_mean_vec_bytes = BytesIO() # noinspection PyTypeChecker numpy.save(expected_mean_vec_bytes, expected_mean_vec) expected_mean_vec_bytes = expected_mean_vec_bytes.getvalue() expected_rotation_bytes = BytesIO() # noinspection PyTypeChecker numpy.save(expected_rotation_bytes, expected_rotation) expected_rotation_bytes = expected_rotation_bytes.getvalue() new_parts = { 'mean_vec_cache': { 'DataMemoryElement': { 'bytes': expected_mean_vec_bytes }, 'type': 'DataMemoryElement' }, 'rotation_cache': { 'DataMemoryElement': { 'bytes': expected_rotation_bytes }, 'type': 'DataMemoryElement' }, 'bit_length': 153, 'itq_iterations': 7, 'normalize': 2, 'random_seed': 58, } c = merge_dict(ItqFunctor.get_default_config(), new_parts) itq = ItqFunctor.from_config(c) # Checking that loaded parameters were correctly set and cache elements # correctly return intended vector/matrix. numpy.testing.assert_equal(itq.mean_vec, [1, 2, 3]) numpy.testing.assert_equal(itq.rotation, [[1, 0, 0], [0, 1, 0], [0, 0, 1]]) self.assertEqual(itq.bit_length, 153) self.assertEqual(itq.itq_iterations, 7) self.assertEqual(itq.normalize, 2) self.assertEqual(itq.random_seed, 58)
def main(): args = cli_parser().parse_args() config = cli.utility_main_helper(default_config, args) log = logging.getLogger(__name__) uuids_list_filepath = config['uuids_list_filepath'] log.info("Initializing ITQ functor") #: :type: smqtk.algorithms.nn_index.lsh.functors.itq.ItqFunctor functor = ItqFunctor.from_config(config['itq_config']) log.info("Initializing DescriptorSet [type=%s]", config['descriptor_set']['type']) #: :type: smqtk.representation.DescriptorSet descriptor_set = from_config_dict( config['descriptor_set'], DescriptorSet.get_impls(), ) if uuids_list_filepath and os.path.isfile(uuids_list_filepath): def uuids_iter(): with open(uuids_list_filepath) as f: for l in f: yield l.strip() log.info("Loading UUIDs list from file: %s", uuids_list_filepath) d_iter = descriptor_set.get_many_descriptors(uuids_iter()) else: log.info("Using UUIDs from loaded DescriptorSet (count=%d)", len(descriptor_set)) d_iter = descriptor_set log.info("Fitting ITQ model") functor.fit(d_iter) log.info("Done")
def test_default_configuration(self): c = ItqFunctor.get_default_config() self.assertEqual(ItqFunctor.from_config(c).get_config(), c)