def test_get_config(self) -> None:
self.assertEqual(MemoryDescriptorSet().get_config(),
MemoryDescriptorSet.get_default_config())
self.assertEqual(
MemoryDescriptorSet(None).get_config(),
MemoryDescriptorSet.get_default_config())
empty_elem = DataMemoryElement()
dme_key = 'smqtk_dataprovider.impls.data_element.memory.DataMemoryElement'
self.assertEqual(
MemoryDescriptorSet(empty_elem).get_config(),
merge_dict(MemoryDescriptorSet.get_default_config(),
{'cache_element': {
'type': dme_key
}}))
dict_pickle_bytes = pickle.dumps({1: 1, 2: 2, 3: 3}, -1)
dict_pickle_bytes_str = dict_pickle_bytes.decode(BYTES_CONFIG_ENCODING)
cache_elem = DataMemoryElement(bytes=dict_pickle_bytes)
self.assertEqual(
MemoryDescriptorSet(cache_elem).get_config(),
merge_dict(
MemoryDescriptorSet.get_default_config(), {
'cache_element': {
dme_key: {
'bytes': dict_pickle_bytes_str
},
'type': dme_key
}
}))
def from_config( # type: ignore
cls: Type[T],
config_dict: Dict,
type_str: str,
uuid: Hashable,
merge_default: bool = True) -> T:
"""
Instantiate a new instance of this class given the desired type, uuid,
and JSON-compliant configuration dictionary.
:param type_str: Type of descriptor. This is usually the name of the
content descriptor that generated this vector.
:param uuid: Unique ID reference of the descriptor.
:param config_dict: JSON compliant dictionary encapsulating
a configuration.
:param merge_default: Merge the given configuration on top of the
default provided by ``get_default_config``.
:return: Constructed instance from the provided config.
"""
c: Dict[str, Any] = {}
merge_dict(c, config_dict)
c['type_str'] = type_str
c['uuid'] = uuid
return super(DescriptorElement, cls).from_config(c, merge_default)
def get_config(self) -> Dict[str, Any]:
c = merge_dict(self.get_default_config(), {
"pickle_protocol": self.pickle_protocol,
})
if self.cache_element:
merge_dict(c['cache_element'],
to_config_dict(self.cache_element))
return c
def get_config(self) -> Dict[str, Any]:
c = merge_dict(self.get_default_config(), {
'leaf_size': self.leaf_size,
'random_seed': self.random_seed,
})
if self.cache_element:
c['cache_element'] = merge_dict(c['cache_element'],
to_config_dict(self.cache_element))
return c
def from_config(
cls: Type[T_LSH],
config_dict: Dict,
merge_default: bool = True
) -> T_LSH:
"""
Instantiate a new instance of this class given the configuration
JSON-compliant dictionary encapsulating initialization arguments.
This method should not be called via super unless and instance of the
class is desired.
:param config_dict: JSON compliant dictionary encapsulating
a configuration.
:type config_dict: dict
:param merge_default: Merge the given configuration on top of the
default provided by ``get_default_config``.
:type merge_default: bool
:return: Constructed instance from the provided config.
:rtype: LSHNearestNeighborIndex
"""
# Controlling merge here so we can control known comment stripping from
# default config.
if merge_default:
merged = cls.get_default_config()
merge_dict(merged, config_dict)
else:
merged = config_dict
merged['lsh_functor'] = \
from_config_dict(merged['lsh_functor'], LshFunctor.get_impls())
merged['descriptor_set'] = \
from_config_dict(merged['descriptor_set'],
DescriptorSet.get_impls())
# Hash index may be None for a default at-query-time linear indexing
if merged['hash_index'] and merged['hash_index']['type']:
merged['hash_index'] = \
from_config_dict(merged['hash_index'],
HashIndex.get_impls())
else:
LOG.debug("No HashIndex impl given. Passing ``None``.")
merged['hash_index'] = None
# remove possible comment added by default generator
if 'hash_index_comment' in merged:
del merged['hash_index_comment']
merged['hash2uuids_kvstore'] = \
from_config_dict(merged['hash2uuids_kvstore'],
KeyValueStore.get_impls())
return super(LSHNearestNeighborIndex, cls).from_config(merged, False)
def test_subset_merge(self) -> None:
""" Test that `b` updates key `a` in dict `a`. """
a = {
'a': 1,
'b': 2,
}
b = {'a': 3}
expected = {
'a': 3,
'b': 2,
}
merge_dict(a, b)
self.assertEqual(a, expected)
def test_merge_dict_shallow(self) -> None:
# basic dict merger
merge_dict(self.a, self.b)
self.assertEqual(self.a, self.expected)
# set values that are mutable structures should be the same instances as
# what's in ``b``.
self.assertEqual(self.a['nested']['l'], self.b['nested']['l'])
self.assertIs(self.a['nested']['l'], self.b['nested']['l'])
self.assertEqual(self.a['nested']['even_deeper']['j'],
self.b['nested']['even_deeper']['j'])
self.assertIs(self.a['nested']['even_deeper']['j'],
self.b['nested']['even_deeper']['j'])
def test_merge_dict_deepcopy(self) -> None:
# dict merger with deepcopy
merge_dict(self.a, self.b, deep_copy=True)
self.assertEqual(self.a, self.expected)
# set values that are mutable structures should be the same instances as
# what's in ``b``.
self.assertEqual(self.a['nested']['l'], self.b['nested']['l'])
self.assertIsNot(self.a['nested']['l'], self.b['nested']['l'])
self.assertEqual(self.a['nested']['even_deeper']['j'],
self.b['nested']['even_deeper']['j'])
self.assertIsNot(self.a['nested']['even_deeper']['j'],
self.b['nested']['even_deeper']['j'])
def test_disjoint_update(self) -> None:
""" Test that ``'c': 3`` gets added to `a`. """
a = {
'a': 1,
'b': 2,
}
b = {'c': 3}
expected = {
'a': 1,
'b': 2,
'c': 3,
}
merge_dict(a, b)
self.assertEqual(a, expected)
def from_config(cls: Type[T],
config_dict: Dict,
merge_default: bool = True) -> T:
if merge_default:
cfg = cls.get_default_config()
merge_dict(cfg, config_dict)
else:
cfg = config_dict
cfg['descriptor_set'] = \
from_config_dict(cfg['descriptor_set'],
DescriptorSet.get_impls())
return super(MRPTNearestNeighborsIndex, cls).from_config(cfg, False)
def get_config(self) -> Dict[str, Any]:
"""
This implementation has no configuration properties.
:return: JSON type compliant configuration dictionary.
:rtype: dict
"""
c = merge_dict(self.get_default_config(), {
"pickle_protocol": self.pickle_protocol,
})
if self.cache_element:
c['cache_element'] = merge_dict(c['cache_element'],
to_config_dict(self.cache_element))
return c
def from_config(cls: Type[T],
config_dict: Dict,
merge_default: bool = True) -> T:
"""
Instantiate a new instance of this class given the configuration
JSON-compliant dictionary encapsulating initialization arguments.
:param config_dict: JSON compliant dictionary encapsulating
a configuration.
:type config_dict: dict
:param merge_default: Merge the given configuration on top of the
default provided by ``get_default_config``.
:type merge_default: bool
:return: Constructed instance from the provided config.
:rtype: KVSDataSet
"""
if merge_default:
config_dict = merge_dict(cls.get_default_config(), config_dict)
# Convert KVStore config to instance for constructor.
kvs_inst = from_config_dict(config_dict['kvstore'],
KeyValueStore.get_impls())
config_dict['kvstore'] = kvs_inst
return super(KVSDataSet, cls).from_config(config_dict, False)
def from_config(cls: Type[T_IF],
config_dict: Dict,
merge_default: bool = True) -> T_IF:
"""
Instantiate a new instance of this class given the JSON-compliant
configuration dictionary encapsulating initialization arguments.
:param config_dict: JSON compliant dictionary encapsulating
a configuration.
:param merge_default: Merge the given configuration on top of the
default provided by ``get_default_config``.
:return: Constructed instance from the provided config.
"""
if merge_default:
config_dict = merge_dict(cls.get_default_config(), config_dict)
data_element_impls = DataElement.get_impls()
# Mean vector cache element.
mean_vec_cache = None
if config_dict['mean_vec_cache'] and \
config_dict['mean_vec_cache']['type']:
mean_vec_cache = from_config_dict(config_dict['mean_vec_cache'],
data_element_impls)
config_dict['mean_vec_cache'] = mean_vec_cache
# Rotation matrix cache element.
rotation_cache = None
if config_dict['rotation_cache'] and \
config_dict['rotation_cache']['type']:
rotation_cache = from_config_dict(config_dict['rotation_cache'],
data_element_impls)
config_dict['rotation_cache'] = rotation_cache
return super(ItqFunctor, cls).from_config(config_dict, False)
def from_config(
cls: Type[MDS],
config_dict: Dict,
merge_default: bool = True
) -> MDS:
"""
Instantiate a new instance of this class given the configuration
JSON-compliant dictionary encapsulating initialization arguments.
:param config_dict: JSON compliant dictionary encapsulating
a configuration.
:param merge_default: Merge the given configuration on top of the
default provided by ``get_default_config``.
:return: Constructed instance from the provided config.
"""
if merge_default:
config_dict = merge_dict(cls.get_default_config(), config_dict)
# Optionally construct cache element from sub-config.
if config_dict['cache_element'] \
and config_dict['cache_element']['type']:
e = from_config_dict(config_dict['cache_element'],
DataElement.get_impls())
config_dict['cache_element'] = e
else:
config_dict['cache_element'] = None
return super(MemoryDescriptorSet, cls).from_config(config_dict, False)
def from_config(cls, config_dict, merge_default=True):
"""
Instantiate a new instance of this class given the configuration
JSON-compliant dictionary encapsulating initialization arguments.
This method should not be called via super unless and instance of the
class is desired.
:param config_dict: JSON compliant dictionary encapsulating
a configuration.
:type config_dict: dict
:param merge_default: Merge the given configuration on top of the
default provided by ``get_default_config``.
:type merge_default: bool
:return: Constructed instance from the provided config.
:rtype: ClassificationElementFactory
"""
if merge_default:
config_dict = merge_dict(cls.get_default_config(), config_dict)
ce_type, ce_conf = cls_conf_from_config_dict(
config_dict, ClassificationElement.get_impls())
return ClassificationElementFactory(ce_type, ce_conf)
def from_config(
cls: Type[T],
config_dict: Dict,
merge_default: bool = True
) -> T:
"""
Instantiate a new instance of this class given the configuration
JSON-compliant dictionary encapsulating initialization arguments.
Overrides base because this implementation's "bytes" argument wants to
be given a ``bytes`` type object. When not None, in python 2 this is a
normal string (not unicode), while in python 3 bytes is a distinct
type.
"""
if merge_default:
config_dict = merge_dict(cls.get_default_config(),
config_dict)
try:
# In python 3, encode input ``str`` into ``bytes``.
# In python 2, even though ``str`` and ``bytes`` are the same
# underlying type, we could be given ``unicode``, which needs to be
# encoded down to ``bytes`` (``str``).
config_dict["bytes"] = \
config_dict['bytes'].encode(BYTES_CONFIG_ENCODING)
except AttributeError:
# If this is a None value, which has no attributes at all, leave it
# alone. If in python 2 and given a unicode string, as is the norm
# return from ``json.load`` and ``json.loads``,
pass
return super(DataMemoryElement, cls).from_config(config_dict,
merge_default=False)
def from_config(
cls: Type[T],
config_dict: Dict,
merge_default: bool = True
) -> T:
"""
Instantiate a new instance of this class given the configuration
JSON-compliant dictionary encapsulating initialization arguments.
This method should not be called via super unless and instance of the
class is desired.
:param config_dict: JSON compliant dictionary encapsulating
a configuration.
:param merge_default: Merge the given configuration on top of the
default provided by ``get_default_config``.
:return: Constructed instance from the provided config.
"""
if merge_default:
config_dict = merge_dict(cls.get_default_config(), config_dict)
de_type, de_conf = cls_conf_from_config_dict(
config_dict, DescriptorElement.get_impls()
)
return cls(de_type, de_conf)
def from_config(cls: Type[T],
config_dict: Dict,
merge_default: bool = True) -> T:
if merge_default:
config_dict = merge_dict(cls.get_default_config(), config_dict)
data_elem_impl_set = DataElement.get_impls()
# Translate prototext and model sub-configs into DataElement instances.
config_dict['network_prototxt'] = \
from_config_dict(config_dict['network_prototxt'],
data_elem_impl_set)
config_dict['network_model'] = \
from_config_dict(config_dict['network_model'],
data_elem_impl_set)
# Translate optionally provided image mean sub-config into a
# DataElement instance. May have been provided as ``None`` or a
# configuration dictionary with type ``None`.
# None, dict[type=None], dict[type=str]
if config_dict['image_mean'] is None \
or config_dict['image_mean'].get('type', None) is None:
config_dict['image_mean'] = None
else:
config_dict['image_mean'] = \
from_config_dict(config_dict['image_mean'], data_elem_impl_set)
return super(CaffeDescriptorGenerator,
cls).from_config(config_dict, merge_default=False)
def from_config(cls: Type[T_FNNI],
config_dict: Dict,
merge_default: bool = True) -> T_FNNI:
"""
Instantiate a new instance of this class given the configuration
JSON-compliant dictionary encapsulating initialization arguments.
This method should not be called via super unless and instance of the
class is desired.
:param config_dict: JSON compliant dictionary encapsulating
a configuration.
:param merge_default: Merge the given configuration on top of the
default provided by ``get_default_config``.
:return: Constructed instance from the provided config.
"""
if merge_default:
cfg = cls.get_default_config()
merge_dict(cfg, config_dict)
else:
cfg = config_dict
cfg['descriptor_set'] = from_config_dict(cfg['descriptor_set'],
DescriptorSet.get_impls())
cfg['uid2idx_kvs'] = from_config_dict(cfg['uid2idx_kvs'],
KeyValueStore.get_impls())
cfg['idx2uid_kvs'] = from_config_dict(cfg['idx2uid_kvs'],
KeyValueStore.get_impls())
if (cfg['index_element'] and cfg['index_element']['type']):
index_element = from_config_dict(cfg['index_element'],
DataElement.get_impls())
cfg['index_element'] = index_element
else:
cfg['index_element'] = None
if (cfg['index_param_element'] and cfg['index_param_element']['type']):
index_param_element = from_config_dict(cfg['index_param_element'],
DataElement.get_impls())
cfg['index_param_element'] = index_param_element
else:
cfg['index_param_element'] = None
return super(FaissNearestNeighborsIndex, cls).from_config(cfg, False)
def get_config(self) -> Dict[str, Any]:
return merge_dict(
self.get_default_config(),
dict(
rank_relevancy=to_config_dict(self._rank_relevancy),
n=self._n,
seed=self._seed,
))
def test_overrides(self) -> None:
""" Test that dict `b` overrides key `b` with a nested dict. """
a = {
'a': 1,
'b': 2,
}
b = {
'b': {
'c': 3
},
}
expected = {
'a': 1,
'b': {
'c': 3,
}
}
merge_dict(a, b)
self.assertEqual(a, expected)
def test_partial_update(self) -> None:
"""
Test that dict `b` updates and adds keys appropriately to dict `a` (adds
key `c`, updates key `a`).
"""
a = {
'a': 1,
'b': 2,
}
b = {
'a': 3,
'c': 4,
}
expected = {
'a': 3,
'b': 2,
'c': 4,
}
merge_dict(a, b)
self.assertEqual(a, expected)
def test_configuration_with_caches(self) -> None:
# This should run without error in both python
# 2 and 3, as str/unicode are JSON compliant in both.
expected_mean_vec = numpy.array([1, 2, 3])
expected_rotation = numpy.eye(3)
expected_mean_vec_bytes = BytesIO()
# noinspection PyTypeChecker
numpy.save(expected_mean_vec_bytes, expected_mean_vec)
expected_mean_vec_str = \
expected_mean_vec_bytes.getvalue().decode(BYTES_CONFIG_ENCODING)
expected_rotation_bytes = BytesIO()
# noinspection PyTypeChecker
numpy.save(expected_rotation_bytes, expected_rotation)
expected_rotation_str = \
expected_rotation_bytes.getvalue().decode(BYTES_CONFIG_ENCODING)
new_parts = {
'mean_vec_cache': {
'smqtk_dataprovider.impls.data_element.memory.DataMemoryElement':
{
'bytes': expected_mean_vec_str
},
'type':
'smqtk_dataprovider.impls.data_element.memory.DataMemoryElement'
},
'rotation_cache': {
'smqtk_dataprovider.impls.data_element.memory.DataMemoryElement':
{
'bytes': expected_rotation_str
},
'type':
'smqtk_dataprovider.impls.data_element.memory.DataMemoryElement'
},
'bit_length': 153,
'itq_iterations': 7,
'normalize': 2,
'random_seed': 58,
}
c = merge_dict(ItqFunctor.get_default_config(), new_parts)
itq = ItqFunctor.from_config(c)
# Checking that loaded parameters were correctly set and cache elements
# correctly return intended vector/matrix.
numpy.testing.assert_equal(itq.mean_vec, [1, 2, 3])
numpy.testing.assert_equal(itq.rotation,
[[1, 0, 0], [0, 1, 0], [0, 0, 1]])
self.assertEqual(itq.bit_length, 153)
self.assertEqual(itq.itq_iterations, 7)
self.assertEqual(itq.normalize, 2)
self.assertEqual(itq.random_seed, 58)
def test_nested(self) -> None:
"""
Test that nested dictionary updates occur correctly without losing
values in sibling or nephew dictionaries.
"""
a = {
'a': 1,
'b': {
'c': 2,
'd': {
'e': 3
},
},
'f': {
'g': 4,
'h': {
'i': 5
}
},
}
b = {'b': {'c': 6}, 'f': {'h': {'i': 7}}, 'j': 8}
expected = {
'a': 1,
'b': {
'c': 6,
'd': {
'e': 3
},
},
'f': {
'g': 4,
'h': {
'i': 7
}
},
'j': 8,
}
merge_dict(a, b)
self.assertEqual(a, expected)
def from_config(cls: Type[T],
config_dict: Dict,
merge_default: bool = True) -> T:
if merge_default:
config_dict = merge_dict(cls.get_default_config(), config_dict)
cache_element = None
if config_dict['cache_element'] and config_dict['cache_element'][
'type']:
cache_element = from_config_dict(config_dict['cache_element'],
DataElement.get_impls())
config_dict['cache_element'] = cache_element
return super(DataMemorySet, cls).from_config(config_dict, False)
def get_default_config(cls) -> Dict:
"""
Generate and return a default configuration dictionary for this class.
It is not be guaranteed that the configuration dictionary returned
from this method is valid for construction of an instance of this class.
:return: Default configuration dictionary for the class.
:rtype: dict
"""
c = super(KVSDataSet, cls).get_default_config()
c['kvstore'] = merge_dict(
make_default_config(KeyValueStore.get_impls()),
to_config_dict(c['kvstore']))
return c
def get_config(self) -> Dict[str, Any]:
# If no cache elements (set to None), return default plugin configs.
c = merge_dict(
self.get_default_config(), {
"bit_length": self.bit_length,
"itq_iterations": self.itq_iterations,
"normalize": self.normalize,
"random_seed": self.random_seed,
})
if self.mean_vec_cache_elem:
c['mean_vec_cache'] = \
to_config_dict(self.mean_vec_cache_elem)
if self.rotation_cache_elem:
c['rotation_cache'] = \
to_config_dict(self.rotation_cache_elem)
return c
def from_config(
cls,
config_dict: Dict,
merge_default: bool = True) -> "ClassifyDescriptorCollection":
if merge_default:
config_dict = merge_dict(cls.get_default_config(), config_dict)
classifier_map = {}
# Copying list of keys so we can update the dictionary as we loop.
for label in list(config_dict.keys()):
# Skip the example section.
if label == cls.EXAMPLE_KEY:
continue
classifier_config = config_dict[label]
classifier = from_config_dict(classifier_config,
ClassifyDescriptor.get_impls())
classifier_map[label] = classifier
return cls(classifiers=classifier_map)
def from_config(cls, config_dict, merge_default=True):
"""
Instantiate a new instance of this class given the configuration
JSON-compliant dictionary encapsulating initialization arguments.
This method should not be called via super unless an instance of the
class is desired.
:param config_dict: JSON compliant dictionary encapsulating
a configuration.
:type config_dict: dict
:param merge_default: Merge the given configuration on top of the
default provided by ``get_default_config``.
:type merge_default: bool
:return: Constructed instance from the provided config.
:rtype: ClassifierCollection
"""
if merge_default:
config_dict = merge_dict(cls.get_default_config(), config_dict)
classifier_map = {}
# Copying list of keys so we can update the dictionary as we loop.
for label in list(config_dict.keys()):
# Skip the example section.
if label == cls.EXAMPLE_KEY:
continue
classifier_config = config_dict[label]
classifier = from_config_dict(classifier_config,
Classifier.get_impls())
classifier_map[label] = classifier
# Don't merge back in "example" default
return super(ClassifierCollection,
cls).from_config({'classifiers': classifier_map},
merge_default=False)
def from_config(
cls: Type[T],
config_dict: Dict,
merge_default: bool = True
) -> T:
"""
Instantiate a new instance of this class given the configuration
JSON-compliant dictionary encapsulating initialization arguments.
This method should not be called via super unless an instance of the
class is desired.
:param config_dict: JSON compliant dictionary encapsulating
a configuration.
:type config_dict: dict
:param merge_default: Merge the given configuration on top of the
default provided by ``get_default_config``.
:type merge_default: bool
:return: Constructed instance from the provided config.
:rtype: SkLearnBallTreeHashIndex
"""
if merge_default:
config_dict = merge_dict(cls.get_default_config(), config_dict)
# Parse ``cache_element`` configuration if set.
cache_element = None
if config_dict['cache_element'] and \
config_dict['cache_element']['type']:
cache_element = \
from_config_dict(config_dict['cache_element'],
DataElement.get_impls())
config_dict['cache_element'] = cache_element
return super(SkLearnBallTreeHashIndex, cls).from_config(config_dict,
False)
