def get_config(self):
c = merge_dict(self.get_default_config(), {
"pickle_protocol": self.pickle_protocol,
})
if self.cache_element:
merge_dict(c['cache_element'], to_config_dict(self.cache_element))
return c
def load_config(config_path, defaults=None):
"""
Load the JSON configuration dictionary from the specified filepath.
If the given path does not point to a valid file, we return an empty
dictionary or the default dictionary if one was provided, returning False
as our second return argument.
:param config_path: Path to the (valid) JSON configuration file.
:type config_path: str
:param defaults: Optional default configuration dictionary to merge loaded
configuration into. If provided, it will be modified in place.
:type defaults: dict | None
:return: The result configuration dictionary and if we successfully loaded
a JSON dictionary from the given filepath.
:rtype: (dict, bool)
"""
if defaults is None:
defaults = {}
loaded = False
if config_path and os.path.isfile(config_path):
with open(config_path) as cf:
merge_dict(defaults, json.load(cf))
loaded = True
return defaults, loaded
def from_config(cls, config_dict, type_str, uuid, merge_default=True):
"""
Instantiate a new instance of this class given the desired type, uuid,
and JSON-compliant configuration dictionary.
:param type_str: Type of descriptor. This is usually the name of the
content descriptor that generated this vector.
:type type_str: str
:param uuid: Unique ID reference of the descriptor.
:type uuid: collections.Hashable
:param config_dict: JSON compliant dictionary encapsulating
a configuration.
:type config_dict: dict
:param merge_default: Merge the given configuration on top of the
default provided by ``get_default_config``.
:type merge_default: bool
:return: Constructed instance from the provided config.
:rtype: DescriptorElement
"""
c = {}
merge_dict(c, config_dict)
c['type_str'] = type_str
c['uuid'] = uuid
return super(DescriptorElement, cls).from_config(c, merge_default)
def make_response_json(message, return_code=200, **params):
"""
Basic message constructor for returning JSON from a flask routing function
:param message: String descriptive message to send back.
:type message: str
:param return_code: HTTP return code for this message. Default is 200.
:type return_code: int
:param params: Other key-value data to include in response JSON.
:type params: JSON-compliant
:return: Flask response and HTTP status code pair.
:rtype: (flask.Response, int)
"""
r = {
"message": message,
"time": {
"unix": time.time(),
"utc": time.asctime(time.gmtime()),
}
}
merge_dict(r, params)
return flask.jsonify(**r), return_code
def from_config(cls, config_dict, merge_default=True):
"""
Instantiate a new instance of this class given the configuration
JSON-compliant dictionary encapsulating initialization arguments.
This method should not be called via super unless and instance of the
class is desired.
:param config_dict: JSON compliant dictionary encapsulating
a configuration.
:type config_dict: dict
:param merge_default: Merge the given configuration on top of the
default provided by ``get_default_config``.
:type merge_default: bool
:return: Constructed instance from the provided config.
:rtype: MRPTNearestNeighborsIndex
"""
if merge_default:
cfg = cls.get_default_config()
merge_dict(cfg, config_dict)
else:
cfg = config_dict
cfg['descriptor_set'] = \
from_config_dict(cfg['descriptor_set'],
DescriptorSet.get_impls())
return super(MRPTNearestNeighborsIndex, cls).from_config(cfg, False)
def test_get_config(self):
self.assertEqual(MemoryDescriptorIndex().get_config(),
MemoryDescriptorIndex.get_default_config())
self.assertEqual(
MemoryDescriptorIndex(None).get_config(),
MemoryDescriptorIndex.get_default_config())
empty_elem = DataMemoryElement()
self.assertEqual(
MemoryDescriptorIndex(empty_elem).get_config(),
merge_dict(MemoryDescriptorIndex.get_default_config(),
{'cache_element': {
'type': 'DataMemoryElement'
}}))
dict_pickle_bytes = pickle.dumps({1: 1, 2: 2, 3: 3}, -1)
dict_pickle_bytes_str = dict_pickle_bytes.decode(BYTES_CONFIG_ENCODING)
cache_elem = DataMemoryElement(bytes=dict_pickle_bytes)
self.assertEqual(
MemoryDescriptorIndex(cache_elem).get_config(),
merge_dict(
MemoryDescriptorIndex.get_default_config(), {
'cache_element': {
'DataMemoryElement': {
'bytes': dict_pickle_bytes_str
},
'type': 'DataMemoryElement'
}
}))
def get_default_config(cls):
"""
Generate and return a default configuration dictionary for this class.
This will be primarily used for generating what the configuration
dictionary would look like for this class without instantiating it.
:return: Default configuration dictionary for the class.
:rtype: dict
"""
c = super(NearestNeighborServiceServer, cls).get_default_config()
merge_dict(
c, {
"descriptor_factory":
DescriptorElementFactory.get_default_config(),
"descriptor_generator":
make_default_config(DescriptorGenerator.get_impls()),
"nn_index":
make_default_config(NearestNeighborsIndex.get_impls()),
"descriptor_index":
make_default_config(DescriptorIndex.get_impls()),
"update_descriptor_index":
False,
})
return c
def test_get_config(self):
self.assertEqual(MemoryDescriptorSet().get_config(),
MemoryDescriptorSet.get_default_config())
self.assertEqual(
MemoryDescriptorSet(None).get_config(),
MemoryDescriptorSet.get_default_config())
empty_elem = DataMemoryElement()
dme_key = 'smqtk.representation.data_element.memory_element.DataMemoryElement'
self.assertEqual(
MemoryDescriptorSet(empty_elem).get_config(),
merge_dict(MemoryDescriptorSet.get_default_config(),
{'cache_element': {
'type': dme_key
}}))
dict_pickle_bytes = pickle.dumps({1: 1, 2: 2, 3: 3}, -1)
dict_pickle_bytes_str = dict_pickle_bytes.decode(BYTES_CONFIG_ENCODING)
cache_elem = DataMemoryElement(bytes=dict_pickle_bytes)
self.assertEqual(
MemoryDescriptorSet(cache_elem).get_config(),
merge_dict(
MemoryDescriptorSet.get_default_config(), {
'cache_element': {
dme_key: {
'bytes': dict_pickle_bytes_str
},
'type': dme_key
}
}))
def get_config(self):
c = merge_dict(self.get_default_config(), {
'leaf_size': self.leaf_size,
'random_seed': self.random_seed,
})
if self.cache_element:
c['cache_element'] = merge_dict(c['cache_element'],
to_config_dict(self.cache_element))
return c
def from_config(cls, config_dict, merge_default=True):
"""
Override to just pass the configuration dictionary to constructor
"""
# Repeated from super method due to overriding how constructor is called
if merge_default:
merged = cls.get_default_config()
merge_dict(merged, config_dict)
config_dict = merged
return cls(config_dict)
def from_config(cls, config_dict, merge_default=True):
"""
Instantiate a new instance of this class given the configuration
JSON-compliant dictionary encapsulating initialization arguments.
This method should not be called via super unless and instance of the
class is desired.
:param config_dict: JSON compliant dictionary encapsulating
a configuration.
:type config_dict: dict
:param merge_default: Merge the given configuration on top of the
default provided by ``get_default_config``.
:type merge_default: bool
:return: Constructed instance from the provided config.
:rtype: LSHNearestNeighborIndex
"""
# Controlling merge here so we can control known comment stripping from
# default config.
if merge_default:
merged = cls.get_default_config()
merge_dict(merged, config_dict)
else:
merged = config_dict
merged['lsh_functor'] = \
from_config_dict(merged['lsh_functor'], LshFunctor.get_impls())
merged['descriptor_set'] = \
from_config_dict(merged['descriptor_set'],
DescriptorSet.get_impls())
# Hash index may be None for a default at-query-time linear indexing
if merged['hash_index'] and merged['hash_index']['type']:
merged['hash_index'] = \
from_config_dict(merged['hash_index'],
HashIndex.get_impls())
else:
cls.get_logger().debug("No HashIndex impl given. Passing "
"``None``.")
merged['hash_index'] = None
# remove possible comment added by default generator
if 'hash_index_comment' in merged:
del merged['hash_index_comment']
merged['hash2uuids_kvstore'] = \
from_config_dict(merged['hash2uuids_kvstore'],
KeyValueStore.get_impls())
return super(LSHNearestNeighborIndex, cls).from_config(merged, False)
def test_subset_merge(self):
""" Test that `b` updates key `a` in dict `a`. """
a = {
'a': 1,
'b': 2,
}
b = {'a': 3}
expected = {
'a': 3,
'b': 2,
}
merge_dict(a, b)
self.assertEqual(a, expected)
def get_default_config(cls):
c = super(IqrSearchDispatcher, cls).get_default_config()
merge_dict(c, {
"mongo": {
"server": "127.0.0.1:27017",
"database": "smqtk",
},
# Each entry in this mapping generates a new tab in the GUI
"iqr_tabs": {
"__default__": IqrSearch.get_default_config(),
},
})
return c
def test_merge_dict_shallow(self):
# basic dict merger
merge_dict(self.a, self.b)
self.assertEqual(self.a, self.expected)
# set values that are mutable structures should be the same instances as
# what's in ``b``.
self.assertEqual(self.a['nested']['l'], self.b['nested']['l'])
self.assertIs(self.a['nested']['l'], self.b['nested']['l'])
self.assertEqual(self.a['nested']['even_deeper']['j'],
self.b['nested']['even_deeper']['j'])
self.assertIs(self.a['nested']['even_deeper']['j'],
self.b['nested']['even_deeper']['j'])
def test_merge_dict_deepcopy(self):
# dict merger with deepcopy
merge_dict(self.a, self.b, deep_copy=True)
self.assertEqual(self.a, self.expected)
# set values that are mutable structures should be the same instances as
# what's in ``b``.
self.assertEqual(self.a['nested']['l'], self.b['nested']['l'])
self.assertIsNot(self.a['nested']['l'], self.b['nested']['l'])
self.assertEqual(self.a['nested']['even_deeper']['j'],
self.b['nested']['even_deeper']['j'])
self.assertIsNot(self.a['nested']['even_deeper']['j'],
self.b['nested']['even_deeper']['j'])
def test_disjoint_update(self):
""" Test that ``'c': 3`` gets added to `a`. """
a = {
'a': 1,
'b': 2,
}
b = {'c': 3}
expected = {
'a': 1,
'b': 2,
'c': 3,
}
merge_dict(a, b)
self.assertEqual(a, expected)
def get_config(self):
"""
This implementation has no configuration properties.
:return: JSON type compliant configuration dictionary.
:rtype: dict
"""
c = merge_dict(self.get_default_config(), {
"pickle_protocol": self.pickle_protocol,
})
if self.cache_element:
c['cache_element'] = merge_dict(c['cache_element'],
to_config_dict(self.cache_element))
return c
def from_config(cls, config_dict, merge_default=True):
"""
Instantiate a new instance of this class given the configuration
JSON-compliant dictionary encapsulating initialization arguments.
This method should not be called via super unless and instance of the
class is desired.
:param config_dict: JSON compliant dictionary encapsulating
a configuration.
:type config_dict: dict
:param merge_default: Merge the given configuration on top of the
default provided by ``get_default_config``.
:type merge_default: bool
:return: Constructed instance from the provided config.
:rtype: LSHNearestNeighborIndex
"""
if merge_default:
cfg = cls.get_default_config()
merge_dict(cfg, config_dict)
else:
cfg = config_dict
cfg['descriptor_set'] = from_config_dict(cfg['descriptor_set'],
DescriptorSet.get_impls())
cfg['uid2idx_kvs'] = from_config_dict(cfg['uid2idx_kvs'],
KeyValueStore.get_impls())
cfg['idx2uid_kvs'] = from_config_dict(cfg['idx2uid_kvs'],
KeyValueStore.get_impls())
if (cfg['index_element'] and cfg['index_element']['type']):
index_element = from_config_dict(cfg['index_element'],
DataElement.get_impls())
cfg['index_element'] = index_element
else:
cfg['index_element'] = None
if (cfg['index_param_element'] and cfg['index_param_element']['type']):
index_param_element = from_config_dict(cfg['index_param_element'],
DataElement.get_impls())
cfg['index_param_element'] = index_param_element
else:
cfg['index_param_element'] = None
return super(FaissNearestNeighborsIndex, cls).from_config(cfg, False)
def from_config(cls, config_dict, merge_default=True):
"""
Instantiate a new instance of this class given the configuration
JSON-compliant dictionary encapsulating initialization arguments.
Overrides base because this implementation's "bytes" argument wants to
be given a ``bytes`` type object. When not None, in python 2 this is a
normal string (not unicode), while in python 3 bytes is a distinct
type.
"""
if merge_default:
config_dict = merge_dict(cls.get_default_config(),
config_dict)
try:
# In python 3, encode input ``str`` into ``bytes``.
# In python 2, even though ``str`` and ``bytes`` are the same
# underlying type, we could be given ``unicode``, which needs to be
# encoded down to ``bytes`` (``str``).
config_dict["bytes"] = \
config_dict['bytes'].encode(BYTES_CONFIG_ENCODING)
except AttributeError:
# If this is a None value, which has no attributes at all, leave it
# alone. If in python 2 and given a unicode string, as is the norm
# return from ``json.load`` and ``json.loads``,
pass
return super(DataMemoryElement, cls).from_config(config_dict,
merge_default=False)
def from_config(cls, config_dict, merge_default=True):
"""
Instantiate a new instance of this class given the configuration
JSON-compliant dictionary encapsulating initialization arguments.
:param config_dict: JSON compliant dictionary encapsulating
a configuration.
:type config_dict: dict
:param merge_default: Merge the given configuration on top of the
default provided by ``get_default_config``.
:type merge_default: bool
:return: Constructed instance from the provided config.
:rtype: KVSDataSet
"""
if merge_default:
config_dict = merge_dict(cls.get_default_config(), config_dict)
# Convert KVStore config to instance for constructor.
kvs_inst = from_config_dict(config_dict['kvstore'],
KeyValueStore.get_impls())
config_dict['kvstore'] = kvs_inst
return super(KVSDataSet, cls).from_config(config_dict, False)
def from_config(cls, config_dict, merge_default=True):
"""
Instantiate a new instance of this class given the configuration
JSON-compliant dictionary encapsulating initialization arguments.
:param config_dict: JSON compliant dictionary encapsulating
a configuration.
:type config_dict: dict
:param merge_default: Merge the given configuration on top of the
default provided by ``get_default_config``.
:type merge_default: bool
:return: Constructed instance from the provided config.
:rtype: MemoryDescriptorSet
"""
if merge_default:
config_dict = merge_dict(cls.get_default_config(), config_dict)
# Optionally construct cache element from sub-config.
if config_dict['cache_element'] \
and config_dict['cache_element']['type']:
e = from_config_dict(config_dict['cache_element'],
DataElement.get_impls())
config_dict['cache_element'] = e
else:
config_dict['cache_element'] = None
return super(MemoryDescriptorSet, cls).from_config(config_dict, False)
def from_config(cls, config_dict, merge_default=True):
if merge_default:
config_dict = merge_dict(cls.get_default_config(), config_dict)
data_elem_impl_set = DataElement.get_impls()
# Translate prototext and model sub-configs into DataElement instances.
config_dict['network_prototxt'] = \
from_config_dict(config_dict['network_prototxt'],
data_elem_impl_set)
config_dict['network_model'] = \
from_config_dict(config_dict['network_model'],
data_elem_impl_set)
# Translate optionally provided image mean sub-config into a
# DataElement instance. May have been provided as ``None`` or a
# configuration dictionary with type ``None`.
# None, dict[type=None], dict[type=str]
if config_dict['image_mean'] is None \
or config_dict['image_mean'].get('type', None) is None:
config_dict['image_mean'] = None
else:
config_dict['image_mean'] = \
from_config_dict(config_dict['image_mean'], data_elem_impl_set)
return super(CaffeDescriptorGenerator, cls)\
.from_config(config_dict, merge_default=False)
def from_config(cls, config_dict, uuid, merge_default=True):
"""
Override of
:meth:`smqtk.utils.configuration.Configurable.from_config` with the
added runtime argument ``uuid``. See parent method documentation for
details.
:param config_dict: JSON compliant dictionary encapsulating
a configuration.
:type config_dict: dict
:param collections.Hashable uuid:
UUID to assign to the produced DetectionElement.
:param merge_default: Merge the given configuration on top of the
default provided by ``get_default_config``.
:type merge_default: bool
:return: Constructed instance from the provided config.
:rtype: DetectionElement
"""
# Override from Configurable
# Handle passing of runtime positional argument(s).
if merge_default:
config_dict = merge_dict(cls.get_default_config(), config_dict)
config_dict['uuid'] = uuid
return super(DetectionElement, cls).from_config(config_dict,
merge_default=False)
def from_config(cls, config_dict, merge_default=True):
"""
Instantiate a new instance of this class given the configuration
JSON-compliant dictionary encapsulating initialization arguments.
This method should not be called via super unless and instance of the
class is desired.
:param config_dict: JSON compliant dictionary encapsulating
a configuration.
:type config_dict: dict
:param merge_default: Merge the given configuration on top of the
default provided by ``get_default_config``.
:type merge_default: bool
:return: Constructed instance from the provided config.
:rtype: ClassificationElementFactory
"""
if merge_default:
config_dict = merge_dict(cls.get_default_config(), config_dict)
ce_type, ce_conf = cls_conf_from_config_dict(
config_dict, ClassificationElement.get_impls()
)
return ClassificationElementFactory(ce_type, ce_conf)
def from_config(cls, c, merge_default=True):
"""
Instantiate a new instance of this class given the configuration
JSON-compliant dictionary encapsulating initialization arguments.
This method should not be called via super unless an instance of the
class is desired.
:param c: JSON compliant dictionary encapsulating
a configuration.
:type c: dict
:param merge_default: Merge the given configuration on top of the
default provided by ``get_default_config``.
:type merge_default: bool
:return: Constructed instance from the provided config.
:rtype: DataMemorySet
"""
if merge_default:
c = merge_dict(cls.get_default_config(), c)
cache_element = None
if c['cache_element'] and c['cache_element']['type']:
cache_element = from_config_dict(c['cache_element'],
DataElement.get_impls())
c['cache_element'] = cache_element
return super(DataMemorySet, cls).from_config(c, False)
def from_config(cls, config_dict, merge_default=True):
"""
Instantiate a new instance of this class given the configuration
JSON-compliant dictionary encapsulating initialization arguments.
This method should not be called via super unless an instance of the
class is desired.
:param config_dict: JSON compliant dictionary encapsulating
a configuration.
:type config_dict: dict
:param merge_default: Merge the given configuration on top of the
default provided by ``get_default_config``.
:type merge_default: bool
:return: Constructed instance from the provided config.
:rtype: SkLearnBallTreeHashIndex
"""
if merge_default:
config_dict = merge_dict(cls.get_default_config(), config_dict)
# Parse ``cache_element`` configuration if set.
cache_element = None
if config_dict['cache_element'] and \
config_dict['cache_element']['type']:
cache_element = \
from_config_dict(config_dict['cache_element'],
DataElement.get_impls())
config_dict['cache_element'] = cache_element
return super(SkLearnBallTreeHashIndex,
cls).from_config(config_dict, False)
def from_config(cls, config_dict, merge_default=True):
# Override from Configurable
if merge_default:
config_dict = merge_dict(cls.get_default_config(), config_dict)
elem_type, elem_conf = cls_conf_from_config_dict(
config_dict, DetectionElement.get_impls())
return DetectionElementFactory(elem_type, elem_conf)
def get_default_config(cls):
"""
Generate and return a default configuration dictionary for this class.
This will be primarily used for generating what the configuration
dictionary would look like for this class without instantiating it.
:return: Default configuration dictionary for the class.
:rtype: dict
"""
c = super(DescriptorServiceServer, cls).get_default_config()
merge_dict(c, {
"descriptor_factory": DescriptorElementFactory.get_default_config(),
"descriptor_generators": {
"example": make_default_config(DescriptorGenerator.get_impls())
}
})
return c
def test_overrides(self):
""" Test that dict `b` overrides key `b` with a nested dict. """
a = {
'a': 1,
'b': 2,
}
b = {
'b': {
'c': 3
},
}
expected = {
'a': 1,
'b': {
'c': 3,
}
}
merge_dict(a, b)
self.assertEqual(a, expected)
def test_partial_update(self):
"""
Test that dict `b` updates and adds keys appropriately to dict `a` (adds
key `c`, updates key `a`).
"""
a = {
'a': 1,
'b': 2,
}
b = {
'a': 3,
'c': 4,
}
expected = {
'a': 3,
'b': 2,
'c': 4,
}
merge_dict(a, b)
self.assertEqual(a, expected)
