def get_config(self): c = merge_dict(self.get_default_config(), { "pickle_protocol": self.pickle_protocol, }) if self.cache_element: merge_dict(c['cache_element'], to_config_dict(self.cache_element)) return c
def load_config(config_path, defaults=None): """ Load the JSON configuration dictionary from the specified filepath. If the given path does not point to a valid file, we return an empty dictionary or the default dictionary if one was provided, returning False as our second return argument. :param config_path: Path to the (valid) JSON configuration file. :type config_path: str :param defaults: Optional default configuration dictionary to merge loaded configuration into. If provided, it will be modified in place. :type defaults: dict | None :return: The result configuration dictionary and if we successfully loaded a JSON dictionary from the given filepath. :rtype: (dict, bool) """ if defaults is None: defaults = {} loaded = False if config_path and os.path.isfile(config_path): with open(config_path) as cf: merge_dict(defaults, json.load(cf)) loaded = True return defaults, loaded
def from_config(cls, config_dict, type_str, uuid, merge_default=True): """ Instantiate a new instance of this class given the desired type, uuid, and JSON-compliant configuration dictionary. :param type_str: Type of descriptor. This is usually the name of the content descriptor that generated this vector. :type type_str: str :param uuid: Unique ID reference of the descriptor. :type uuid: collections.Hashable :param config_dict: JSON compliant dictionary encapsulating a configuration. :type config_dict: dict :param merge_default: Merge the given configuration on top of the default provided by ``get_default_config``. :type merge_default: bool :return: Constructed instance from the provided config. :rtype: DescriptorElement """ c = {} merge_dict(c, config_dict) c['type_str'] = type_str c['uuid'] = uuid return super(DescriptorElement, cls).from_config(c, merge_default)
def make_response_json(message, return_code=200, **params): """ Basic message constructor for returning JSON from a flask routing function :param message: String descriptive message to send back. :type message: str :param return_code: HTTP return code for this message. Default is 200. :type return_code: int :param params: Other key-value data to include in response JSON. :type params: JSON-compliant :return: Flask response and HTTP status code pair. :rtype: (flask.Response, int) """ r = { "message": message, "time": { "unix": time.time(), "utc": time.asctime(time.gmtime()), } } merge_dict(r, params) return flask.jsonify(**r), return_code
def from_config(cls, config_dict, merge_default=True): """ Instantiate a new instance of this class given the configuration JSON-compliant dictionary encapsulating initialization arguments. This method should not be called via super unless and instance of the class is desired. :param config_dict: JSON compliant dictionary encapsulating a configuration. :type config_dict: dict :param merge_default: Merge the given configuration on top of the default provided by ``get_default_config``. :type merge_default: bool :return: Constructed instance from the provided config. :rtype: MRPTNearestNeighborsIndex """ if merge_default: cfg = cls.get_default_config() merge_dict(cfg, config_dict) else: cfg = config_dict cfg['descriptor_set'] = \ from_config_dict(cfg['descriptor_set'], DescriptorSet.get_impls()) return super(MRPTNearestNeighborsIndex, cls).from_config(cfg, False)
def test_get_config(self): self.assertEqual(MemoryDescriptorIndex().get_config(), MemoryDescriptorIndex.get_default_config()) self.assertEqual( MemoryDescriptorIndex(None).get_config(), MemoryDescriptorIndex.get_default_config()) empty_elem = DataMemoryElement() self.assertEqual( MemoryDescriptorIndex(empty_elem).get_config(), merge_dict(MemoryDescriptorIndex.get_default_config(), {'cache_element': { 'type': 'DataMemoryElement' }})) dict_pickle_bytes = pickle.dumps({1: 1, 2: 2, 3: 3}, -1) dict_pickle_bytes_str = dict_pickle_bytes.decode(BYTES_CONFIG_ENCODING) cache_elem = DataMemoryElement(bytes=dict_pickle_bytes) self.assertEqual( MemoryDescriptorIndex(cache_elem).get_config(), merge_dict( MemoryDescriptorIndex.get_default_config(), { 'cache_element': { 'DataMemoryElement': { 'bytes': dict_pickle_bytes_str }, 'type': 'DataMemoryElement' } }))
def get_default_config(cls): """ Generate and return a default configuration dictionary for this class. This will be primarily used for generating what the configuration dictionary would look like for this class without instantiating it. :return: Default configuration dictionary for the class. :rtype: dict """ c = super(NearestNeighborServiceServer, cls).get_default_config() merge_dict( c, { "descriptor_factory": DescriptorElementFactory.get_default_config(), "descriptor_generator": make_default_config(DescriptorGenerator.get_impls()), "nn_index": make_default_config(NearestNeighborsIndex.get_impls()), "descriptor_index": make_default_config(DescriptorIndex.get_impls()), "update_descriptor_index": False, }) return c
def test_get_config(self): self.assertEqual(MemoryDescriptorSet().get_config(), MemoryDescriptorSet.get_default_config()) self.assertEqual( MemoryDescriptorSet(None).get_config(), MemoryDescriptorSet.get_default_config()) empty_elem = DataMemoryElement() dme_key = 'smqtk.representation.data_element.memory_element.DataMemoryElement' self.assertEqual( MemoryDescriptorSet(empty_elem).get_config(), merge_dict(MemoryDescriptorSet.get_default_config(), {'cache_element': { 'type': dme_key }})) dict_pickle_bytes = pickle.dumps({1: 1, 2: 2, 3: 3}, -1) dict_pickle_bytes_str = dict_pickle_bytes.decode(BYTES_CONFIG_ENCODING) cache_elem = DataMemoryElement(bytes=dict_pickle_bytes) self.assertEqual( MemoryDescriptorSet(cache_elem).get_config(), merge_dict( MemoryDescriptorSet.get_default_config(), { 'cache_element': { dme_key: { 'bytes': dict_pickle_bytes_str }, 'type': dme_key } }))
def get_config(self): c = merge_dict(self.get_default_config(), { 'leaf_size': self.leaf_size, 'random_seed': self.random_seed, }) if self.cache_element: c['cache_element'] = merge_dict(c['cache_element'], to_config_dict(self.cache_element)) return c
def from_config(cls, config_dict, merge_default=True): """ Override to just pass the configuration dictionary to constructor """ # Repeated from super method due to overriding how constructor is called if merge_default: merged = cls.get_default_config() merge_dict(merged, config_dict) config_dict = merged return cls(config_dict)
def from_config(cls, config_dict, merge_default=True): """ Instantiate a new instance of this class given the configuration JSON-compliant dictionary encapsulating initialization arguments. This method should not be called via super unless and instance of the class is desired. :param config_dict: JSON compliant dictionary encapsulating a configuration. :type config_dict: dict :param merge_default: Merge the given configuration on top of the default provided by ``get_default_config``. :type merge_default: bool :return: Constructed instance from the provided config. :rtype: LSHNearestNeighborIndex """ # Controlling merge here so we can control known comment stripping from # default config. if merge_default: merged = cls.get_default_config() merge_dict(merged, config_dict) else: merged = config_dict merged['lsh_functor'] = \ from_config_dict(merged['lsh_functor'], LshFunctor.get_impls()) merged['descriptor_set'] = \ from_config_dict(merged['descriptor_set'], DescriptorSet.get_impls()) # Hash index may be None for a default at-query-time linear indexing if merged['hash_index'] and merged['hash_index']['type']: merged['hash_index'] = \ from_config_dict(merged['hash_index'], HashIndex.get_impls()) else: cls.get_logger().debug("No HashIndex impl given. Passing " "``None``.") merged['hash_index'] = None # remove possible comment added by default generator if 'hash_index_comment' in merged: del merged['hash_index_comment'] merged['hash2uuids_kvstore'] = \ from_config_dict(merged['hash2uuids_kvstore'], KeyValueStore.get_impls()) return super(LSHNearestNeighborIndex, cls).from_config(merged, False)
def test_subset_merge(self): """ Test that `b` updates key `a` in dict `a`. """ a = { 'a': 1, 'b': 2, } b = {'a': 3} expected = { 'a': 3, 'b': 2, } merge_dict(a, b) self.assertEqual(a, expected)
def get_default_config(cls): c = super(IqrSearchDispatcher, cls).get_default_config() merge_dict(c, { "mongo": { "server": "127.0.0.1:27017", "database": "smqtk", }, # Each entry in this mapping generates a new tab in the GUI "iqr_tabs": { "__default__": IqrSearch.get_default_config(), }, }) return c
def test_merge_dict_shallow(self): # basic dict merger merge_dict(self.a, self.b) self.assertEqual(self.a, self.expected) # set values that are mutable structures should be the same instances as # what's in ``b``. self.assertEqual(self.a['nested']['l'], self.b['nested']['l']) self.assertIs(self.a['nested']['l'], self.b['nested']['l']) self.assertEqual(self.a['nested']['even_deeper']['j'], self.b['nested']['even_deeper']['j']) self.assertIs(self.a['nested']['even_deeper']['j'], self.b['nested']['even_deeper']['j'])
def test_merge_dict_deepcopy(self): # dict merger with deepcopy merge_dict(self.a, self.b, deep_copy=True) self.assertEqual(self.a, self.expected) # set values that are mutable structures should be the same instances as # what's in ``b``. self.assertEqual(self.a['nested']['l'], self.b['nested']['l']) self.assertIsNot(self.a['nested']['l'], self.b['nested']['l']) self.assertEqual(self.a['nested']['even_deeper']['j'], self.b['nested']['even_deeper']['j']) self.assertIsNot(self.a['nested']['even_deeper']['j'], self.b['nested']['even_deeper']['j'])
def test_disjoint_update(self): """ Test that ``'c': 3`` gets added to `a`. """ a = { 'a': 1, 'b': 2, } b = {'c': 3} expected = { 'a': 1, 'b': 2, 'c': 3, } merge_dict(a, b) self.assertEqual(a, expected)
def get_config(self): """ This implementation has no configuration properties. :return: JSON type compliant configuration dictionary. :rtype: dict """ c = merge_dict(self.get_default_config(), { "pickle_protocol": self.pickle_protocol, }) if self.cache_element: c['cache_element'] = merge_dict(c['cache_element'], to_config_dict(self.cache_element)) return c
def from_config(cls, config_dict, merge_default=True): """ Instantiate a new instance of this class given the configuration JSON-compliant dictionary encapsulating initialization arguments. This method should not be called via super unless and instance of the class is desired. :param config_dict: JSON compliant dictionary encapsulating a configuration. :type config_dict: dict :param merge_default: Merge the given configuration on top of the default provided by ``get_default_config``. :type merge_default: bool :return: Constructed instance from the provided config. :rtype: LSHNearestNeighborIndex """ if merge_default: cfg = cls.get_default_config() merge_dict(cfg, config_dict) else: cfg = config_dict cfg['descriptor_set'] = from_config_dict(cfg['descriptor_set'], DescriptorSet.get_impls()) cfg['uid2idx_kvs'] = from_config_dict(cfg['uid2idx_kvs'], KeyValueStore.get_impls()) cfg['idx2uid_kvs'] = from_config_dict(cfg['idx2uid_kvs'], KeyValueStore.get_impls()) if (cfg['index_element'] and cfg['index_element']['type']): index_element = from_config_dict(cfg['index_element'], DataElement.get_impls()) cfg['index_element'] = index_element else: cfg['index_element'] = None if (cfg['index_param_element'] and cfg['index_param_element']['type']): index_param_element = from_config_dict(cfg['index_param_element'], DataElement.get_impls()) cfg['index_param_element'] = index_param_element else: cfg['index_param_element'] = None return super(FaissNearestNeighborsIndex, cls).from_config(cfg, False)
def from_config(cls, config_dict, merge_default=True): """ Instantiate a new instance of this class given the configuration JSON-compliant dictionary encapsulating initialization arguments. Overrides base because this implementation's "bytes" argument wants to be given a ``bytes`` type object. When not None, in python 2 this is a normal string (not unicode), while in python 3 bytes is a distinct type. """ if merge_default: config_dict = merge_dict(cls.get_default_config(), config_dict) try: # In python 3, encode input ``str`` into ``bytes``. # In python 2, even though ``str`` and ``bytes`` are the same # underlying type, we could be given ``unicode``, which needs to be # encoded down to ``bytes`` (``str``). config_dict["bytes"] = \ config_dict['bytes'].encode(BYTES_CONFIG_ENCODING) except AttributeError: # If this is a None value, which has no attributes at all, leave it # alone. If in python 2 and given a unicode string, as is the norm # return from ``json.load`` and ``json.loads``, pass return super(DataMemoryElement, cls).from_config(config_dict, merge_default=False)
def from_config(cls, config_dict, merge_default=True): """ Instantiate a new instance of this class given the configuration JSON-compliant dictionary encapsulating initialization arguments. :param config_dict: JSON compliant dictionary encapsulating a configuration. :type config_dict: dict :param merge_default: Merge the given configuration on top of the default provided by ``get_default_config``. :type merge_default: bool :return: Constructed instance from the provided config. :rtype: KVSDataSet """ if merge_default: config_dict = merge_dict(cls.get_default_config(), config_dict) # Convert KVStore config to instance for constructor. kvs_inst = from_config_dict(config_dict['kvstore'], KeyValueStore.get_impls()) config_dict['kvstore'] = kvs_inst return super(KVSDataSet, cls).from_config(config_dict, False)
def from_config(cls, config_dict, merge_default=True): """ Instantiate a new instance of this class given the configuration JSON-compliant dictionary encapsulating initialization arguments. :param config_dict: JSON compliant dictionary encapsulating a configuration. :type config_dict: dict :param merge_default: Merge the given configuration on top of the default provided by ``get_default_config``. :type merge_default: bool :return: Constructed instance from the provided config. :rtype: MemoryDescriptorSet """ if merge_default: config_dict = merge_dict(cls.get_default_config(), config_dict) # Optionally construct cache element from sub-config. if config_dict['cache_element'] \ and config_dict['cache_element']['type']: e = from_config_dict(config_dict['cache_element'], DataElement.get_impls()) config_dict['cache_element'] = e else: config_dict['cache_element'] = None return super(MemoryDescriptorSet, cls).from_config(config_dict, False)
def from_config(cls, config_dict, merge_default=True): if merge_default: config_dict = merge_dict(cls.get_default_config(), config_dict) data_elem_impl_set = DataElement.get_impls() # Translate prototext and model sub-configs into DataElement instances. config_dict['network_prototxt'] = \ from_config_dict(config_dict['network_prototxt'], data_elem_impl_set) config_dict['network_model'] = \ from_config_dict(config_dict['network_model'], data_elem_impl_set) # Translate optionally provided image mean sub-config into a # DataElement instance. May have been provided as ``None`` or a # configuration dictionary with type ``None`. # None, dict[type=None], dict[type=str] if config_dict['image_mean'] is None \ or config_dict['image_mean'].get('type', None) is None: config_dict['image_mean'] = None else: config_dict['image_mean'] = \ from_config_dict(config_dict['image_mean'], data_elem_impl_set) return super(CaffeDescriptorGenerator, cls)\ .from_config(config_dict, merge_default=False)
def from_config(cls, config_dict, uuid, merge_default=True): """ Override of :meth:`smqtk.utils.configuration.Configurable.from_config` with the added runtime argument ``uuid``. See parent method documentation for details. :param config_dict: JSON compliant dictionary encapsulating a configuration. :type config_dict: dict :param collections.Hashable uuid: UUID to assign to the produced DetectionElement. :param merge_default: Merge the given configuration on top of the default provided by ``get_default_config``. :type merge_default: bool :return: Constructed instance from the provided config. :rtype: DetectionElement """ # Override from Configurable # Handle passing of runtime positional argument(s). if merge_default: config_dict = merge_dict(cls.get_default_config(), config_dict) config_dict['uuid'] = uuid return super(DetectionElement, cls).from_config(config_dict, merge_default=False)
def from_config(cls, config_dict, merge_default=True): """ Instantiate a new instance of this class given the configuration JSON-compliant dictionary encapsulating initialization arguments. This method should not be called via super unless and instance of the class is desired. :param config_dict: JSON compliant dictionary encapsulating a configuration. :type config_dict: dict :param merge_default: Merge the given configuration on top of the default provided by ``get_default_config``. :type merge_default: bool :return: Constructed instance from the provided config. :rtype: ClassificationElementFactory """ if merge_default: config_dict = merge_dict(cls.get_default_config(), config_dict) ce_type, ce_conf = cls_conf_from_config_dict( config_dict, ClassificationElement.get_impls() ) return ClassificationElementFactory(ce_type, ce_conf)
def from_config(cls, c, merge_default=True): """ Instantiate a new instance of this class given the configuration JSON-compliant dictionary encapsulating initialization arguments. This method should not be called via super unless an instance of the class is desired. :param c: JSON compliant dictionary encapsulating a configuration. :type c: dict :param merge_default: Merge the given configuration on top of the default provided by ``get_default_config``. :type merge_default: bool :return: Constructed instance from the provided config. :rtype: DataMemorySet """ if merge_default: c = merge_dict(cls.get_default_config(), c) cache_element = None if c['cache_element'] and c['cache_element']['type']: cache_element = from_config_dict(c['cache_element'], DataElement.get_impls()) c['cache_element'] = cache_element return super(DataMemorySet, cls).from_config(c, False)
def from_config(cls, config_dict, merge_default=True): """ Instantiate a new instance of this class given the configuration JSON-compliant dictionary encapsulating initialization arguments. This method should not be called via super unless an instance of the class is desired. :param config_dict: JSON compliant dictionary encapsulating a configuration. :type config_dict: dict :param merge_default: Merge the given configuration on top of the default provided by ``get_default_config``. :type merge_default: bool :return: Constructed instance from the provided config. :rtype: SkLearnBallTreeHashIndex """ if merge_default: config_dict = merge_dict(cls.get_default_config(), config_dict) # Parse ``cache_element`` configuration if set. cache_element = None if config_dict['cache_element'] and \ config_dict['cache_element']['type']: cache_element = \ from_config_dict(config_dict['cache_element'], DataElement.get_impls()) config_dict['cache_element'] = cache_element return super(SkLearnBallTreeHashIndex, cls).from_config(config_dict, False)
def from_config(cls, config_dict, merge_default=True): # Override from Configurable if merge_default: config_dict = merge_dict(cls.get_default_config(), config_dict) elem_type, elem_conf = cls_conf_from_config_dict( config_dict, DetectionElement.get_impls()) return DetectionElementFactory(elem_type, elem_conf)
def get_default_config(cls): """ Generate and return a default configuration dictionary for this class. This will be primarily used for generating what the configuration dictionary would look like for this class without instantiating it. :return: Default configuration dictionary for the class. :rtype: dict """ c = super(DescriptorServiceServer, cls).get_default_config() merge_dict(c, { "descriptor_factory": DescriptorElementFactory.get_default_config(), "descriptor_generators": { "example": make_default_config(DescriptorGenerator.get_impls()) } }) return c
def test_overrides(self): """ Test that dict `b` overrides key `b` with a nested dict. """ a = { 'a': 1, 'b': 2, } b = { 'b': { 'c': 3 }, } expected = { 'a': 1, 'b': { 'c': 3, } } merge_dict(a, b) self.assertEqual(a, expected)
def test_partial_update(self): """ Test that dict `b` updates and adds keys appropriately to dict `a` (adds key `c`, updates key `a`). """ a = { 'a': 1, 'b': 2, } b = { 'a': 3, 'c': 4, } expected = { 'a': 3, 'b': 2, 'c': 4, } merge_dict(a, b) self.assertEqual(a, expected)