def test_nested_set(self):
dict = {
'a': {
'b': {
'c': 1,
'd': 2,
},
},
'e': {
'f': 3,
'g': 4,
},
}
new_val = 3
expected = deepcopy(dict)
expected['a']['b'] = new_val
self.assertEqual(DictUtil.nested_set(deepcopy(dict), 'a.b', new_val), expected)
new_val = 10
expected = deepcopy(dict)
expected['a']['b']['c'] = new_val
self.assertEqual(DictUtil.nested_set(deepcopy(dict), 'a.b.c', new_val), expected)
new_val = {'x': {'y': 'z'}}
expected = deepcopy(dict)
expected['e'] = new_val
self.assertEqual(DictUtil.nested_set(deepcopy(dict), 'e', new_val), expected)
new_val = [-1, 0, 1]
expected = deepcopy(dict)
expected['e']['g'] = new_val
self.assertEqual(DictUtil.nested_set(deepcopy(dict), 'e.g', new_val), expected)
def test_set_value(self):
key = 'key'
key_a_dict_value = 8
key_b_dict_value = 3
key_c_dict_value = 'hi'
test_dict = {
'a_dict':
{'nested': {'key': key_a_dict_value,
'not_key': 9}},
'b_dict':
{'nested': {'key': key_b_dict_value,
'not_key': 15}},
'c_dict':
{'key': {'key': key_c_dict_value,
'not_key': 15}}
}
self.assertEqual(test_dict['a_dict']['nested'][key], key_a_dict_value)
self.assertEqual(test_dict['b_dict']['nested'][key], key_b_dict_value)
self.assertEqual(test_dict['c_dict']['key'][key], key_c_dict_value)
new_value = 11
DictUtil.set_value(test_dict, key, new_value)
self.assertEqual(test_dict['a_dict']['nested'][key], new_value)
self.assertEqual(test_dict['b_dict']['nested'][key], new_value)
self.assertEqual(test_dict['c_dict']['key'][key], key_c_dict_value)
DictUtil.set_value(test_dict, key, new_value, match_type=False)
self.assertEqual(test_dict['a_dict']['nested'][key], new_value)
self.assertEqual(test_dict['b_dict']['nested'][key], new_value)
self.assertEqual(test_dict['c_dict']['key'][key], new_value)
def test_dict_filtering(self):
d = {'a': 1, 'b': 2, 'c': 3}
self.assertEqual(DictUtil.filtered_dict(d, []), {})
self.assertEqual(DictUtil.filtered_dict(d, ['a']), {'a': 1})
self.assertEqual(DictUtil.filtered_dict(d, ['a', 'd', 'a', 'b', 'c']), d)
self.assertEqual({(k, v) for k, v in DictUtil.filtered_iteritems(d, ['a', 'b', 'd'])},
{('a', 1), ('b', 2)})
def convert_metadata(self, metadata_class):
""" Convert the saved simulation metadata into HDF5 attributes on a `metadata` Group
Args:
metadata_class (:obj:`EnhancedDataClass`): the class that stored the metadata
"""
metadata = metadata_class.read_dataclass(self.results_dir)
# open the metadata file
metadata_file = self._hdf_file()
with h5py.File(metadata_file, 'a') as hdf5_file:
# open or make a group in metadata_file called 'metadata'
if self.METADATA_GROUP in hdf5_file:
metadata_group = hdf5_file[self.METADATA_GROUP]
else:
metadata_group = hdf5_file.create_group(self.METADATA_GROUP)
# make a nested dict of the metadata
metadata_as_dict = dataclasses.asdict(metadata)
metadata_class_name = self.METADATA_CLASS_TO_NAME[metadata_class]
flat_metadata_as_dict = DictUtil.flatten_dict(metadata_as_dict)
for key, value in flat_metadata_as_dict.items():
# make a dotted string for each value in the metadata
# metadata_as_dict keys cannot contain '.' because they're attribute names
separator = '.'
name = f'{metadata_class_name}{separator}{separator.join(key)}'
# make an attribute for each value
if value is None:
value = 'NA'
if not isinstance(value, (int, float, str)):
value = str(value)
metadata_group.attrs[name] = value
def test_nested_in(self):
dict = {
'a': {
'b': {
'c': 1,
'd': 2,
},
},
'e': {
'f': 3,
'g': 4,
},
}
self.assertEqual(DictUtil.nested_in(dict, 'a.b'), 'a' in dict and 'b' in dict['a'])
self.assertEqual(DictUtil.nested_in(dict, 'a.b.c'), 'a' in dict and 'b' in dict['a'] and 'c' in dict['a']['b'])
self.assertEqual(DictUtil.nested_in(dict, 'a.b.e'), 'a' in dict and 'b' in dict['a'] and 'e' in dict['a']['b'])
def test_nested_get(self):
dict = {
'a': {
'b': {
'c': 1,
'd': 2,
},
},
'e': {
'f': 3,
'g': 4,
},
}
self.assertEqual(DictUtil.nested_get(dict, 'a.b'), dict['a']['b'])
self.assertEqual(DictUtil.nested_get(dict, 'a.b.c'), dict['a']['b']['c'])
self.assertEqual(DictUtil.nested_get(dict, 'e'), dict['e'])
self.assertEqual(DictUtil.nested_get(dict, 'e.g'), dict['e']['g'])
def get_metadata(self):
""" Get simulation metadata from the HDF5 file
Returns:
:obj:`dict`: a nested dictionary of metadata corresponding to the attributes in
:obj:`SimulationMetadata` and :obj:`WCSimulationMetadata`
"""
hdf5_file = h5py.File(self._hdf_file(), 'r')
metadata_attrs = hdf5_file[self.METADATA_GROUP].attrs
metadata_attrs_as_dict = {}
for key, value in metadata_attrs.items():
metadata_attrs_as_dict[key] = value
return DictUtil.expand_dict(metadata_attrs_as_dict)
def get_config(self, extra=None, context=None):
""" Setup configuration from config file(s), environment variables, and/or function arguments.
1. Setup configuration from default values specified in `paths.default`.
2. If `paths.user` is set, find the first file in it that exists, and override
the default configuration with the values specified in the file.
3. Override configuration with values from environment variables. Environment variables
can be set with the following syntax::
CONFIG.level1.level2...=val
4. Override configuration with additional configuration in `extra`.
5. Substitute context into templates
6. Validate configuration against the schema specified in `paths.schema`.
Args:
extra (:obj:`dict`, optional): additional configuration to override
context (:obj:`dict`, optional): context for template substitution
Returns:
:obj:`configobj.ConfigObj`: nested dictionary with the configuration settings loaded from the configuration source(s).
Raises:
:obj:`InvalidConfigError`: if configuration doesn't validate against schema
:obj:`ValueError`: if no configuration is found
"""
# read configuration schema/specification
config_specification = ConfigObj(self.paths.schema,
list_values=False,
_inspec=True)
# read default configuration
value_sources = []
if os.path.isfile(self.paths.default):
value_sources.append(self.paths.default)
config = ConfigObj(infile=self.paths.default,
configspec=config_specification)
self.validate(config, value_sources)
# read user's configuration files
for user_config_filename in self.paths.user:
if os.path.isfile(user_config_filename):
override_config = ConfigObj(infile=user_config_filename,
configspec=config_specification)
config.merge(override_config)
self.validate(config, [user_config_filename])
break
# read configuration from environment variables
value_sources = []
for key, val in os.environ.items():
if key.startswith('CONFIG__DOT__'):
nested_keys = key[13:].split('__DOT__')
if nested_keys[0] in config:
DictUtil.nested_set(config, nested_keys, val)
value_sources.append(
"Environment variable '{}'".format(key))
self.validate(config, value_sources)
# merge extra configuration
if extra is None:
extra = {}
else:
config.merge(extra)
self.validate(config, ["'extra' argument"])
# ensure that a configuration is found
if not config:
raise ValueError(
("No configuration found in:\n"
" Default path: {}\n"
" User paths: {}\n"
" Extras: {}\n"
" Environment variables").format(self.paths.default,
', '.join(self.paths.user),
extra))
# perform template substitution
to_sub = [config]
while to_sub:
dictionary = to_sub.pop()
keys = list(dictionary.keys())
for key in keys:
val = dictionary[key]
key2 = string.Template(key).substitute(context)
val2 = val
if isinstance(val, dict):
to_sub.append(val)
elif isinstance(val, (list, tuple)):
val2 = [
string.Template(v).substitute(context) for v in val
]
elif isinstance(val, six.string_types):
val2 = string.Template(val).substitute(context)
dictionary.pop(key)
dictionary[key2] = val2
# re-validate configuration against schema after substitution
self.validate(config, value_sources)
# return config
return config
def test_to_string_sorted_by_key(self):
self.assertEqual(DictUtil.to_string_sorted_by_key(None), '{}')
self.assertEqual(DictUtil.to_string_sorted_by_key({'b': 2, 'c': 3, 'a': 1, 'd': 4}), "{'a': 1, 'b': 2, 'c': 3, 'd': 4}")