def add_config_to_tree(self, *config_files):
"""Add configuration to tree."""
conf = {}
for config_file in config_files:
if os.path.isfile(config_file):
with open(config_file) as fd:
resample_config = yaml.load(fd, Loader=yaml.UnsafeLoader)
if resample_config is None:
# empty file
continue
resampling_section = resample_config.get(self.prefix, {})
if not resampling_section:
logging.debug(
"Config '{}' has no '{}' section or it is empty".
format(config_file, self.prefix))
continue
conf = recursive_dict_update(conf, resampling_section)
elif isinstance(config_file, dict):
conf = recursive_dict_update(conf, config_file)
else:
logger.debug("Loading resampling config string")
d = yaml.load(config_file, Loader=yaml.UnsafeLoader)
if not isinstance(d, dict):
raise ValueError(
"YAML file doesn't exist or string is not YAML dict: {}"
.format(config_file))
conf = recursive_dict_update(conf, d)
self._build_tree(conf)
def add_config_to_tree(self, *decision_dict):
"""Add configuration to tree."""
conf = {}
for config_file in decision_dict:
if os.path.isfile(config_file):
with open(config_file) as fd:
enhancement_config = yaml.load(fd, Loader=UnsafeLoader)
if enhancement_config is None:
# empty file
continue
enhancement_section = enhancement_config.get(
self.prefix, {})
if not enhancement_section:
LOG.debug(
"Config '{}' has no '{}' section or it is empty".
format(config_file, self.prefix))
continue
conf = recursive_dict_update(conf, enhancement_section)
elif isinstance(config_file, dict):
conf = recursive_dict_update(conf, config_file)
else:
LOG.debug("Loading enhancement config string")
d = yaml.load(config_file, Loader=UnsafeLoader)
if not isinstance(d, dict):
raise ValueError(
"YAML file doesn't exist or string is not YAML dict: {}"
.format(config_file))
conf = recursive_dict_update(conf, d)
self._build_tree(conf)
def _load_config(self, composite_configs):
if not isinstance(composite_configs, (list, tuple)):
composite_configs = [composite_configs]
conf = {}
for composite_config in composite_configs:
with open(composite_config, 'r', encoding='utf-8') as conf_file:
conf = recursive_dict_update(
conf, yaml.load(conf_file, Loader=UnsafeLoader))
try:
sensor_name = conf['sensor_name']
except KeyError:
logger.debug('No "sensor_name" tag found in %s, skipping.',
composite_configs)
return
sensor_id = sensor_name.split('/')[-1]
sensor_deps = sensor_name.split('/')[:-1]
compositors = self.compositors.setdefault(sensor_id, DatasetDict())
modifiers = self.modifiers.setdefault(sensor_id, {})
for sensor_dep in reversed(sensor_deps):
if sensor_dep not in self.compositors or sensor_dep not in self.modifiers:
self.load_sensor_composites(sensor_dep)
if sensor_deps:
compositors.update(self.compositors[sensor_deps[-1]])
modifiers.update(self.modifiers[sensor_deps[-1]])
id_keys = self._get_sensor_id_keys(conf, sensor_id, sensor_deps)
mod_config_helper = _ModifierConfigHelper(modifiers, id_keys)
configured_modifiers = conf.get('modifiers', {})
mod_config_helper.parse_config(configured_modifiers, composite_configs)
comp_config_helper = _CompositeConfigHelper(compositors, id_keys)
configured_composites = conf.get('composites', {})
comp_config_helper.parse_config(configured_composites,
composite_configs)
def _load_config(composite_configs):
if not isinstance(composite_configs, (list, tuple)):
composite_configs = [composite_configs]
conf = {}
for composite_config in composite_configs:
with open(composite_config, 'r', encoding='utf-8') as conf_file:
conf = recursive_dict_update(conf, yaml.load(conf_file, Loader=UnsafeLoader))
try:
sensor_name = conf['sensor_name']
except KeyError:
logger.debug('No "sensor_name" tag found in %s, skipping.',
composite_configs)
return
sensor_compositors = {} # DatasetDict()
sensor_modifiers = {}
dep_id_keys = None
sensor_deps = sensor_name.split('/')[:-1]
if sensor_deps:
# get dependent
for sensor_dep in sensor_deps:
dep_comps, dep_mods, dep_id_keys = load_compositor_configs_for_sensor(sensor_dep)
# the last parent should include all of its parents so only add the last one
sensor_compositors.update(dep_comps)
sensor_modifiers.update(dep_mods)
id_keys = _get_sensor_id_keys(conf, dep_id_keys)
mod_config_helper = _ModifierConfigHelper(sensor_modifiers, id_keys)
configured_modifiers = conf.get('modifiers', {})
mod_config_helper.parse_config(configured_modifiers, composite_configs)
comp_config_helper = _CompositeConfigHelper(sensor_compositors, id_keys)
configured_composites = conf.get('composites', {})
comp_config_helper.parse_config(configured_composites, composite_configs)
return sensor_compositors, sensor_modifiers, id_keys
def load_yaml_config(self, conf):
"""Load a YAML configuration file and recursively update the overall configuration."""
with open(conf, 'r', encoding='utf-8') as fd:
self.config = recursive_dict_update(self.config, yaml.load(fd, Loader=UnsafeLoader))
def add_config_to_tree(self, *decision_dicts):
"""Add a configuration to the tree."""
conf = {}
for decision_dict in decision_dicts:
conf = recursive_dict_update(conf, decision_dict)
self._build_tree(conf)