def add_config_to_tree(self, *decision_dict):
"""Add configuration to tree."""
conf = {}
for config_file in decision_dict:
if os.path.isfile(config_file):
with open(config_file) as fd:
enhancement_config = yaml.load(fd, Loader=UnsafeLoader)
if enhancement_config is None:
# empty file
continue
enhancement_section = enhancement_config.get(
self.prefix, {})
if not enhancement_section:
LOG.debug(
"Config '{}' has no '{}' section or it is empty".
format(config_file, self.prefix))
continue
conf = recursive_dict_update(conf, enhancement_section)
elif isinstance(config_file, dict):
conf = recursive_dict_update(conf, config_file)
else:
LOG.debug("Loading enhancement config string")
d = yaml.load(config_file, Loader=UnsafeLoader)
if not isinstance(d, dict):
raise ValueError(
"YAML file doesn't exist or string is not YAML dict: {}"
.format(config_file))
conf = recursive_dict_update(conf, d)
self._build_tree(conf)
def add_config_to_tree(self, *config_files):
conf = {}
for config_file in config_files:
if os.path.isfile(config_file):
with open(config_file) as fd:
conf = recursive_dict_update(conf, yaml.load(fd))
else:
LOG.debug("Loading enhancement config string")
conf = recursive_dict_update(conf, yaml.load(config_file))
self._build_tree(conf)
def __init__(self,
config_files,
start_time=None,
end_time=None,
area=None):
self.config = {}
self._start_time = start_time
self._end_time = end_time
self._area = area
self.config_files = config_files
for config_file in config_files:
with open(config_file) as fd:
self.config = recursive_dict_update(self.config, yaml.load(fd))
self.info = self.config['reader']
self.name = self.info['name']
self.file_patterns = []
for file_type in self.config['file_types'].values():
# correct separator if needed
file_patterns = [
os.path.join(*pattern.split('/'))
for pattern in file_type['file_patterns']
]
self.file_patterns.extend(file_patterns)
if not isinstance(self.info['sensors'], (list, tuple)):
self.info['sensors'] = [self.info['sensors']]
self.sensor_names = self.info['sensors']
self.datasets = self.config['datasets']
self.info['filenames'] = []
self.ids = {}
self.load_ds_ids_from_config()
def _load_config(self, composite_configs, **kwargs):
if not isinstance(composite_configs, (list, tuple)):
composite_configs = [composite_configs]
conf = {}
for composite_config in composite_configs:
with open(composite_config) as conf_file:
conf = recursive_dict_update(conf, yaml.load(conf_file))
try:
sensor_name = conf['sensor_name']
except KeyError:
LOG.debug('No "sensor_name" tag found in %s, skipping.',
composite_config)
return
sensor_id = sensor_name.split('/')[-1]
sensor_deps = sensor_name.split('/')[:-1]
compositors = self.compositors.setdefault(sensor_id, DatasetDict())
modifiers = self.modifiers.setdefault(sensor_id, {})
for sensor_dep in reversed(sensor_deps):
if sensor_dep not in self.compositors or sensor_dep not in self.modifiers:
self.load_sensor_composites(sensor_dep)
if sensor_deps:
compositors.update(self.compositors[sensor_deps[-1]])
modifiers.update(self.modifiers[sensor_deps[-1]])
for composite_type in ['modifiers', 'composites']:
if composite_type not in conf:
continue
for composite_name in conf[composite_type]:
self._process_composite_config(composite_name, conf,
composite_type, sensor_id, composite_config, **kwargs)
def __init__(self,
config_files,
start_time=None,
end_time=None,
area=None):
self.config = {}
self._start_time = start_time
self._end_time = end_time
self._area = area
self.config_files = config_files
for config_file in config_files:
with open(config_file) as fd:
self.config = recursive_dict_update(self.config, yaml.load(fd))
self.info = self.config['reader']
self.name = self.info['name']
self.file_patterns = []
for file_type in self.config['file_types'].values():
self.file_patterns.extend(file_type['file_patterns'])
self.sensor_names = self.info['sensors']
self.datasets = self.config['datasets']
self.info['filenames'] = []
self.ids = {}
self.get_dataset_ids()
def __init__(self, config_files):
"""Load information from YAML configuration file about how to read data files."""
self.config = {}
self.config_files = config_files
for config_file in config_files:
with open(config_file) as fd:
self.config = recursive_dict_update(self.config, yaml.load(fd, Loader=UnsafeLoader))
self.info = self.config['reader']
self.name = self.info['name']
self.file_patterns = []
for file_type, filetype_info in self.config['file_types'].items():
filetype_info.setdefault('file_type', file_type)
# correct separator if needed
file_patterns = [os.path.join(*pattern.split('/'))
for pattern in filetype_info['file_patterns']]
filetype_info['file_patterns'] = file_patterns
self.file_patterns.extend(file_patterns)
if 'sensors' in self.info and not isinstance(self.info['sensors'], (list, tuple)):
self.info['sensors'] = [self.info['sensors']]
self.datasets = self.config.get('datasets', {})
self._id_keys = self.info.get('data_identification_keys', default_id_keys_config)
self._co_keys = self.info.get('coord_identification_keys', default_co_keys_config)
self.info['filenames'] = []
self.all_ids = {}
self.load_ds_ids_from_config()
def __init__(self, config_files):
self.config = {}
self.config_files = config_files
for config_file in config_files:
with open(config_file) as fd:
self.config = recursive_dict_update(self.config, yaml.load(fd))
self.info = self.config['reader']
self.name = self.info['name']
self.file_patterns = []
for file_type, filetype_info in self.config['file_types'].items():
filetype_info.setdefault('file_type', file_type)
# correct separator if needed
file_patterns = [
os.path.join(*pattern.split('/'))
for pattern in filetype_info['file_patterns']
]
filetype_info['file_patterns'] = file_patterns
self.file_patterns.extend(file_patterns)
if not isinstance(self.info['sensors'], (list, tuple)):
self.info['sensors'] = [self.info['sensors']]
self.datasets = self.config.get('datasets', {})
self.info['filenames'] = []
self.ids = {}
self.load_ds_ids_from_config()
def _load_config(self, composite_configs, **kwargs):
if not isinstance(composite_configs, (list, tuple)):
composite_configs = [composite_configs]
conf = {}
for composite_config in composite_configs:
with open(composite_config) as conf_file:
conf = recursive_dict_update(conf, yaml.load(conf_file))
try:
sensor_name = conf['sensor_name']
except KeyError:
LOG.debug('No "sensor_name" tag found in %s, skipping.',
composite_config)
return
sensor_id = sensor_name.split('/')[-1]
sensor_deps = sensor_name.split('/')[:-1]
compositors = self.compositors.setdefault(sensor_id, DatasetDict())
modifiers = self.modifiers.setdefault(sensor_id, {})
for sensor_dep in reversed(sensor_deps):
if sensor_dep not in self.compositors or sensor_dep not in self.modifiers:
self.load_sensor_composites(sensor_dep)
if sensor_deps:
compositors.update(self.compositors[sensor_deps[-1]])
modifiers.update(self.modifiers[sensor_deps[-1]])
for composite_type in ['modifiers', 'composites']:
if composite_type not in conf:
continue
for composite_name in conf[composite_type]:
self._process_composite_config(composite_name, conf,
composite_type, sensor_id, composite_config, **kwargs)
def add_config_to_tree(self, *decision_dict):
conf = {}
for config_file in decision_dict:
if os.path.isfile(config_file):
with open(config_file) as fd:
conf = recursive_dict_update(conf,
yaml.load(fd)[self.prefix])
elif isinstance(config_file, dict):
conf = recursive_dict_update(conf, config_file)
else:
LOG.debug("Loading enhancement config string")
d = yaml.load(config_file)
if not isinstance(d, dict):
raise ValueError(
"YAML file doesn't exist or string is not YAML dict: {}"
.format(config_file))
conf = recursive_dict_update(conf, d)
self._build_tree(conf)
def load_writer_config(self, config_files, **kwargs):
conf = {}
for conf_fn in config_files:
with open(conf_fn) as fd:
conf = recursive_dict_update(conf, yaml.load(fd))
writer_class = conf['writer']['writer']
writer = writer_class(ppp_config_dir=self.ppp_config_dir,
config_files=config_files,
**kwargs)
return writer
def add_config_to_tree(self, *decision_dict):
conf = {}
for config_file in decision_dict:
if os.path.isfile(config_file):
with open(config_file) as fd:
enhancement_section = yaml.load(fd).get(self.prefix, {})
if not enhancement_section:
LOG.debug("Config '{}' has no '{}' section or it is empty".format(config_file, self.prefix))
continue
conf = recursive_dict_update(conf, enhancement_section)
elif isinstance(config_file, dict):
conf = recursive_dict_update(conf, config_file)
else:
LOG.debug("Loading enhancement config string")
d = yaml.load(config_file)
if not isinstance(d, dict):
raise ValueError(
"YAML file doesn't exist or string is not YAML dict: {}".format(config_file))
conf = recursive_dict_update(conf, d)
self._build_tree(conf)
def _load_config(self, composite_configs):
if not isinstance(composite_configs, (list, tuple)):
composite_configs = [composite_configs]
conf = {}
for composite_config in composite_configs:
with open(composite_config, 'r', encoding='utf-8') as conf_file:
conf = recursive_dict_update(
conf, yaml.load(conf_file, Loader=UnsafeLoader))
try:
sensor_name = conf['sensor_name']
except KeyError:
logger.debug('No "sensor_name" tag found in %s, skipping.',
composite_configs)
return
sensor_id = sensor_name.split('/')[-1]
sensor_deps = sensor_name.split('/')[:-1]
compositors = self.compositors.setdefault(sensor_id, DatasetDict())
modifiers = self.modifiers.setdefault(sensor_id, {})
for sensor_dep in reversed(sensor_deps):
if sensor_dep not in self.compositors or sensor_dep not in self.modifiers:
self.load_sensor_composites(sensor_dep)
if sensor_deps:
compositors.update(self.compositors[sensor_deps[-1]])
modifiers.update(self.modifiers[sensor_deps[-1]])
id_keys = self._get_sensor_id_keys(conf, sensor_id, sensor_deps)
mod_config_helper = _ModifierConfigHelper(modifiers, id_keys)
configured_modifiers = conf.get('modifiers', {})
mod_config_helper.parse_config(configured_modifiers, composite_configs)
comp_config_helper = _CompositeConfigHelper(compositors, id_keys)
configured_composites = conf.get('composites', {})
comp_config_helper.parse_config(configured_composites,
composite_configs)
def __init__(self, config_files):
self.config = {}
self.config_files = config_files
for config_file in config_files:
with open(config_file) as fd:
self.config = recursive_dict_update(self.config, yaml.load(fd))
self.info = self.config['reader']
self.name = self.info['name']
self.file_patterns = []
for file_type, filetype_info in self.config['file_types'].items():
filetype_info.setdefault('file_type', file_type)
# correct separator if needed
file_patterns = [os.path.join(*pattern.split('/'))
for pattern in filetype_info['file_patterns']]
filetype_info['file_patterns'] = file_patterns
self.file_patterns.extend(file_patterns)
if not isinstance(self.info['sensors'], (list, tuple)):
self.info['sensors'] = [self.info['sensors']]
self.datasets = self.config.get('datasets', {})
self.info['filenames'] = []
self.ids = {}
self.load_ds_ids_from_config()
def __init__(self, config_files,
start_time=None,
end_time=None,
area=None):
self.config = {}
self._start_time = start_time
self._end_time = end_time
self._area = area
self.config_files = config_files
for config_file in config_files:
with open(config_file) as fd:
self.config = recursive_dict_update(self.config, yaml.load(fd))
self.info = self.config['reader']
self.name = self.info['name']
self.file_patterns = []
for file_type in self.config['file_types'].values():
self.file_patterns.extend(file_type['file_patterns'])
self.sensor_names = self.info['sensors']
self.datasets = self.config['datasets']
self.info['filenames'] = []
self.ids = {}
self.get_dataset_ids()
def add_config_to_tree(self, *decision_dicts):
conf = {}
for decision_dict in decision_dicts:
conf = recursive_dict_update(conf, decision_dict)
self._build_tree(conf)
def load_yaml_config(self, conf):
with open(conf) as fd:
self.config = recursive_dict_update(self.config, yaml.load(fd))
def load_yaml_config(self, conf):
"""Load a YAML configuration file and recursively update the overall configuration."""
with open(conf) as fd:
self.config = recursive_dict_update(
self.config, yaml.load(fd, Loader=UnsafeLoader))
def load_yaml_config(self, conf):
with open(conf) as fd:
self.config = recursive_dict_update(self.config, yaml.load(fd))
def add_config_to_tree(self, *decision_dicts):
conf = {}
for decision_dict in decision_dicts:
conf = recursive_dict_update(conf, decision_dict)
self._build_tree(conf)
def add_config_to_tree(self, *decision_dicts):
"""Add a configuration to the tree."""
conf = {}
for decision_dict in decision_dicts:
conf = recursive_dict_update(conf, decision_dict)
self._build_tree(conf)
