def add_config_to_tree(self, *decision_dict): """Add configuration to tree.""" conf = {} for config_file in decision_dict: if os.path.isfile(config_file): with open(config_file) as fd: enhancement_config = yaml.load(fd, Loader=UnsafeLoader) if enhancement_config is None: # empty file continue enhancement_section = enhancement_config.get( self.prefix, {}) if not enhancement_section: LOG.debug( "Config '{}' has no '{}' section or it is empty". format(config_file, self.prefix)) continue conf = recursive_dict_update(conf, enhancement_section) elif isinstance(config_file, dict): conf = recursive_dict_update(conf, config_file) else: LOG.debug("Loading enhancement config string") d = yaml.load(config_file, Loader=UnsafeLoader) if not isinstance(d, dict): raise ValueError( "YAML file doesn't exist or string is not YAML dict: {}" .format(config_file)) conf = recursive_dict_update(conf, d) self._build_tree(conf)
def add_config_to_tree(self, *config_files): conf = {} for config_file in config_files: if os.path.isfile(config_file): with open(config_file) as fd: conf = recursive_dict_update(conf, yaml.load(fd)) else: LOG.debug("Loading enhancement config string") conf = recursive_dict_update(conf, yaml.load(config_file)) self._build_tree(conf)
def __init__(self, config_files, start_time=None, end_time=None, area=None): self.config = {} self._start_time = start_time self._end_time = end_time self._area = area self.config_files = config_files for config_file in config_files: with open(config_file) as fd: self.config = recursive_dict_update(self.config, yaml.load(fd)) self.info = self.config['reader'] self.name = self.info['name'] self.file_patterns = [] for file_type in self.config['file_types'].values(): # correct separator if needed file_patterns = [ os.path.join(*pattern.split('/')) for pattern in file_type['file_patterns'] ] self.file_patterns.extend(file_patterns) if not isinstance(self.info['sensors'], (list, tuple)): self.info['sensors'] = [self.info['sensors']] self.sensor_names = self.info['sensors'] self.datasets = self.config['datasets'] self.info['filenames'] = [] self.ids = {} self.load_ds_ids_from_config()
def _load_config(self, composite_configs, **kwargs): if not isinstance(composite_configs, (list, tuple)): composite_configs = [composite_configs] conf = {} for composite_config in composite_configs: with open(composite_config) as conf_file: conf = recursive_dict_update(conf, yaml.load(conf_file)) try: sensor_name = conf['sensor_name'] except KeyError: LOG.debug('No "sensor_name" tag found in %s, skipping.', composite_config) return sensor_id = sensor_name.split('/')[-1] sensor_deps = sensor_name.split('/')[:-1] compositors = self.compositors.setdefault(sensor_id, DatasetDict()) modifiers = self.modifiers.setdefault(sensor_id, {}) for sensor_dep in reversed(sensor_deps): if sensor_dep not in self.compositors or sensor_dep not in self.modifiers: self.load_sensor_composites(sensor_dep) if sensor_deps: compositors.update(self.compositors[sensor_deps[-1]]) modifiers.update(self.modifiers[sensor_deps[-1]]) for composite_type in ['modifiers', 'composites']: if composite_type not in conf: continue for composite_name in conf[composite_type]: self._process_composite_config(composite_name, conf, composite_type, sensor_id, composite_config, **kwargs)
def __init__(self, config_files, start_time=None, end_time=None, area=None): self.config = {} self._start_time = start_time self._end_time = end_time self._area = area self.config_files = config_files for config_file in config_files: with open(config_file) as fd: self.config = recursive_dict_update(self.config, yaml.load(fd)) self.info = self.config['reader'] self.name = self.info['name'] self.file_patterns = [] for file_type in self.config['file_types'].values(): self.file_patterns.extend(file_type['file_patterns']) self.sensor_names = self.info['sensors'] self.datasets = self.config['datasets'] self.info['filenames'] = [] self.ids = {} self.get_dataset_ids()
def __init__(self, config_files): """Load information from YAML configuration file about how to read data files.""" self.config = {} self.config_files = config_files for config_file in config_files: with open(config_file) as fd: self.config = recursive_dict_update(self.config, yaml.load(fd, Loader=UnsafeLoader)) self.info = self.config['reader'] self.name = self.info['name'] self.file_patterns = [] for file_type, filetype_info in self.config['file_types'].items(): filetype_info.setdefault('file_type', file_type) # correct separator if needed file_patterns = [os.path.join(*pattern.split('/')) for pattern in filetype_info['file_patterns']] filetype_info['file_patterns'] = file_patterns self.file_patterns.extend(file_patterns) if 'sensors' in self.info and not isinstance(self.info['sensors'], (list, tuple)): self.info['sensors'] = [self.info['sensors']] self.datasets = self.config.get('datasets', {}) self._id_keys = self.info.get('data_identification_keys', default_id_keys_config) self._co_keys = self.info.get('coord_identification_keys', default_co_keys_config) self.info['filenames'] = [] self.all_ids = {} self.load_ds_ids_from_config()
def __init__(self, config_files): self.config = {} self.config_files = config_files for config_file in config_files: with open(config_file) as fd: self.config = recursive_dict_update(self.config, yaml.load(fd)) self.info = self.config['reader'] self.name = self.info['name'] self.file_patterns = [] for file_type, filetype_info in self.config['file_types'].items(): filetype_info.setdefault('file_type', file_type) # correct separator if needed file_patterns = [ os.path.join(*pattern.split('/')) for pattern in filetype_info['file_patterns'] ] filetype_info['file_patterns'] = file_patterns self.file_patterns.extend(file_patterns) if not isinstance(self.info['sensors'], (list, tuple)): self.info['sensors'] = [self.info['sensors']] self.datasets = self.config.get('datasets', {}) self.info['filenames'] = [] self.ids = {} self.load_ds_ids_from_config()
def add_config_to_tree(self, *decision_dict): conf = {} for config_file in decision_dict: if os.path.isfile(config_file): with open(config_file) as fd: conf = recursive_dict_update(conf, yaml.load(fd)[self.prefix]) elif isinstance(config_file, dict): conf = recursive_dict_update(conf, config_file) else: LOG.debug("Loading enhancement config string") d = yaml.load(config_file) if not isinstance(d, dict): raise ValueError( "YAML file doesn't exist or string is not YAML dict: {}" .format(config_file)) conf = recursive_dict_update(conf, d) self._build_tree(conf)
def load_writer_config(self, config_files, **kwargs): conf = {} for conf_fn in config_files: with open(conf_fn) as fd: conf = recursive_dict_update(conf, yaml.load(fd)) writer_class = conf['writer']['writer'] writer = writer_class(ppp_config_dir=self.ppp_config_dir, config_files=config_files, **kwargs) return writer
def add_config_to_tree(self, *decision_dict): conf = {} for config_file in decision_dict: if os.path.isfile(config_file): with open(config_file) as fd: enhancement_section = yaml.load(fd).get(self.prefix, {}) if not enhancement_section: LOG.debug("Config '{}' has no '{}' section or it is empty".format(config_file, self.prefix)) continue conf = recursive_dict_update(conf, enhancement_section) elif isinstance(config_file, dict): conf = recursive_dict_update(conf, config_file) else: LOG.debug("Loading enhancement config string") d = yaml.load(config_file) if not isinstance(d, dict): raise ValueError( "YAML file doesn't exist or string is not YAML dict: {}".format(config_file)) conf = recursive_dict_update(conf, d) self._build_tree(conf)
def _load_config(self, composite_configs): if not isinstance(composite_configs, (list, tuple)): composite_configs = [composite_configs] conf = {} for composite_config in composite_configs: with open(composite_config, 'r', encoding='utf-8') as conf_file: conf = recursive_dict_update( conf, yaml.load(conf_file, Loader=UnsafeLoader)) try: sensor_name = conf['sensor_name'] except KeyError: logger.debug('No "sensor_name" tag found in %s, skipping.', composite_configs) return sensor_id = sensor_name.split('/')[-1] sensor_deps = sensor_name.split('/')[:-1] compositors = self.compositors.setdefault(sensor_id, DatasetDict()) modifiers = self.modifiers.setdefault(sensor_id, {}) for sensor_dep in reversed(sensor_deps): if sensor_dep not in self.compositors or sensor_dep not in self.modifiers: self.load_sensor_composites(sensor_dep) if sensor_deps: compositors.update(self.compositors[sensor_deps[-1]]) modifiers.update(self.modifiers[sensor_deps[-1]]) id_keys = self._get_sensor_id_keys(conf, sensor_id, sensor_deps) mod_config_helper = _ModifierConfigHelper(modifiers, id_keys) configured_modifiers = conf.get('modifiers', {}) mod_config_helper.parse_config(configured_modifiers, composite_configs) comp_config_helper = _CompositeConfigHelper(compositors, id_keys) configured_composites = conf.get('composites', {}) comp_config_helper.parse_config(configured_composites, composite_configs)
def __init__(self, config_files): self.config = {} self.config_files = config_files for config_file in config_files: with open(config_file) as fd: self.config = recursive_dict_update(self.config, yaml.load(fd)) self.info = self.config['reader'] self.name = self.info['name'] self.file_patterns = [] for file_type, filetype_info in self.config['file_types'].items(): filetype_info.setdefault('file_type', file_type) # correct separator if needed file_patterns = [os.path.join(*pattern.split('/')) for pattern in filetype_info['file_patterns']] filetype_info['file_patterns'] = file_patterns self.file_patterns.extend(file_patterns) if not isinstance(self.info['sensors'], (list, tuple)): self.info['sensors'] = [self.info['sensors']] self.datasets = self.config.get('datasets', {}) self.info['filenames'] = [] self.ids = {} self.load_ds_ids_from_config()
def add_config_to_tree(self, *decision_dicts): conf = {} for decision_dict in decision_dicts: conf = recursive_dict_update(conf, decision_dict) self._build_tree(conf)
def load_yaml_config(self, conf): with open(conf) as fd: self.config = recursive_dict_update(self.config, yaml.load(fd))
def load_yaml_config(self, conf): """Load a YAML configuration file and recursively update the overall configuration.""" with open(conf) as fd: self.config = recursive_dict_update( self.config, yaml.load(fd, Loader=UnsafeLoader))
def add_config_to_tree(self, *decision_dicts): """Add a configuration to the tree.""" conf = {} for decision_dict in decision_dicts: conf = recursive_dict_update(conf, decision_dict) self._build_tree(conf)