def default_config(config, command_name, logger): default_config = {} # loading modules default config fn = join(default_configs_root, 'modules.yaml') default_modules_args = load_config_file(fn) default_config['modules'] = {} for module, module_config in config['modules'].items(): default_args = default_modules_args[module_config['_name']] default_config['modules'][module] = default_args # loading datasets default config fn = join(default_configs_root, 'datasets.yaml') default_datasets_args = load_config_file(fn) default_config['datasets'] = {} for dataset, dataset_config in config['datasets'].items(): default_args = default_datasets_args[dataset_config['_name']] default_config['datasets'][dataset] = default_args # loading experiment default configs fn = join(default_configs_root, 'experiment.yaml') default_experiment_args = load_config_file(fn) default_args = default_experiment_args[config['experiment']['_name']] default_config['experiment'] = default_args return default_config
def run_named_config(self, config_name): if os.path.exists(config_name): nc = ConfigDict(load_config_file(config_name)) else: nc = self.named_configs[config_name] cfg = nc(fixed=self.get_config_updates_recursive(), preset=self.presets, fallback=self.fallback) return undogmatize(cfg)
def run_named_config(self, config_name): if os.path.exists(config_name): nc = ConfigDict(load_config_file(config_name)) else: nc = self.named_configs[config_name] cfg = nc(fixed=self.config_updates, preset=self.presets, fallback=self.fallback) return undogmatize(cfg)
def run_named_config(self, config_name): if os.path.isfile(config_name): nc = ConfigDict(load_config_file(config_name)) else: if config_name not in self.named_configs: raise NamedConfigNotFoundError(named_config=config_name, available_named_configs=tuple( self.named_configs.keys())) nc = self.named_configs[config_name] cfg = nc(fixed=self.get_config_updates_recursive(), preset=self.presets, fallback=self.fallback) return undogmatize(cfg)
def add_config_file(self, filename): """ Read and add a configuration file to the configuration. Supported formats so far are: ``json``, ``pickle`` and ``yaml``. :param filename: The filename of the configuration file to be loaded. Has to have the appropriate file-ending. :type filename: str """ if not os.path.exists(filename): raise IOError('File not found {}'.format(filename)) abspath = os.path.abspath(filename) conf_dict = load_config_file(abspath) self.add_config(conf_dict)
def get_component_configs(config, component_name, default_configs_file): """ :param config: The global config given by the user. :param component_name: The key of the root-level element to process in the config. :param default_configs_file: The path of the file containing the default configs for the current component. :return: A dict containing the default configurations for each element under the given component. """ component_configs = {} default_configs = load_config_file(default_configs_file) for name, specified_config in config[component_name].items(): selected_config = specified_config['_name'] component_configs[name] = default_configs[selected_config] return component_configs
def run_named_config(self, config_name): if os.path.isfile(config_name): nc = ConfigDict(load_config_file(config_name)) else: if config_name not in self.named_configs: raise NamedConfigNotFoundError( named_config=config_name, available_named_configs=tuple(self.named_configs.keys())) nc = self.named_configs[config_name] cfg = nc(fixed=self.get_config_updates_recursive(), preset=self.presets, fallback=self.fallback) return undogmatize(cfg)
def default_config(config, command_name, logger): default_config = {} components = ['datasets', 'modules', 'optimizers'] for comp in components: file_name = '{}.yaml'.format(comp) default_file_path = join(default_configs_root, file_name) default_config[comp] = get_component_configs(config, comp, default_file_path) # loading experiment default configs fn = join(default_configs_root, 'experiment.yaml') default_experiment_args = load_config_file(fn) default_args = default_experiment_args[config['experiment']['_name']] default_config['experiment'] = default_args return default_config
def load_default_config(config, command_name, logger): default_config = {} for comp, comp_conf in config.items(): fn = f'configs/default/{comp}.yaml' if os.path.isfile(fn):# and ('_name' not in comp_conf): comp_default_configs = load_config_file(fn) if '_name' in comp_conf: comp_default_config = load_component_default_config(comp_conf, comp_default_configs) else: comp_default_config = {} for mod, mod_config in comp_conf.items(): if isinstance(mod_config, dict): comp_default_config[mod] = load_component_default_config(mod_config, comp_default_configs) default_config[comp] = comp_default_config return default_config
def _create_config_dict(cfg_or_file, kw_conf): if cfg_or_file is not None and kw_conf: raise ValueError("cannot combine keyword config with " "positional argument") if cfg_or_file is None: if not kw_conf: raise ValueError("attempted to add empty config") return ConfigDict(kw_conf) elif isinstance(cfg_or_file, dict): return ConfigDict(cfg_or_file) elif isinstance(cfg_or_file, basestring): if not os.path.exists(cfg_or_file): raise IOError('File not found {}'.format(cfg_or_file)) abspath = os.path.abspath(cfg_or_file) return ConfigDict(load_config_file(abspath)) else: raise TypeError("Invalid argument type {}".format( type(cfg_or_file)))
def _create_config_dict(cfg_or_file, kw_conf): if cfg_or_file is not None and kw_conf: raise ValueError("cannot combine keyword config with " "positional argument") if cfg_or_file is None: if not kw_conf: raise ValueError("attempted to add empty config") return ConfigDict(kw_conf) elif isinstance(cfg_or_file, dict): return ConfigDict(cfg_or_file) elif isinstance(cfg_or_file, basestring): if not os.path.exists(cfg_or_file): raise IOError('File not found {}'.format(cfg_or_file)) abspath = os.path.abspath(cfg_or_file) return ConfigDict(load_config_file(abspath)) else: raise TypeError("Invalid argument type {}" .format(type(cfg_or_file)))
def set_up_config(self): if self.config is not None: return self.config # gather presets fallback = {} for sr_path, subrunner in self.subrunners.items(): if self.path and is_prefix(self.path, sr_path): path = sr_path[len(self.path):].strip('.') set_by_dotted_path(fallback, path, subrunner.config) else: set_by_dotted_path(fallback, sr_path, subrunner.config) # dogmatize to make the subrunner configurations read-only const_fallback = dogmatize(fallback) const_fallback.revelation() self.config = {} # named configs first cfg_list = [] for ncfg in self.named_configs_to_use: if os.path.exists(ncfg): cfg_list.append(ConfigDict(load_config_file(ncfg))) else: cfg_list.append(self.named_configs[ncfg]) self.config_updates, _ = chain_evaluate_config_scopes( cfg_list, fixed=self.config_updates, preset=self.config, fallback=const_fallback) # unnamed (default) configs second self.config, self.summaries = chain_evaluate_config_scopes( self.config_scopes, fixed=self.config_updates, preset=self.config, fallback=const_fallback) self.get_config_modifications()
def use_named_config(self, config_name): if os.path.exists(config_name): self.named_configs_to_use.append( ConfigDict(load_config_file(config_name))) else: self.named_configs_to_use.append(self.named_configs[config_name])