def default_config(config, command_name, logger):
default_config = {}
# loading modules default config
fn = join(default_configs_root, 'modules.yaml')
default_modules_args = load_config_file(fn)
default_config['modules'] = {}
for module, module_config in config['modules'].items():
default_args = default_modules_args[module_config['_name']]
default_config['modules'][module] = default_args
# loading datasets default config
fn = join(default_configs_root, 'datasets.yaml')
default_datasets_args = load_config_file(fn)
default_config['datasets'] = {}
for dataset, dataset_config in config['datasets'].items():
default_args = default_datasets_args[dataset_config['_name']]
default_config['datasets'][dataset] = default_args
# loading experiment default configs
fn = join(default_configs_root, 'experiment.yaml')
default_experiment_args = load_config_file(fn)
default_args = default_experiment_args[config['experiment']['_name']]
default_config['experiment'] = default_args
return default_config
def run_named_config(self, config_name):
if os.path.exists(config_name):
nc = ConfigDict(load_config_file(config_name))
else:
nc = self.named_configs[config_name]
cfg = nc(fixed=self.get_config_updates_recursive(),
preset=self.presets,
fallback=self.fallback)
return undogmatize(cfg)
def run_named_config(self, config_name):
if os.path.exists(config_name):
nc = ConfigDict(load_config_file(config_name))
else:
nc = self.named_configs[config_name]
cfg = nc(fixed=self.config_updates,
preset=self.presets,
fallback=self.fallback)
return undogmatize(cfg)
def run_named_config(self, config_name):
if os.path.isfile(config_name):
nc = ConfigDict(load_config_file(config_name))
else:
if config_name not in self.named_configs:
raise NamedConfigNotFoundError(named_config=config_name,
available_named_configs=tuple(
self.named_configs.keys()))
nc = self.named_configs[config_name]
cfg = nc(fixed=self.get_config_updates_recursive(),
preset=self.presets,
fallback=self.fallback)
return undogmatize(cfg)
def add_config_file(self, filename):
"""
Read and add a configuration file to the configuration.
Supported formats so far are: ``json``, ``pickle`` and ``yaml``.
:param filename: The filename of the configuration file to be loaded.
Has to have the appropriate file-ending.
:type filename: str
"""
if not os.path.exists(filename):
raise IOError('File not found {}'.format(filename))
abspath = os.path.abspath(filename)
conf_dict = load_config_file(abspath)
self.add_config(conf_dict)
def get_component_configs(config, component_name, default_configs_file):
"""
:param config: The global config given by the user.
:param component_name: The key of the root-level element to process in the config.
:param default_configs_file: The path of the file containing the default configs for the current component.
:return: A dict containing the default configurations for each element under the given component.
"""
component_configs = {}
default_configs = load_config_file(default_configs_file)
for name, specified_config in config[component_name].items():
selected_config = specified_config['_name']
component_configs[name] = default_configs[selected_config]
return component_configs
def add_config_file(self, filename):
"""
Read and add a configuration file to the configuration.
Supported formats so far are: ``json``, ``pickle`` and ``yaml``.
:param filename: The filename of the configuration file to be loaded.
Has to have the appropriate file-ending.
:type filename: str
"""
if not os.path.exists(filename):
raise IOError('File not found {}'.format(filename))
abspath = os.path.abspath(filename)
conf_dict = load_config_file(abspath)
self.add_config(conf_dict)
def run_named_config(self, config_name):
if os.path.isfile(config_name):
nc = ConfigDict(load_config_file(config_name))
else:
if config_name not in self.named_configs:
raise NamedConfigNotFoundError(
named_config=config_name,
available_named_configs=tuple(self.named_configs.keys()))
nc = self.named_configs[config_name]
cfg = nc(fixed=self.get_config_updates_recursive(),
preset=self.presets,
fallback=self.fallback)
return undogmatize(cfg)
def default_config(config, command_name, logger):
default_config = {}
components = ['datasets', 'modules', 'optimizers']
for comp in components:
file_name = '{}.yaml'.format(comp)
default_file_path = join(default_configs_root, file_name)
default_config[comp] = get_component_configs(config, comp, default_file_path)
# loading experiment default configs
fn = join(default_configs_root, 'experiment.yaml')
default_experiment_args = load_config_file(fn)
default_args = default_experiment_args[config['experiment']['_name']]
default_config['experiment'] = default_args
return default_config
def load_default_config(config, command_name, logger):
default_config = {}
for comp, comp_conf in config.items():
fn = f'configs/default/{comp}.yaml'
if os.path.isfile(fn):# and ('_name' not in comp_conf):
comp_default_configs = load_config_file(fn)
if '_name' in comp_conf:
comp_default_config = load_component_default_config(comp_conf, comp_default_configs)
else:
comp_default_config = {}
for mod, mod_config in comp_conf.items():
if isinstance(mod_config, dict):
comp_default_config[mod] = load_component_default_config(mod_config, comp_default_configs)
default_config[comp] = comp_default_config
return default_config
def _create_config_dict(cfg_or_file, kw_conf):
if cfg_or_file is not None and kw_conf:
raise ValueError("cannot combine keyword config with "
"positional argument")
if cfg_or_file is None:
if not kw_conf:
raise ValueError("attempted to add empty config")
return ConfigDict(kw_conf)
elif isinstance(cfg_or_file, dict):
return ConfigDict(cfg_or_file)
elif isinstance(cfg_or_file, basestring):
if not os.path.exists(cfg_or_file):
raise IOError('File not found {}'.format(cfg_or_file))
abspath = os.path.abspath(cfg_or_file)
return ConfigDict(load_config_file(abspath))
else:
raise TypeError("Invalid argument type {}".format(
type(cfg_or_file)))
def _create_config_dict(cfg_or_file, kw_conf):
if cfg_or_file is not None and kw_conf:
raise ValueError("cannot combine keyword config with "
"positional argument")
if cfg_or_file is None:
if not kw_conf:
raise ValueError("attempted to add empty config")
return ConfigDict(kw_conf)
elif isinstance(cfg_or_file, dict):
return ConfigDict(cfg_or_file)
elif isinstance(cfg_or_file, basestring):
if not os.path.exists(cfg_or_file):
raise IOError('File not found {}'.format(cfg_or_file))
abspath = os.path.abspath(cfg_or_file)
return ConfigDict(load_config_file(abspath))
else:
raise TypeError("Invalid argument type {}"
.format(type(cfg_or_file)))
def set_up_config(self):
if self.config is not None:
return self.config
# gather presets
fallback = {}
for sr_path, subrunner in self.subrunners.items():
if self.path and is_prefix(self.path, sr_path):
path = sr_path[len(self.path):].strip('.')
set_by_dotted_path(fallback, path, subrunner.config)
else:
set_by_dotted_path(fallback, sr_path, subrunner.config)
# dogmatize to make the subrunner configurations read-only
const_fallback = dogmatize(fallback)
const_fallback.revelation()
self.config = {}
# named configs first
cfg_list = []
for ncfg in self.named_configs_to_use:
if os.path.exists(ncfg):
cfg_list.append(ConfigDict(load_config_file(ncfg)))
else:
cfg_list.append(self.named_configs[ncfg])
self.config_updates, _ = chain_evaluate_config_scopes(
cfg_list,
fixed=self.config_updates,
preset=self.config,
fallback=const_fallback)
# unnamed (default) configs second
self.config, self.summaries = chain_evaluate_config_scopes(
self.config_scopes,
fixed=self.config_updates,
preset=self.config,
fallback=const_fallback)
self.get_config_modifications()
def use_named_config(self, config_name):
if os.path.exists(config_name):
self.named_configs_to_use.append(
ConfigDict(load_config_file(config_name)))
else:
self.named_configs_to_use.append(self.named_configs[config_name])
def use_named_config(self, config_name):
if os.path.exists(config_name):
self.named_configs_to_use.append(
ConfigDict(load_config_file(config_name)))
else:
self.named_configs_to_use.append(self.named_configs[config_name])