def from_yaml(cls, f, resolve_imports=True):
"""
Returns an AttrDict initialized from the given path or
file object ``f``, which must point to a YAML file.
If ``resolve_imports`` is True, ``import:`` statements are
resolved recursively, else they are treated like any other key.
When resolving import statements, anything defined locally
overrides definitions in the imported file.
"""
if isinstance(f, str):
with open(f, 'r') as src:
loaded = cls(_yaml_load(src))
else:
loaded = cls(_yaml_load(f))
if resolve_imports and 'import' in loaded:
for k in loaded['import']:
imported = cls.from_yaml(relative_path(f, k))
# loaded is added to imported (i.e. it takes precedence)
imported.union(loaded)
loaded = imported
# 'import' key no longer needed, so we drop it
loaded.pop('import', None)
return loaded
def _resolve_imports(cls, loaded, resolve_imports, base_path=None):
if (isinstance(resolve_imports, bool) and resolve_imports is True
and "import" in loaded):
loaded_dict = loaded
elif (isinstance(resolve_imports, str)
and resolve_imports + ".import" in loaded.keys_nested()):
loaded_dict = loaded.get_key(resolve_imports)
else: # Return right away if no importing to be done
return loaded
# If we end up here, we have something to import
imports = loaded_dict.get_key("import")
if not isinstance(imports, list):
raise ValueError("`import` must be a list.")
for k in imports:
if base_path:
path = relative_path(base_path, k)
else:
path = k
imported = cls.from_yaml(path)
# loaded is added to imported (i.e. it takes precedence)
imported.union(loaded_dict)
loaded_dict = imported
# 'import' key itself is no longer needed
loaded_dict.del_key("import")
if isinstance(resolve_imports, str):
loaded.set_key(resolve_imports, loaded_dict)
else:
loaded = loaded_dict
return loaded
def _resolve_imports(cls, loaded, resolve_imports, base_path=None):
if isinstance(resolve_imports, bool) and resolve_imports is True and 'import' in loaded:
loaded_dict = loaded
elif isinstance(resolve_imports, str) and resolve_imports + '.import' in loaded.keys_nested():
loaded_dict = loaded.get_key(resolve_imports)
else: # Return right away if no importing to be done
return loaded
# If we end up here, we have something to import
imports = loaded_dict.get_key('import')
if not isinstance(imports, list):
raise ValueError('`import` must be a list.')
for k in imports:
if base_path:
path = relative_path(base_path, k)
else:
path = k
imported = cls.from_yaml(path)
# loaded is added to imported (i.e. it takes precedence)
imported.union(loaded_dict)
loaded_dict = imported
# 'import' key itself is no longer needed
loaded_dict.del_key('import')
if isinstance(resolve_imports, str):
loaded.set_key(resolve_imports, loaded_dict)
else:
loaded = loaded_dict
return loaded
def _resolve_imports(cls, loaded, resolve_imports, base_path=None):
if resolve_imports and 'import' in loaded:
for k in loaded['import']:
if base_path:
path = relative_path(base_path, k)
else:
path = k
imported = cls.from_yaml(path)
# loaded is added to imported (i.e. it takes precedence)
imported.union(loaded)
loaded = imported
# 'import' key no longer needed, so we drop it
loaded.pop('import', None)
return loaded
def apply_overrides(config, scenario=None, override_dict=None):
"""
Generate processed Model configuration, applying any scenarios overrides.
Parameters
----------
config : AttrDict
a model configuration AttrDict
scenario : str, optional
override_dict : str or dict or AttrDict, optional
If a YAML string, converted to AttrDict
"""
debug_comments = AttrDict()
base_model_config_file = os.path.join(
os.path.dirname(calliope.__file__),
'config', 'model.yaml'
)
config_model = AttrDict.from_yaml(base_model_config_file)
# Interpret timeseries_data_path as relative
config.model.timeseries_data_path = relative_path(
config.config_path, config.model.timeseries_data_path
)
# The input files are allowed to override other model defaults
config_model.union(config, allow_override=True)
# First pass of applying override dict before applying scenarios,
# so that can override scenario definitions by override_dict
if override_dict:
if isinstance(override_dict, str):
override_dict = AttrDict.from_yaml_string(override_dict)
elif not isinstance(override_dict, AttrDict):
override_dict = AttrDict(override_dict)
warnings = checks.check_overrides(config_model, override_dict)
exceptions.print_warnings_and_raise_errors(warnings=warnings)
config_model.union(
override_dict, allow_override=True, allow_replacement=True
)
if scenario:
scenarios = config_model.get('scenarios', {})
if scenario in scenarios:
# Manually defined scenario names cannot be the same as single
# overrides or any combination of semicolon-delimited overrides
if all([i in config_model.get('overrides', {})
for i in scenario.split(',')]):
raise exceptions.ModelError(
'Manually defined scenario cannot be a combination of override names.'
)
if not isinstance(scenarios[scenario], str):
raise exceptions.ModelError(
'Scenario definition must be string of comma-separated overrides.'
)
overrides = scenarios[scenario].split(',')
logger.info(
'Using scenario `{}` leading to the application of '
'overrides `{}`.'.format(scenario, scenarios[scenario])
)
else:
overrides = str(scenario).split(',')
logger.info(
'Applying overrides `{}` without a '
'specific scenario name.'.format(scenario)
)
overrides_from_scenario = combine_overrides(config_model, overrides)
warnings = checks.check_overrides(config_model, overrides_from_scenario)
exceptions.print_warnings_and_raise_errors(warnings=warnings)
config_model.union(
overrides_from_scenario, allow_override=True, allow_replacement=True
)
for k, v in overrides_from_scenario.as_dict_flat().items():
debug_comments.set_key(
'{}'.format(k),
'Applied from override')
else:
overrides = []
# Second pass of applying override dict after applying scenarios,
# so that scenario-based overrides are overridden by override_dict!
if override_dict:
config_model.union(
override_dict, allow_override=True, allow_replacement=True
)
for k, v in override_dict.as_dict_flat().items():
debug_comments.set_key(
'{}'.format(k),
'Overridden via override dictionary.')
return config_model, debug_comments, overrides, scenario
def apply_overrides(config, scenario=None, override_dict=None):
"""
Generate processed Model configuration, applying any scenarios overrides.
Parameters
----------
config : AttrDict
a model configuration AttrDict
scenario : str, optional
override_dict : str or dict or AttrDict, optional
If a YAML string, converted to AttrDict
"""
debug_comments = AttrDict()
config_model = AttrDict.from_yaml(
os.path.join(os.path.dirname(calliope.__file__), "config",
"defaults.yaml"))
# Interpret timeseries_data_path as relative
if "timeseries_data_path" in config.model:
config.model.timeseries_data_path = relative_path(
config.config_path, config.model.timeseries_data_path)
# FutureWarning: check if config includes an explicit objective cost class.
# Added in 0.6.4-dev, to be removed in v0.7.0-dev.
has_explicit_cost_class = isinstance(
config.get_key("run.objective_options.cost_class", None), dict)
# The input files are allowed to override other model defaults
config_model.union(config, allow_override=True)
# First pass of applying override dict before applying scenarios,
# so that can override scenario definitions by override_dict
if override_dict:
if isinstance(override_dict, str):
override_dict = AttrDict.from_yaml_string(override_dict)
elif not isinstance(override_dict, AttrDict):
override_dict = AttrDict(override_dict)
warning_messages = checks.check_overrides(config_model, override_dict)
exceptions.print_warnings_and_raise_errors(warnings=warning_messages)
# FutureWarning: If config does not include an explicit objective cost class, check override dict.
# Added in 0.6.4-dev, to be removed in v0.7.0-dev.
if has_explicit_cost_class is False:
has_explicit_cost_class = isinstance(
override_dict.get_key("run.objective_options.cost_class",
None), dict)
config_model.union(override_dict,
allow_override=True,
allow_replacement=True)
if scenario:
scenario_overrides = load_overrides_from_scenario(
config_model, scenario)
if not all(i in config_model.get("overrides", {})
for i in scenario_overrides):
raise exceptions.ModelError(
"Scenario definition must be a list of override or other scenario names."
)
else:
logger.info(
"Applying the following overrides from scenario definition: {} "
.format(scenario_overrides))
overrides_from_scenario = combine_overrides(config_model,
scenario_overrides)
warning_messages = checks.check_overrides(config_model,
overrides_from_scenario)
exceptions.print_warnings_and_raise_errors(warnings=warning_messages)
# FutureWarning: If config nor override_dict include an explicit objective cost class, check scenario dict.
# Added in 0.6.4-dev, to be removed in v0.7.0-dev
if has_explicit_cost_class is False:
has_explicit_cost_class = isinstance(
overrides_from_scenario.get_key(
"run.objective_options.cost_class", None),
dict,
)
config_model.union(overrides_from_scenario,
allow_override=True,
allow_replacement=True)
for k, v in overrides_from_scenario.as_dict_flat().items():
debug_comments.set_key("{}".format(k), "Applied from override")
else:
scenario_overrides = []
# Second pass of applying override dict after applying scenarios,
# so that scenario-based overrides are overridden by override_dict!
if override_dict:
config_model.union(override_dict,
allow_override=True,
allow_replacement=True)
for k, v in override_dict.as_dict_flat().items():
debug_comments.set_key("{}".format(k),
"Overridden via override dictionary.")
# FutureWarning: raise cost class warning here.
# Warning that there will be no default cost class in 0.7.0 #
# Added in 0.6.4-dev, to be removed in v0.7.0-dev
if has_explicit_cost_class is False:
warnings.warn(
"There will be no default cost class for the objective function in "
'v0.7.0 (currently "monetary" with a weight of 1). '
"Explicitly specify the cost class(es) you would like to use "
'under `run.objective_options.cost_class`. E.g. `{"monetary": 1}` to '
"replicate the current default.",
FutureWarning,
)
# Drop default nodes, links, and techs
config_model.del_key("techs.default_tech")
config_model.del_key("nodes.default_node")
config_model.del_key("links.default_node_from,default_node_to")
return config_model, debug_comments, scenario_overrides, scenario
def apply_overrides(config, scenario=None, override_dict=None):
"""
Generate processed Model configuration, applying any scenarios overrides.
Parameters
----------
config : AttrDict
a model configuration AttrDict
scenario : str, optional
override_dict : str or dict or AttrDict, optional
If a YAML string, converted to AttrDict
"""
debug_comments = AttrDict()
config_model = AttrDict.from_yaml(os.path.join(
os.path.dirname(calliope.__file__), 'config', 'defaults.yaml'
))
# Interpret timeseries_data_path as relative
config.model.timeseries_data_path = relative_path(
config.config_path, config.model.timeseries_data_path
)
# FutureWarning: check if config includes an explicit objective cost class.
# Added in 0.6.4-dev, to be removed in v0.7.0-dev.
has_explicit_cost_class = isinstance(config.get_key('run.objective_options.cost_class', None), dict)
# The input files are allowed to override other model defaults
config_model.union(config, allow_override=True)
# First pass of applying override dict before applying scenarios,
# so that can override scenario definitions by override_dict
if override_dict:
if isinstance(override_dict, str):
override_dict = AttrDict.from_yaml_string(override_dict)
elif not isinstance(override_dict, AttrDict):
override_dict = AttrDict(override_dict)
warning_messages = checks.check_overrides(config_model, override_dict)
exceptions.print_warnings_and_raise_errors(warnings=warning_messages)
# FutureWarning: If config does not include an explicit objective cost class, check override dict.
# Added in 0.6.4-dev, to be removed in v0.7.0-dev.
if has_explicit_cost_class is False:
has_explicit_cost_class = isinstance(override_dict.get_key('run.objective_options.cost_class', None), dict)
config_model.union(
override_dict, allow_override=True, allow_replacement=True
)
if scenario:
scenarios = config_model.get('scenarios', {})
if scenario in scenarios.keys():
# Manually defined scenario names cannot be the same as single
# overrides or any combination of semicolon-delimited overrides
if all([i in config_model.get('overrides', {})
for i in scenario.split(',')]):
raise exceptions.ModelError(
'Manually defined scenario cannot be a combination of override names.'
)
if not isinstance(scenarios[scenario], list):
raise exceptions.ModelError(
'Scenario definition must be a list of override names.'
)
overrides = [str(i) for i in scenarios[scenario]]
logger.info(
'Using scenario `{}` leading to the application of '
'overrides `{}`.'.format(scenario, overrides)
)
else:
overrides = str(scenario).split(',')
logger.info(
'Applying the following overrides without a '
'specific scenario name: {}'.format(overrides)
)
overrides_from_scenario = combine_overrides(config_model, overrides)
warning_messages = checks.check_overrides(config_model, overrides_from_scenario)
exceptions.print_warnings_and_raise_errors(warnings=warning_messages)
# FutureWarning: If config nor override_dict include an explicit objective cost class, check scenario dict.
# Added in 0.6.4-dev, to be removed in v0.7.0-dev
if has_explicit_cost_class is False:
has_explicit_cost_class = isinstance(overrides_from_scenario.get_key('run.objective_options.cost_class', None), dict)
config_model.union(
overrides_from_scenario, allow_override=True, allow_replacement=True
)
for k, v in overrides_from_scenario.as_dict_flat().items():
debug_comments.set_key(
'{}'.format(k),
'Applied from override')
else:
overrides = []
# Second pass of applying override dict after applying scenarios,
# so that scenario-based overrides are overridden by override_dict!
if override_dict:
config_model.union(
override_dict, allow_override=True, allow_replacement=True
)
for k, v in override_dict.as_dict_flat().items():
debug_comments.set_key(
'{}'.format(k),
'Overridden via override dictionary.')
# FutureWarning: raise cost class warning here.
# Warning that there will be no default cost class in 0.7.0 #
# Added in 0.6.4-dev, to be removed in v0.7.0-dev
if has_explicit_cost_class is False:
warnings.warn(
'There will be no default cost class for the objective function in '
'v0.7.0 (currently "monetary" with a weight of 1). '
'Explicitly specify the cost class(es) you would like to use '
'under `run.objective_options.cost_class`. E.g. `{"monetary": 1}` to '
'replicate the current default.',
FutureWarning
)
# Drop default locations, links, and techs
config_model.del_key('techs.default_tech')
config_model.del_key('locations.default_location')
config_model.del_key('links.default_location_from,default_location_to')
config_model.del_key('group_constraints.default_group')
return config_model, debug_comments, overrides, scenario
def apply_overrides(config, override_file=None, override_dict=None):
"""
Generate processed Model configuration, applying any overrides.
Parameters
----------
config : AttrDict
a model configuration AttrDict
override_file : str, optional
override_dict : dict or AttrDict, optional
"""
debug_comments = AttrDict()
base_model_config_file = os.path.join(os.path.dirname(calliope.__file__),
'config', 'model.yaml')
config_model = AttrDict.from_yaml(base_model_config_file)
default_tech_groups = list(config_model.tech_groups.keys())
# README CHANGED: `model` is not a list any longer -
# it is now always a single file
# README CHANGED: order of arguments to relative_path reversed
# README CHANGED: data_path option removed -- need to make sure
# that for parallel runs, data_path relative to the currently
# open model config file always works
# Interpret timeseries_data_path as relative
config.model.timeseries_data_path = relative_path(
config.config_path, config.model.timeseries_data_path)
# Check if overriding coordinates are in the same coordinate system. If not,
# delete all incumbent coordinates, ready for the new coordinates to come in
def check_and_remove_coordinates(config_model, override):
if (any(
['coordinates' in k
for k in config_model.as_dict_flat().keys()]) and any(
['coordinates' in k
for k in override.as_dict_flat().keys()])):
config_keys = [
k for k in config_model.as_dict_flat().keys()
if 'coordinates.' in k
]
config_coordinates = [
k.split('coordinates.')[-1] for k in config_keys
]
override_coordinates = set(
k.split('coordinates.')[-1]
for k in override.as_dict_flat().keys() if 'coordinates.' in k)
if config_coordinates != override_coordinates:
for key in config_keys:
config_model.del_key(key)
# The input files are allowed to override other model defaults
config_model.union(config, allow_override=True)
# FIXME: if applying an override that doesn't exist in model, should warn
# the user about possible mis-spelling
# Apply overrides via 'override_file', which contains the path to a YAML file
if override_file:
# Due to the possible occurrance of `C:\path_to_file\file.yaml:override` we have to split
# override_file into `path_to_file`, `file.yaml` and `override` before
# merging `path_to_file` and `file.yaml` back together
path_to_file, override_file_with_group = os.path.split(override_file)
override_file, override_groups = override_file_with_group.split(':')
override_file_path = os.path.join(path_to_file, override_file)
override_from_file = combine_overrides(override_file_path,
override_groups)
check_and_remove_coordinates(config_model, override_from_file)
config_model.union(override_from_file,
allow_override=True,
allow_replacement=True)
for k, v in override_from_file.as_dict_flat().items():
debug_comments.set_key(
'{}'.format(k),
'Overridden via override: {}'.format(override_file))
# Apply overrides via 'override', which is an AttrDict
if override_dict:
if not isinstance(override_dict, AttrDict):
override_dict = AttrDict(override_dict)
check_and_remove_coordinates(config_model, override_dict)
config_model.union(override_dict,
allow_override=True,
allow_replacement=True)
for k, v in override_dict.as_dict_flat().items():
debug_comments.set_key('{}'.format(k),
'Overridden via override dictionary.')
return config_model, debug_comments
def apply_overrides(config, override_file=None, override_dict=None):
"""
Generate processed Model configuration, applying any overrides.
Parameters
----------
config : AttrDict
a model configuration AttrDict
override_file : str, optional
override_dict : dict or AttrDict, optional
"""
debug_comments = AttrDict()
base_model_config_file = os.path.join(os.path.dirname(calliope.__file__),
'config', 'model.yaml')
config_model = AttrDict.from_yaml(base_model_config_file)
default_tech_groups = list(config_model.tech_groups.keys())
# README CHANGED: `model` is not a list any longer -
# it is now always a single file
# README CHANGED: order of arguments to relative_path reversed
# README CHANGED: data_path option removed -- need to make sure
# that for parallel runs, data_path relative to the currently
# open model config file always works
# Interpret timeseries_data_path as relative
config.model.timeseries_data_path = relative_path(
config.config_path, config.model.timeseries_data_path)
# The input files are allowed to override other model defaults
config_model.union(config, allow_override=True)
# Apply overrides via 'override_file', which contains the path to a YAML file
if override_file:
# Due to the possible occurrance of `C:\path_to_file\file.yaml:override` we have to split
# override_file into `path_to_file`, `file.yaml` and `override` before
# merging `path_to_file` and `file.yaml` back together
path_to_file, override_file_with_group = os.path.split(override_file)
override_file, override_groups = override_file_with_group.split(':')
override_file_path = os.path.join(path_to_file, override_file)
override_from_file = combine_overrides(override_file_path,
override_groups)
warnings = checks.check_overrides(config_model, override_from_file)
exceptions.print_warnings_and_raise_errors(warnings=warnings)
config_model.union(override_from_file,
allow_override=True,
allow_replacement=True)
for k, v in override_from_file.as_dict_flat().items():
debug_comments.set_key(
'{}'.format(k),
'Overridden via override: {}'.format(override_file))
# Apply overrides via 'override', which is an AttrDict
if override_dict:
if not isinstance(override_dict, AttrDict):
override_dict = AttrDict(override_dict)
warnings = checks.check_overrides(config_model, override_dict)
exceptions.print_warnings_and_raise_errors(warnings=warnings)
config_model.union(override_dict,
allow_override=True,
allow_replacement=True)
for k, v in override_dict.as_dict_flat().items():
debug_comments.set_key('{}'.format(k),
'Overridden via override dictionary.')
return config_model, debug_comments
def apply_overrides(config, scenario=None, override_dict=None):
"""
Generate processed Model configuration, applying any scenarios overrides.
Parameters
----------
config : AttrDict
a model configuration AttrDict
scenario : str, optional
override_dict : str or dict or AttrDict, optional
If a YAML string, converted to AttrDict
"""
debug_comments = AttrDict()
base_model_config_file = os.path.join(
os.path.dirname(calliope.__file__),
'config', 'model.yaml'
)
config_model = AttrDict.from_yaml(base_model_config_file)
# Interpret timeseries_data_path as relative
config.model.timeseries_data_path = relative_path(
config.config_path, config.model.timeseries_data_path
)
# The input files are allowed to override other model defaults
config_model.union(config, allow_override=True)
# First pass of applying override dict before applying scenarios,
# so that can override scenario definitions by override_dict
if override_dict:
if isinstance(override_dict, str):
override_dict = AttrDict.from_yaml_string(override_dict)
elif not isinstance(override_dict, AttrDict):
override_dict = AttrDict(override_dict)
warnings = checks.check_overrides(config_model, override_dict)
exceptions.print_warnings_and_raise_errors(warnings=warnings)
config_model.union(
override_dict, allow_override=True, allow_replacement=True
)
if scenario:
scenarios = config_model.get('scenarios', {})
if scenario in scenarios.keys():
# Manually defined scenario names cannot be the same as single
# overrides or any combination of semicolon-delimited overrides
if all([i in config_model.get('overrides', {})
for i in scenario.split(',')]):
raise exceptions.ModelError(
'Manually defined scenario cannot be a combination of override names.'
)
if not isinstance(scenarios[scenario], list):
raise exceptions.ModelError(
'Scenario definition must be a list of override names.'
)
overrides = [str(i) for i in scenarios[scenario]]
logger.info(
'Using scenario `{}` leading to the application of '
'overrides `{}`.'.format(scenario, overrides)
)
else:
overrides = str(scenario).split(',')
logger.info(
'Applying the following overrides without a '
'specific scenario name: {}'.format(overrides)
)
overrides_from_scenario = combine_overrides(config_model, overrides)
warnings = checks.check_overrides(config_model, overrides_from_scenario)
exceptions.print_warnings_and_raise_errors(warnings=warnings)
config_model.union(
overrides_from_scenario, allow_override=True, allow_replacement=True
)
for k, v in overrides_from_scenario.as_dict_flat().items():
debug_comments.set_key(
'{}'.format(k),
'Applied from override')
else:
overrides = []
# Second pass of applying override dict after applying scenarios,
# so that scenario-based overrides are overridden by override_dict!
if override_dict:
config_model.union(
override_dict, allow_override=True, allow_replacement=True
)
for k, v in override_dict.as_dict_flat().items():
debug_comments.set_key(
'{}'.format(k),
'Overridden via override dictionary.')
return config_model, debug_comments, overrides, scenario
