def apply_overrides(config, scenario=None, override_dict=None):
"""
Generate processed Model configuration, applying any scenarios overrides.
Parameters
----------
config : AttrDict
a model configuration AttrDict
scenario : str, optional
override_dict : str or dict or AttrDict, optional
If a YAML string, converted to AttrDict
"""
debug_comments = AttrDict()
base_model_config_file = os.path.join(
os.path.dirname(calliope.__file__),
'config', 'model.yaml'
)
config_model = AttrDict.from_yaml(base_model_config_file)
# Interpret timeseries_data_path as relative
config.model.timeseries_data_path = relative_path(
config.config_path, config.model.timeseries_data_path
)
# The input files are allowed to override other model defaults
config_model.union(config, allow_override=True)
# First pass of applying override dict before applying scenarios,
# so that can override scenario definitions by override_dict
if override_dict:
if isinstance(override_dict, str):
override_dict = AttrDict.from_yaml_string(override_dict)
elif not isinstance(override_dict, AttrDict):
override_dict = AttrDict(override_dict)
warnings = checks.check_overrides(config_model, override_dict)
exceptions.print_warnings_and_raise_errors(warnings=warnings)
config_model.union(
override_dict, allow_override=True, allow_replacement=True
)
if scenario:
scenarios = config_model.get('scenarios', {})
if scenario in scenarios:
# Manually defined scenario names cannot be the same as single
# overrides or any combination of semicolon-delimited overrides
if all([i in config_model.get('overrides', {})
for i in scenario.split(',')]):
raise exceptions.ModelError(
'Manually defined scenario cannot be a combination of override names.'
)
if not isinstance(scenarios[scenario], str):
raise exceptions.ModelError(
'Scenario definition must be string of comma-separated overrides.'
)
overrides = scenarios[scenario].split(',')
logger.info(
'Using scenario `{}` leading to the application of '
'overrides `{}`.'.format(scenario, scenarios[scenario])
)
else:
overrides = str(scenario).split(',')
logger.info(
'Applying overrides `{}` without a '
'specific scenario name.'.format(scenario)
)
overrides_from_scenario = combine_overrides(config_model, overrides)
warnings = checks.check_overrides(config_model, overrides_from_scenario)
exceptions.print_warnings_and_raise_errors(warnings=warnings)
config_model.union(
overrides_from_scenario, allow_override=True, allow_replacement=True
)
for k, v in overrides_from_scenario.as_dict_flat().items():
debug_comments.set_key(
'{}'.format(k),
'Applied from override')
else:
overrides = []
# Second pass of applying override dict after applying scenarios,
# so that scenario-based overrides are overridden by override_dict!
if override_dict:
config_model.union(
override_dict, allow_override=True, allow_replacement=True
)
for k, v in override_dict.as_dict_flat().items():
debug_comments.set_key(
'{}'.format(k),
'Overridden via override dictionary.')
return config_model, debug_comments, overrides, scenario
def generate_simple_sets(model_run):
"""
Generate basic sets for a given pre-processed ``model_run``.
Parameters
----------
model_run : AttrDict
"""
sets = AttrDict()
flat_techs = model_run.techs.as_dict(flat=True)
flat_locations = model_run.locations.as_dict(flat=True)
sets.resources = set(
flatten_list(v for k, v in flat_techs.items() if '.carrier' in k))
sets.carriers = sets.resources - set(['resource'])
sets.carrier_tiers = set(
key.split('.carrier_')[1] for key in flat_techs.keys()
if '.carrier_' in key)
sets.costs = set(
k.split('costs.')[-1].split('.')[0] for k in flat_locations.keys()
if '.costs.' in k)
sets.locs = set(model_run.locations.keys())
sets.techs_non_transmission = set()
tech_groups = [
'demand', 'supply', 'supply_plus', 'conversion', 'conversion_plus',
'storage'
]
for tech_group in tech_groups:
sets['techs_{}'.format(tech_group)] = set(
k for k, v in model_run.techs.items()
if v.inheritance[-1] == tech_group)
sets.techs_non_transmission.update(sets['techs_{}'.format(tech_group)])
sets.techs_transmission_names = set(k for k, v in model_run.techs.items()
if v.inheritance[-1] == 'transmission')
# This builds the "tech:loc" expansion of transmission technologies
techs_transmission = set()
for loc_name, loc_config in model_run.locations.items():
for link_name, link_config in loc_config.get('links', {}).items():
for tech_name in link_config.techs:
techs_transmission.add('{}:{}'.format(tech_name, link_name))
sets.techs_transmission = techs_transmission
sets.techs = sets.techs_non_transmission | sets.techs_transmission_names
# this extracts location coordinate information
coordinates = set(
k.split('.')[-1] for k in flat_locations.keys()
if '.coordinates.' in k)
if coordinates:
sets.coordinates = coordinates
# `timesteps` set is built from the results of timeseries_data processing
sets.timesteps = list(model_run.timesteps.astype(str))
model_run.del_key('timesteps')
# `techlists` are strings with comma-separated techs used for grouping in
# some model-wide constraints
sets.techlists = set()
for k in model_run.model.get_key('group_share', {}).keys():
sets.techlists.add(k)
# `constraint_groups` are the group names per constraint that is defined
# at a group level
sets.group_constraints = set()
group_constraints = AttrDict({
name: data
for name, data in model_run['group_constraints'].items()
if data.get("exists", True)
})
if len(group_constraints.keys()) > 0:
sets.group_constraints.update(
i.split('.')[1] for i in group_constraints.as_dict_flat().keys()
if i.split('.')[1] not in ['techs', 'locs'])
for constr in sets.group_constraints:
sets['group_names_' + constr] = set(
k for k, v in group_constraints.items() if constr in v.keys())
return sets
def apply_overrides(config, scenario=None, override_dict=None):
"""
Generate processed Model configuration, applying any scenarios overrides.
Parameters
----------
config : AttrDict
a model configuration AttrDict
scenario : str, optional
override_dict : str or dict or AttrDict, optional
If a YAML string, converted to AttrDict
"""
debug_comments = AttrDict()
config_model = AttrDict.from_yaml(
os.path.join(os.path.dirname(calliope.__file__), "config",
"defaults.yaml"))
# Interpret timeseries_data_path as relative
if "timeseries_data_path" in config.model:
config.model.timeseries_data_path = relative_path(
config.config_path, config.model.timeseries_data_path)
# FutureWarning: check if config includes an explicit objective cost class.
# Added in 0.6.4-dev, to be removed in v0.7.0-dev.
has_explicit_cost_class = isinstance(
config.get_key("run.objective_options.cost_class", None), dict)
# The input files are allowed to override other model defaults
config_model.union(config, allow_override=True)
# First pass of applying override dict before applying scenarios,
# so that can override scenario definitions by override_dict
if override_dict:
if isinstance(override_dict, str):
override_dict = AttrDict.from_yaml_string(override_dict)
elif not isinstance(override_dict, AttrDict):
override_dict = AttrDict(override_dict)
warning_messages = checks.check_overrides(config_model, override_dict)
exceptions.print_warnings_and_raise_errors(warnings=warning_messages)
# FutureWarning: If config does not include an explicit objective cost class, check override dict.
# Added in 0.6.4-dev, to be removed in v0.7.0-dev.
if has_explicit_cost_class is False:
has_explicit_cost_class = isinstance(
override_dict.get_key("run.objective_options.cost_class",
None), dict)
config_model.union(override_dict,
allow_override=True,
allow_replacement=True)
if scenario:
scenario_overrides = load_overrides_from_scenario(
config_model, scenario)
if not all(i in config_model.get("overrides", {})
for i in scenario_overrides):
raise exceptions.ModelError(
"Scenario definition must be a list of override or other scenario names."
)
else:
logger.info(
"Applying the following overrides from scenario definition: {} "
.format(scenario_overrides))
overrides_from_scenario = combine_overrides(config_model,
scenario_overrides)
warning_messages = checks.check_overrides(config_model,
overrides_from_scenario)
exceptions.print_warnings_and_raise_errors(warnings=warning_messages)
# FutureWarning: If config nor override_dict include an explicit objective cost class, check scenario dict.
# Added in 0.6.4-dev, to be removed in v0.7.0-dev
if has_explicit_cost_class is False:
has_explicit_cost_class = isinstance(
overrides_from_scenario.get_key(
"run.objective_options.cost_class", None),
dict,
)
config_model.union(overrides_from_scenario,
allow_override=True,
allow_replacement=True)
for k, v in overrides_from_scenario.as_dict_flat().items():
debug_comments.set_key("{}".format(k), "Applied from override")
else:
scenario_overrides = []
# Second pass of applying override dict after applying scenarios,
# so that scenario-based overrides are overridden by override_dict!
if override_dict:
config_model.union(override_dict,
allow_override=True,
allow_replacement=True)
for k, v in override_dict.as_dict_flat().items():
debug_comments.set_key("{}".format(k),
"Overridden via override dictionary.")
# FutureWarning: raise cost class warning here.
# Warning that there will be no default cost class in 0.7.0 #
# Added in 0.6.4-dev, to be removed in v0.7.0-dev
if has_explicit_cost_class is False:
warnings.warn(
"There will be no default cost class for the objective function in "
'v0.7.0 (currently "monetary" with a weight of 1). '
"Explicitly specify the cost class(es) you would like to use "
'under `run.objective_options.cost_class`. E.g. `{"monetary": 1}` to '
"replicate the current default.",
FutureWarning,
)
# Drop default nodes, links, and techs
config_model.del_key("techs.default_tech")
config_model.del_key("nodes.default_node")
config_model.del_key("links.default_node_from,default_node_to")
return config_model, debug_comments, scenario_overrides, scenario
def apply_overrides(config, scenario=None, override_dict=None):
"""
Generate processed Model configuration, applying any scenarios overrides.
Parameters
----------
config : AttrDict
a model configuration AttrDict
scenario : str, optional
override_dict : str or dict or AttrDict, optional
If a YAML string, converted to AttrDict
"""
debug_comments = AttrDict()
config_model = AttrDict.from_yaml(os.path.join(
os.path.dirname(calliope.__file__), 'config', 'defaults.yaml'
))
# Interpret timeseries_data_path as relative
config.model.timeseries_data_path = relative_path(
config.config_path, config.model.timeseries_data_path
)
# FutureWarning: check if config includes an explicit objective cost class.
# Added in 0.6.4-dev, to be removed in v0.7.0-dev.
has_explicit_cost_class = isinstance(config.get_key('run.objective_options.cost_class', None), dict)
# The input files are allowed to override other model defaults
config_model.union(config, allow_override=True)
# First pass of applying override dict before applying scenarios,
# so that can override scenario definitions by override_dict
if override_dict:
if isinstance(override_dict, str):
override_dict = AttrDict.from_yaml_string(override_dict)
elif not isinstance(override_dict, AttrDict):
override_dict = AttrDict(override_dict)
warning_messages = checks.check_overrides(config_model, override_dict)
exceptions.print_warnings_and_raise_errors(warnings=warning_messages)
# FutureWarning: If config does not include an explicit objective cost class, check override dict.
# Added in 0.6.4-dev, to be removed in v0.7.0-dev.
if has_explicit_cost_class is False:
has_explicit_cost_class = isinstance(override_dict.get_key('run.objective_options.cost_class', None), dict)
config_model.union(
override_dict, allow_override=True, allow_replacement=True
)
if scenario:
scenarios = config_model.get('scenarios', {})
if scenario in scenarios.keys():
# Manually defined scenario names cannot be the same as single
# overrides or any combination of semicolon-delimited overrides
if all([i in config_model.get('overrides', {})
for i in scenario.split(',')]):
raise exceptions.ModelError(
'Manually defined scenario cannot be a combination of override names.'
)
if not isinstance(scenarios[scenario], list):
raise exceptions.ModelError(
'Scenario definition must be a list of override names.'
)
overrides = [str(i) for i in scenarios[scenario]]
logger.info(
'Using scenario `{}` leading to the application of '
'overrides `{}`.'.format(scenario, overrides)
)
else:
overrides = str(scenario).split(',')
logger.info(
'Applying the following overrides without a '
'specific scenario name: {}'.format(overrides)
)
overrides_from_scenario = combine_overrides(config_model, overrides)
warning_messages = checks.check_overrides(config_model, overrides_from_scenario)
exceptions.print_warnings_and_raise_errors(warnings=warning_messages)
# FutureWarning: If config nor override_dict include an explicit objective cost class, check scenario dict.
# Added in 0.6.4-dev, to be removed in v0.7.0-dev
if has_explicit_cost_class is False:
has_explicit_cost_class = isinstance(overrides_from_scenario.get_key('run.objective_options.cost_class', None), dict)
config_model.union(
overrides_from_scenario, allow_override=True, allow_replacement=True
)
for k, v in overrides_from_scenario.as_dict_flat().items():
debug_comments.set_key(
'{}'.format(k),
'Applied from override')
else:
overrides = []
# Second pass of applying override dict after applying scenarios,
# so that scenario-based overrides are overridden by override_dict!
if override_dict:
config_model.union(
override_dict, allow_override=True, allow_replacement=True
)
for k, v in override_dict.as_dict_flat().items():
debug_comments.set_key(
'{}'.format(k),
'Overridden via override dictionary.')
# FutureWarning: raise cost class warning here.
# Warning that there will be no default cost class in 0.7.0 #
# Added in 0.6.4-dev, to be removed in v0.7.0-dev
if has_explicit_cost_class is False:
warnings.warn(
'There will be no default cost class for the objective function in '
'v0.7.0 (currently "monetary" with a weight of 1). '
'Explicitly specify the cost class(es) you would like to use '
'under `run.objective_options.cost_class`. E.g. `{"monetary": 1}` to '
'replicate the current default.',
FutureWarning
)
# Drop default locations, links, and techs
config_model.del_key('techs.default_tech')
config_model.del_key('locations.default_location')
config_model.del_key('links.default_location_from,default_location_to')
config_model.del_key('group_constraints.default_group')
return config_model, debug_comments, overrides, scenario
def apply_overrides(config, override_file=None, override_dict=None):
"""
Generate processed Model configuration, applying any overrides.
Parameters
----------
config : AttrDict
a model configuration AttrDict
override_file : str, optional
override_dict : dict or AttrDict, optional
"""
debug_comments = AttrDict()
base_model_config_file = os.path.join(os.path.dirname(calliope.__file__),
'config', 'model.yaml')
config_model = AttrDict.from_yaml(base_model_config_file)
default_tech_groups = list(config_model.tech_groups.keys())
# README CHANGED: `model` is not a list any longer -
# it is now always a single file
# README CHANGED: order of arguments to relative_path reversed
# README CHANGED: data_path option removed -- need to make sure
# that for parallel runs, data_path relative to the currently
# open model config file always works
# Interpret timeseries_data_path as relative
config.model.timeseries_data_path = relative_path(
config.config_path, config.model.timeseries_data_path)
# Check if overriding coordinates are in the same coordinate system. If not,
# delete all incumbent coordinates, ready for the new coordinates to come in
def check_and_remove_coordinates(config_model, override):
if (any(
['coordinates' in k
for k in config_model.as_dict_flat().keys()]) and any(
['coordinates' in k
for k in override.as_dict_flat().keys()])):
config_keys = [
k for k in config_model.as_dict_flat().keys()
if 'coordinates.' in k
]
config_coordinates = [
k.split('coordinates.')[-1] for k in config_keys
]
override_coordinates = set(
k.split('coordinates.')[-1]
for k in override.as_dict_flat().keys() if 'coordinates.' in k)
if config_coordinates != override_coordinates:
for key in config_keys:
config_model.del_key(key)
# The input files are allowed to override other model defaults
config_model.union(config, allow_override=True)
# FIXME: if applying an override that doesn't exist in model, should warn
# the user about possible mis-spelling
# Apply overrides via 'override_file', which contains the path to a YAML file
if override_file:
# Due to the possible occurrance of `C:\path_to_file\file.yaml:override` we have to split
# override_file into `path_to_file`, `file.yaml` and `override` before
# merging `path_to_file` and `file.yaml` back together
path_to_file, override_file_with_group = os.path.split(override_file)
override_file, override_groups = override_file_with_group.split(':')
override_file_path = os.path.join(path_to_file, override_file)
override_from_file = combine_overrides(override_file_path,
override_groups)
check_and_remove_coordinates(config_model, override_from_file)
config_model.union(override_from_file,
allow_override=True,
allow_replacement=True)
for k, v in override_from_file.as_dict_flat().items():
debug_comments.set_key(
'{}'.format(k),
'Overridden via override: {}'.format(override_file))
# Apply overrides via 'override', which is an AttrDict
if override_dict:
if not isinstance(override_dict, AttrDict):
override_dict = AttrDict(override_dict)
check_and_remove_coordinates(config_model, override_dict)
config_model.union(override_dict,
allow_override=True,
allow_replacement=True)
for k, v in override_dict.as_dict_flat().items():
debug_comments.set_key('{}'.format(k),
'Overridden via override dictionary.')
return config_model, debug_comments
def apply_overrides(config, override_file=None, override_dict=None):
"""
Generate processed Model configuration, applying any overrides.
Parameters
----------
config : AttrDict
a model configuration AttrDict
override_file : str, optional
override_dict : dict or AttrDict, optional
"""
debug_comments = AttrDict()
base_model_config_file = os.path.join(os.path.dirname(calliope.__file__),
'config', 'model.yaml')
config_model = AttrDict.from_yaml(base_model_config_file)
default_tech_groups = list(config_model.tech_groups.keys())
# README CHANGED: `model` is not a list any longer -
# it is now always a single file
# README CHANGED: order of arguments to relative_path reversed
# README CHANGED: data_path option removed -- need to make sure
# that for parallel runs, data_path relative to the currently
# open model config file always works
# Interpret timeseries_data_path as relative
config.model.timeseries_data_path = relative_path(
config.config_path, config.model.timeseries_data_path)
# The input files are allowed to override other model defaults
config_model.union(config, allow_override=True)
# Apply overrides via 'override_file', which contains the path to a YAML file
if override_file:
# Due to the possible occurrance of `C:\path_to_file\file.yaml:override` we have to split
# override_file into `path_to_file`, `file.yaml` and `override` before
# merging `path_to_file` and `file.yaml` back together
path_to_file, override_file_with_group = os.path.split(override_file)
override_file, override_groups = override_file_with_group.split(':')
override_file_path = os.path.join(path_to_file, override_file)
override_from_file = combine_overrides(override_file_path,
override_groups)
warnings = checks.check_overrides(config_model, override_from_file)
exceptions.print_warnings_and_raise_errors(warnings=warnings)
config_model.union(override_from_file,
allow_override=True,
allow_replacement=True)
for k, v in override_from_file.as_dict_flat().items():
debug_comments.set_key(
'{}'.format(k),
'Overridden via override: {}'.format(override_file))
# Apply overrides via 'override', which is an AttrDict
if override_dict:
if not isinstance(override_dict, AttrDict):
override_dict = AttrDict(override_dict)
warnings = checks.check_overrides(config_model, override_dict)
exceptions.print_warnings_and_raise_errors(warnings=warnings)
config_model.union(override_dict,
allow_override=True,
allow_replacement=True)
for k, v in override_dict.as_dict_flat().items():
debug_comments.set_key('{}'.format(k),
'Overridden via override dictionary.')
return config_model, debug_comments
def apply_overrides(config, scenario=None, override_dict=None):
"""
Generate processed Model configuration, applying any scenarios overrides.
Parameters
----------
config : AttrDict
a model configuration AttrDict
scenario : str, optional
override_dict : str or dict or AttrDict, optional
If a YAML string, converted to AttrDict
"""
debug_comments = AttrDict()
base_model_config_file = os.path.join(
os.path.dirname(calliope.__file__),
'config', 'model.yaml'
)
config_model = AttrDict.from_yaml(base_model_config_file)
# Interpret timeseries_data_path as relative
config.model.timeseries_data_path = relative_path(
config.config_path, config.model.timeseries_data_path
)
# The input files are allowed to override other model defaults
config_model.union(config, allow_override=True)
# First pass of applying override dict before applying scenarios,
# so that can override scenario definitions by override_dict
if override_dict:
if isinstance(override_dict, str):
override_dict = AttrDict.from_yaml_string(override_dict)
elif not isinstance(override_dict, AttrDict):
override_dict = AttrDict(override_dict)
warnings = checks.check_overrides(config_model, override_dict)
exceptions.print_warnings_and_raise_errors(warnings=warnings)
config_model.union(
override_dict, allow_override=True, allow_replacement=True
)
if scenario:
scenarios = config_model.get('scenarios', {})
if scenario in scenarios.keys():
# Manually defined scenario names cannot be the same as single
# overrides or any combination of semicolon-delimited overrides
if all([i in config_model.get('overrides', {})
for i in scenario.split(',')]):
raise exceptions.ModelError(
'Manually defined scenario cannot be a combination of override names.'
)
if not isinstance(scenarios[scenario], list):
raise exceptions.ModelError(
'Scenario definition must be a list of override names.'
)
overrides = [str(i) for i in scenarios[scenario]]
logger.info(
'Using scenario `{}` leading to the application of '
'overrides `{}`.'.format(scenario, overrides)
)
else:
overrides = str(scenario).split(',')
logger.info(
'Applying the following overrides without a '
'specific scenario name: {}'.format(overrides)
)
overrides_from_scenario = combine_overrides(config_model, overrides)
warnings = checks.check_overrides(config_model, overrides_from_scenario)
exceptions.print_warnings_and_raise_errors(warnings=warnings)
config_model.union(
overrides_from_scenario, allow_override=True, allow_replacement=True
)
for k, v in overrides_from_scenario.as_dict_flat().items():
debug_comments.set_key(
'{}'.format(k),
'Applied from override')
else:
overrides = []
# Second pass of applying override dict after applying scenarios,
# so that scenario-based overrides are overridden by override_dict!
if override_dict:
config_model.union(
override_dict, allow_override=True, allow_replacement=True
)
for k, v in override_dict.as_dict_flat().items():
debug_comments.set_key(
'{}'.format(k),
'Overridden via override dictionary.')
return config_model, debug_comments, overrides, scenario
