def check_overrides(config_model, override):
"""
Perform checks on the override dict and override file inputs to ensure they
are not doing something silly.
"""
model_warnings = []
info = []
for key in override.as_dict_flat().keys():
if key in config_model.as_dict_flat().keys():
info.append(
'Override applied to {}: {} -> {}'
.format(key, config_model.get_key(key), override.get_key(key))
)
else:
info.append(
'`{}`:{} applied from override as new configuration'
.format(key, override.get_key(key))
)
# Check if overriding coordinates are in the same coordinate system. If not,
# delete all incumbent coordinates, ready for the new coordinates to come in
if (any(['coordinates' in k for k in config_model.as_dict_flat().keys()]) and
any(['coordinates' in k for k in override.as_dict_flat().keys()])):
# get keys that might be deleted and incumbent coordinate system
config_keys = [k for k in config_model.as_dict_flat().keys() if 'coordinates.' in k]
config_coordinates = set([k.split('coordinates.')[-1] for k in config_keys])
# get overriding coordinate system
override_coordinates = set(
k.split('coordinates.')[-1] for k in override.as_dict_flat().keys()
if 'coordinates.' in k
)
# compare overriding and incumbent, deleting incumbent if overriding is different
if config_coordinates != override_coordinates:
for key in config_keys:
config_model.del_key(key)
model_warnings.append(
'Updated from coordinate system {} to {}, using overrides'
.format(config_coordinates, override_coordinates)
)
if info:
logger.info('\n'.join(info))
return model_warnings
def check_overrides(config_model, override):
"""
Perform checks on the override dict and override file inputs to ensure they
are not doing something silly.
"""
model_warnings = []
info = []
for key in override.as_dict_flat().keys():
if key in config_model.as_dict_flat().keys():
info.append('Override applied to {}: {} -> {}'.format(
key, config_model.get_key(key), override.get_key(key)))
else:
info.append(
'`{}`:{} applied from override as new configuration'.format(
key, override.get_key(key)))
# Check if overriding coordinates are in the same coordinate system. If not,
# delete all incumbent coordinates, ready for the new coordinates to come in
if (any(['coordinates' in k for k in config_model.as_dict_flat().keys()])
and any(
['coordinates' in k for k in override.as_dict_flat().keys()])):
# get keys that might be deleted and incumbent coordinate system
config_keys = [
k for k in config_model.as_dict_flat().keys()
if 'coordinates.' in k
]
config_coordinates = set(
[k.split('coordinates.')[-1] for k in config_keys])
# get overriding coordinate system
override_coordinates = set(
k.split('coordinates.')[-1]
for k in override.as_dict_flat().keys() if 'coordinates.' in k)
# compare overriding and incumbent, deleting incumbent if overriding is different
if config_coordinates != override_coordinates:
for key in config_keys:
config_model.del_key(key)
model_warnings.append(
'Updated from coordinate system {} to {}, using overrides'.
format(config_coordinates, override_coordinates))
if info:
logger.info('\n'.join(info))
return model_warnings
def apply_overrides(config, scenario=None, override_dict=None):
"""
Generate processed Model configuration, applying any scenarios overrides.
Parameters
----------
config : AttrDict
a model configuration AttrDict
scenario : str, optional
override_dict : str or dict or AttrDict, optional
If a YAML string, converted to AttrDict
"""
debug_comments = AttrDict()
base_model_config_file = os.path.join(
os.path.dirname(calliope.__file__),
'config', 'model.yaml'
)
config_model = AttrDict.from_yaml(base_model_config_file)
# Interpret timeseries_data_path as relative
config.model.timeseries_data_path = relative_path(
config.config_path, config.model.timeseries_data_path
)
# The input files are allowed to override other model defaults
config_model.union(config, allow_override=True)
# First pass of applying override dict before applying scenarios,
# so that can override scenario definitions by override_dict
if override_dict:
if isinstance(override_dict, str):
override_dict = AttrDict.from_yaml_string(override_dict)
elif not isinstance(override_dict, AttrDict):
override_dict = AttrDict(override_dict)
warnings = checks.check_overrides(config_model, override_dict)
exceptions.print_warnings_and_raise_errors(warnings=warnings)
config_model.union(
override_dict, allow_override=True, allow_replacement=True
)
if scenario:
scenarios = config_model.get('scenarios', {})
if scenario in scenarios:
# Manually defined scenario names cannot be the same as single
# overrides or any combination of semicolon-delimited overrides
if all([i in config_model.get('overrides', {})
for i in scenario.split(',')]):
raise exceptions.ModelError(
'Manually defined scenario cannot be a combination of override names.'
)
if not isinstance(scenarios[scenario], str):
raise exceptions.ModelError(
'Scenario definition must be string of comma-separated overrides.'
)
overrides = scenarios[scenario].split(',')
logger.info(
'Using scenario `{}` leading to the application of '
'overrides `{}`.'.format(scenario, scenarios[scenario])
)
else:
overrides = str(scenario).split(',')
logger.info(
'Applying overrides `{}` without a '
'specific scenario name.'.format(scenario)
)
overrides_from_scenario = combine_overrides(config_model, overrides)
warnings = checks.check_overrides(config_model, overrides_from_scenario)
exceptions.print_warnings_and_raise_errors(warnings=warnings)
config_model.union(
overrides_from_scenario, allow_override=True, allow_replacement=True
)
for k, v in overrides_from_scenario.as_dict_flat().items():
debug_comments.set_key(
'{}'.format(k),
'Applied from override')
else:
overrides = []
# Second pass of applying override dict after applying scenarios,
# so that scenario-based overrides are overridden by override_dict!
if override_dict:
config_model.union(
override_dict, allow_override=True, allow_replacement=True
)
for k, v in override_dict.as_dict_flat().items():
debug_comments.set_key(
'{}'.format(k),
'Overridden via override dictionary.')
return config_model, debug_comments, overrides, scenario
def convert_model(run_config_path, model_config_path, out_path):
"""
Convert a model specified by a model YAML file
Parameters
----------
run_config_path: str
is merged with the model configuration and saved into the
main model configuration file given by ``model_config``
model_config_path: str
model configuration file
out_path: str
path into which to save ``model_config`` and all other YAML
files imported by it -- recreates original directory structure
at that location, so recommendation is to specify an empty
subdirectory or a new directory (will be created)
Returns
-------
None
"""
state = {'ensure_feasibility': False}
converted_run_config = AttrDict()
run_config = load_with_import_resolution(run_config_path)
for k, v in run_config.items():
# We consider any files imported in run configuration, but
# disregard file names and simply merge everything together
# into the new model configuration
converted_run_config.update(convert_run_dict(v, _CONVERSIONS))
new_model_config = AttrDict()
model_config = load_with_import_resolution(model_config_path)
# Get all techs from old model that need to be tech_groups in the new one
merged_model_config = AttrDict.from_yaml(model_config_path)
run_config_overrides = AttrDict.from_yaml(run_config_path).get_key(
'override', None)
if run_config_overrides:
merged_model_config.union(run_config_overrides, allow_override=True)
tech_groups = set()
for tech, tech_dict in merged_model_config.techs.items():
parent = tech_dict.get('parent', None)
if parent and parent not in _TECH_GROUPS:
tech_groups.add(parent)
for k, v in model_config.items():
new_model_config[k] = convert_model_dict(v,
_CONVERSIONS,
tech_groups=tech_groups,
state=state)
# Merge run_config into main model config file
new_model_config[model_config_path].union(converted_run_config)
# README: For future use we probably want a configuration to specify
# a calliope version it's compatible with / built for
new_model_config[model_config_path].set_key('model.calliope_version',
'0.6.0')
# Set ensure_feasibility if the old model used unmet_demand
if state['ensure_feasibility']:
new_model_config[model_config_path].set_key('run.ensure_feasibility',
True)
logger.info(
'Found no longer supported `unmet_demand` techs, setting `run.ensure_feasibility` \n'
'to True to replace them. See the docs for more info:\n'
'https://calliope.readthedocs.io/en/stable/user/building.html#allowing-for-unmet-demand'
)
# README: adding top-level interest_rate and lifetime definitions
# for all techs EXCEPT demand,
# to mirror the fact that there used to be defaults
defaults_v05 = AttrDict()
cost_classes = [ # Get a list of all cost classes in model
k.split('costs.', 1)[-1].split('.', 1)[0]
for k in new_model_config.keys_nested() if 'costs.' in k
]
for t in [i for i in _TECH_GROUPS if i != 'demand']:
defaults_v05.set_key('tech_groups.{}.constraints.lifetime'.format(t),
25)
for cc in cost_classes:
interest = 0.1 if cc == 'monetary' else 0
defaults_v05.set_key(
'tech_groups.{}.costs.{}.interest_rate'.format(t, cc),
interest)
new_model_config[model_config_path].union(defaults_v05)
# For each file in new_model_config, save it to its same
# position from the old path in the `out_path`
for f in new_model_config:
out_dir, out_filename = os.path.split(
f.replace(os.path.commonpath([model_config_path, f]), '.'))
if f == model_config_path:
out_dir_model_config_path = out_dir
out_filename = os.path.basename(model_config_path)
out_file = os.path.join(out_path, out_dir, out_filename)
os.makedirs(os.path.join(out_path, out_dir), exist_ok=True)
new_model_config[f].to_yaml(out_file)
# Read each CSV file in the model data dir and apply index
full_new_config = AttrDict.from_yaml(
os.path.join(out_path, out_dir_model_config_path,
os.path.basename(model_config_path)))
ts_dir = full_new_config.get_key('model.timeseries_data_path')
ts_path_in = os.path.join(os.path.dirname(model_config_path), ts_dir)
ts_path_out = os.path.join(os.path.join(out_path, ts_dir))
os.makedirs(ts_path_out, exist_ok=True)
index_t = pd.read_csv(os.path.join(ts_path_in, 'set_t.csv'),
index_col=0,
header=None)[1]
for f in glob.glob(os.path.join(ts_path_in, '*.csv')):
if 'set_t.csv' not in f:
df = pd.read_csv(f, index_col=0)
df.index = index_t
df.index.name = None
df.to_csv(os.path.join(ts_path_out, os.path.basename(f)))
def apply_overrides(config, scenario=None, override_dict=None):
"""
Generate processed Model configuration, applying any scenarios overrides.
Parameters
----------
config : AttrDict
a model configuration AttrDict
scenario : str, optional
override_dict : str or dict or AttrDict, optional
If a YAML string, converted to AttrDict
"""
debug_comments = AttrDict()
base_model_config_file = os.path.join(
os.path.dirname(calliope.__file__),
'config', 'model.yaml'
)
config_model = AttrDict.from_yaml(base_model_config_file)
# Interpret timeseries_data_path as relative
config.model.timeseries_data_path = relative_path(
config.config_path, config.model.timeseries_data_path
)
# The input files are allowed to override other model defaults
config_model.union(config, allow_override=True)
# First pass of applying override dict before applying scenarios,
# so that can override scenario definitions by override_dict
if override_dict:
if isinstance(override_dict, str):
override_dict = AttrDict.from_yaml_string(override_dict)
elif not isinstance(override_dict, AttrDict):
override_dict = AttrDict(override_dict)
warnings = checks.check_overrides(config_model, override_dict)
exceptions.print_warnings_and_raise_errors(warnings=warnings)
config_model.union(
override_dict, allow_override=True, allow_replacement=True
)
if scenario:
scenarios = config_model.get('scenarios', {})
if scenario in scenarios.keys():
# Manually defined scenario names cannot be the same as single
# overrides or any combination of semicolon-delimited overrides
if all([i in config_model.get('overrides', {})
for i in scenario.split(',')]):
raise exceptions.ModelError(
'Manually defined scenario cannot be a combination of override names.'
)
if not isinstance(scenarios[scenario], list):
raise exceptions.ModelError(
'Scenario definition must be a list of override names.'
)
overrides = [str(i) for i in scenarios[scenario]]
logger.info(
'Using scenario `{}` leading to the application of '
'overrides `{}`.'.format(scenario, overrides)
)
else:
overrides = str(scenario).split(',')
logger.info(
'Applying the following overrides without a '
'specific scenario name: {}'.format(overrides)
)
overrides_from_scenario = combine_overrides(config_model, overrides)
warnings = checks.check_overrides(config_model, overrides_from_scenario)
exceptions.print_warnings_and_raise_errors(warnings=warnings)
config_model.union(
overrides_from_scenario, allow_override=True, allow_replacement=True
)
for k, v in overrides_from_scenario.as_dict_flat().items():
debug_comments.set_key(
'{}'.format(k),
'Applied from override')
else:
overrides = []
# Second pass of applying override dict after applying scenarios,
# so that scenario-based overrides are overridden by override_dict!
if override_dict:
config_model.union(
override_dict, allow_override=True, allow_replacement=True
)
for k, v in override_dict.as_dict_flat().items():
debug_comments.set_key(
'{}'.format(k),
'Overridden via override dictionary.')
return config_model, debug_comments, overrides, scenario