def add_attributes(model_run):
attr_dict = AttrDict()
attr_dict['model'] = model_run.model.copy()
attr_dict['run'] = model_run.run.copy()
# Some keys are killed right away
for k in ['model.time', 'model.data_path', 'model.timeseries_data_path',
'run.config_run_path', 'run.model']:
try:
attr_dict.del_key(k)
except KeyError:
pass
# Now we flatten the AttrDict into a dict
attr_dict = attr_dict.as_dict(flat=True)
# Anything empty or None in the flattened dict is also killed
for k in list(attr_dict.keys()):
val = attr_dict[k]
if val is None or (hasattr(val, '__iter__') and not val):
del attr_dict[k]
attr_dict['calliope_version'] = __version__
attr_dict['applied_overrides'] = model_run['applied_overrides']
attr_dict['scenario'] = model_run['scenario']
default_tech_dict = checks.defaults.default_tech.as_dict()
default_location_dict = checks.defaults.default_location.as_dict()
attr_dict['defaults'] = ruamel.yaml.dump({
**default_tech_dict['constraints'],
**{'cost_{}'.format(k): v for k, v in default_tech_dict['costs']['default'].items()},
**default_location_dict
})
return attr_dict
def add_attributes(model_run):
attr_dict = AttrDict()
attr_dict['model'] = model_run.model.copy()
attr_dict['run'] = model_run.run.copy()
# Some keys are killed right away
for k in ['model.time', 'model.data_path', 'model.timeseries_data_path',
'run.config_run_path', 'run.model']:
try:
attr_dict.del_key(k)
except KeyError:
pass
# Now we flatten the AttrDict into a dict
attr_dict = attr_dict.as_dict(flat=True)
# Anything empty or None in the flattened dict is also killed
for k in list(attr_dict.keys()):
val = attr_dict[k]
if val is None or (hasattr(val, '__iter__') and not val):
del attr_dict[k]
attr_dict['calliope_version'] = __version__
default_tech_dict = checks.defaults.default_tech.as_dict()
default_location_dict = checks.defaults.default_location.as_dict()
attr_dict['defaults'] = ruamel.yaml.dump({
**default_tech_dict['constraints'],
**{'cost_{}'.format(k): v for k, v in default_tech_dict['costs']['default'].items()},
**default_location_dict
})
return attr_dict
def process_locations(model_config, modelrun_techs):
"""
Process locations by taking an AttrDict that may include compact keys
such as ``1,2,3``, and returning an AttrDict with:
* exactly one key per location with all of its settings
* fully resolved installed technologies for each location
* fully expanded transmission links for each location
Parameters
----------
model_config : AttrDict
modelrun_techs : AttrDict
Returns
-------
locations : AttrDict
locations_comments : AttrDict
"""
techs_in = model_config.techs.copy()
tech_groups_in = model_config.tech_groups
locations_in = model_config.locations
links_in = model_config.get("links", AttrDict())
allowed_from_file = DEFAULTS.model.file_allowed
warnings = []
errors = []
locations_comments = AttrDict()
##
# Expand compressed `loc1,loc2,loc3,loc4: ...` definitions
##
locations = AttrDict()
for key in locations_in:
if ("--" in key) or ("," in key):
key_locs = explode_locations(key)
for subkey in key_locs:
_set_loc_key(locations, subkey, locations_in[key])
else:
_set_loc_key(locations, key, locations_in[key])
##
# Kill any locations that the modeller does not want to exist
##
for loc in list(locations.keys()):
if not locations[loc].get("exists", True):
locations.del_key(loc)
##
# Process technologies
##
techs_to_delete = []
for tech_name in techs_in:
if not techs_in[tech_name].get("exists", True):
techs_to_delete.append(tech_name)
continue
# Get inheritance chain generated in process_techs()
inheritance_chain = modelrun_techs[tech_name].inheritance
# Get and save list of required_constraints from base technology
base_tech = inheritance_chain[-1]
rq = model_config.tech_groups[base_tech].required_constraints
# locations[loc_name].techs[tech_name].required_constraints = rq
techs_in[tech_name].required_constraints = rq
# Kill any techs that the modeller does not want to exist
for tech_name in techs_to_delete:
del techs_in[tech_name]
##
# Fully expand all installed technologies for the location,
# filling in any undefined parameters from defaults
##
location_techs_to_delete = []
for loc_name, loc in locations.items():
if "techs" not in loc:
# Mark this as a transmission-only node if it has not allowed
# any technologies
locations[loc_name].transmission_node = True
locations_comments.set_key(
"{}.transmission_node".format(loc_name),
"Automatically inserted: specifies that this node is "
"a transmission-only node.",
)
continue # No need to process any technologies at this node
for tech_name in loc.techs:
if tech_name in techs_to_delete:
# Techs that were removed need not be further considered
continue
if not isinstance(locations[loc_name].techs[tech_name], dict):
locations[loc_name].techs[tech_name] = AttrDict()
# Starting at top of the inheritance chain, for each level,
# check if the level has location-specific group settings
# and keep merging together the settings, overwriting as we
# go along.
tech_settings = AttrDict()
for parent in reversed(modelrun_techs[tech_name].inheritance):
# Does the parent group have model-wide settings?
tech_settings.union(tech_groups_in[parent],
allow_override=True)
# Does the parent group have location-specific settings?
if ("tech_groups" in locations[loc_name]
and parent in locations[loc_name].tech_groups):
tech_settings.union(
locations[loc_name].tech_groups[parent],
allow_override=True)
# Now overwrite with the tech's own model-wide
# and location-specific settings
tech_settings.union(techs_in[tech_name], allow_override=True)
if tech_name in locations[loc_name].techs:
tech_settings.union(locations[loc_name].techs[tech_name],
allow_override=True)
tech_settings = cleanup_undesired_keys(tech_settings)
# Resolve columns in filename if necessary
file_or_df_configs = [
i for i in tech_settings.keys_nested()
if (isinstance(tech_settings.get_key(i), str) and (
"file=" in tech_settings.get_key(i)
or "df=" in tech_settings.get_key(i)))
]
for config_key in file_or_df_configs:
config_value = tech_settings.get_key(config_key, "")
if ":" not in config_value:
config_value = "{}:{}".format(config_value, loc_name)
tech_settings.set_key(config_key, config_value)
tech_settings = check_costs_and_compute_depreciation_rates(
tech_name, loc_name, tech_settings, warnings, errors)
# Now merge the tech settings into the location-specific
# tech dict -- but if a tech specifies ``exists: false``,
# we kill it at this location
if not tech_settings.get("exists", True):
location_techs_to_delete.append("{}.techs.{}".format(
loc_name, tech_name))
else:
locations[loc_name].techs[tech_name].union(tech_settings,
allow_override=True)
for k in location_techs_to_delete:
locations.del_key(k)
# Generate all transmission links
processed_links = AttrDict()
for link in links_in:
loc_from, loc_to = [i.strip() for i in link.split(",")]
# Skip this link entirely if it has been told not to exist
if not links_in[link].get("exists", True):
continue
# Also skip this link - and warn about it - if it links to a
# now-inexistant (because removed) location
if loc_from not in locations.keys() or loc_to not in locations.keys():
warnings.append(
"Not building the link {},{} because one or both of its "
"locations have been removed from the model by setting "
"``exists: false``".format(loc_from, loc_to))
continue
processed_transmission_techs = AttrDict()
for tech_name in links_in[link].techs:
# Skip techs that have been told not to exist
# for this particular link
if not links_in[link].get_key("techs.{}.exists".format(tech_name),
True):
continue
if tech_name not in processed_transmission_techs:
tech_settings = AttrDict()
# Combine model-wide settings from all parent groups
for parent in reversed(modelrun_techs[tech_name].inheritance):
tech_settings.union(tech_groups_in[parent],
allow_override=True)
# Now overwrite with the tech's own model-wide settings
tech_settings.union(techs_in[tech_name], allow_override=True)
# Add link-specific constraint overrides
if links_in[link].techs[tech_name]:
tech_settings.union(links_in[link].techs[tech_name],
allow_override=True)
tech_settings = cleanup_undesired_keys(tech_settings)
tech_settings = process_per_distance_constraints(
tech_name,
tech_settings,
locations,
locations_comments,
loc_from,
loc_to,
)
tech_settings = check_costs_and_compute_depreciation_rates(
tech_name, link, tech_settings, warnings, errors)
processed_transmission_techs[tech_name] = tech_settings
else:
tech_settings = processed_transmission_techs[tech_name]
processed_links.set_key(
"{}.links.{}.techs.{}".format(loc_from, loc_to, tech_name),
tech_settings.copy(),
)
processed_links.set_key(
"{}.links.{}.techs.{}".format(loc_to, loc_from, tech_name),
tech_settings.copy(),
)
# If this is a one-way link, we set the constraints for energy_prod
# and energy_con accordingly on both parts of the link
if tech_settings.get_key("constraints.one_way", False):
processed_links.set_key(
"{}.links.{}.techs.{}.constraints.energy_prod".format(
loc_from, loc_to, tech_name),
False,
)
processed_links.set_key(
"{}.links.{}.techs.{}.constraints.energy_con".format(
loc_to, loc_from, tech_name),
False,
)
locations.union(processed_links, allow_override=True)
return locations, locations_comments, list(set(warnings)), list(
set(errors))
def process_locations(model_config, modelrun_techs):
"""
Process locations by taking an AttrDict that may include compact keys
such as ``1,2,3``, and returning an AttrDict with:
* exactly one key per location with all of its settings
* fully resolved installed technologies for each location
* fully expanded transmission links for each location
Parameters
----------
model_config : AttrDict
modelrun_techs : AttrDict
Returns
-------
locations : AttrDict
locations_comments : AttrDict
"""
techs_in = model_config.techs.copy()
tech_groups_in = model_config.tech_groups
locations_in = model_config.locations
links_in = model_config.get('links', AttrDict())
allowed_from_file = defaults['file_allowed']
warnings = []
errors = []
locations_comments = AttrDict()
##
# Expand compressed `loc1,loc2,loc3,loc4: ...` definitions
##
locations = AttrDict()
for key in locations_in:
if ('--' in key) or (',' in key):
key_locs = explode_locations(key)
for subkey in key_locs:
_set_loc_key(locations, subkey, locations_in[key])
else:
_set_loc_key(locations, key, locations_in[key])
##
# Kill any locations that the modeller does not want to exist
##
for loc in list(locations.keys()):
if not locations[loc].get('exists', True):
locations.del_key(loc)
##
# Process technologies
##
techs_to_delete = []
for tech_name in techs_in:
if not techs_in[tech_name].get('exists', True):
techs_to_delete.append(tech_name)
continue
# Get inheritance chain generated in process_techs()
inheritance_chain = modelrun_techs[tech_name].inheritance
# Get and save list of required_constraints from base technology
base_tech = inheritance_chain[-1]
rq = model_config.tech_groups[base_tech].required_constraints
# locations[loc_name].techs[tech_name].required_constraints = rq
techs_in[tech_name].required_constraints = rq
# Kill any techs that the modeller does not want to exist
for tech_name in techs_to_delete:
del techs_in[tech_name]
##
# Fully expand all installed technologies for the location,
# filling in any undefined parameters from defaults
##
location_techs_to_delete = []
for loc_name, loc in locations.items():
if 'techs' not in loc:
# Mark this as a transmission-only node if it has not allowed
# any technologies
locations[loc_name].transmission_node = True
locations_comments.set_key(
'{}.transmission_node'.format(loc_name),
'Automatically inserted: specifies that this node is '
'a transmission-only node.'
)
continue # No need to process any technologies at this node
for tech_name in loc.techs:
if tech_name in techs_to_delete:
# Techs that were removed need not be further considered
continue
if not isinstance(locations[loc_name].techs[tech_name], dict):
locations[loc_name].techs[tech_name] = AttrDict()
# Starting at top of the inheritance chain, for each level,
# check if the level has location-specific group settings
# and keep merging together the settings, overwriting as we
# go along.
tech_settings = AttrDict()
for parent in reversed(modelrun_techs[tech_name].inheritance):
# Does the parent group have model-wide settings?
tech_settings.union(tech_groups_in[parent], allow_override=True)
# Does the parent group have location-specific settings?
if ('tech_groups' in locations[loc_name] and
parent in locations[loc_name].tech_groups):
tech_settings.union(
locations[loc_name].tech_groups[parent],
allow_override=True)
# Now overwrite with the tech's own model-wide
# and location-specific settings
tech_settings.union(techs_in[tech_name], allow_override=True)
if tech_name in locations[loc_name].techs:
tech_settings.union(
locations[loc_name].techs[tech_name],
allow_override=True)
tech_settings = cleanup_undesired_keys(tech_settings)
# Resolve columns in filename if necessary
file_configs = [
i for i in tech_settings.keys_nested()
if (isinstance(tech_settings.get_key(i), str) and
'file=' in tech_settings.get_key(i))
]
for config_key in file_configs:
if config_key.split('.')[-1] not in allowed_from_file:
# Allow any custom settings that end with _time_varying
# FIXME: add this to docs
if config_key.endswith('_time_varying'):
warn('Using custom constraint '
'{} with time-varying data.'.format(config_key))
else:
raise ModelError('`file=` not allowed in {}'.format(config_key))
config_value = tech_settings.get_key(config_key, '')
if ':' not in config_value:
config_value = '{}:{}'.format(config_value, loc_name)
tech_settings.set_key(config_key, config_value)
tech_settings = compute_depreciation_rates(tech_name, tech_settings, warnings, errors)
# Now merge the tech settings into the location-specific
# tech dict -- but if a tech specifies ``exists: false``,
# we kill it at this location
if not tech_settings.get('exists', True):
location_techs_to_delete.append('{}.techs.{}'.format(loc_name, tech_name))
else:
locations[loc_name].techs[tech_name].union(
tech_settings, allow_override=True
)
for k in location_techs_to_delete:
locations.del_key(k)
# Generate all transmission links
processed_links = AttrDict()
for link in links_in:
loc_from, loc_to = link.split(',')
# Skip this link entirely if it has been told not to exist
if not links_in[link].get('exists', True):
continue
# Also skip this link - and warn about it - if it links to a
# now-inexistant (because removed) location
if (loc_from not in locations.keys() or loc_to not in locations.keys()):
warnings.append(
'Not building the link {},{} because one or both of its '
'locations have been removed from the model by setting '
'``exists: false``'.format(loc_from, loc_to)
)
continue
processed_transmission_techs = AttrDict()
for tech_name in links_in[link].techs:
# Skip techs that have been told not to exist
# for this particular link
if not links_in[link].get_key('techs.{}.exists'.format(tech_name), True):
continue
if tech_name not in processed_transmission_techs:
tech_settings = AttrDict()
# Combine model-wide settings from all parent groups
for parent in reversed(modelrun_techs[tech_name].inheritance):
tech_settings.union(
tech_groups_in[parent],
allow_override=True
)
# Now overwrite with the tech's own model-wide settings
tech_settings.union(
techs_in[tech_name],
allow_override=True
)
# Add link-specific constraint overrides
if links_in[link].techs[tech_name]:
tech_settings.union(
links_in[link].techs[tech_name],
allow_override=True
)
tech_settings = cleanup_undesired_keys(tech_settings)
tech_settings = process_per_distance_constraints(tech_name, tech_settings, locations, locations_comments, loc_from, loc_to)
tech_settings = compute_depreciation_rates(tech_name, tech_settings, warnings, errors)
processed_transmission_techs[tech_name] = tech_settings
else:
tech_settings = processed_transmission_techs[tech_name]
processed_links.set_key(
'{}.links.{}.techs.{}'.format(loc_from, loc_to, tech_name),
tech_settings.copy()
)
processed_links.set_key(
'{}.links.{}.techs.{}'.format(loc_to, loc_from, tech_name),
tech_settings.copy()
)
# If this is a one-way link, we set the constraints for energy_prod
# and energy_con accordingly on both parts of the link
if tech_settings.get_key('constraints.one_way', False):
processed_links.set_key(
'{}.links.{}.techs.{}.constraints.energy_prod'.format(loc_from, loc_to, tech_name),
False)
processed_links.set_key(
'{}.links.{}.techs.{}.constraints.energy_con'.format(loc_to, loc_from, tech_name),
False)
locations.union(processed_links, allow_override=True)
return locations, locations_comments, list(set(warnings)), list(set(errors))
