def generate_simple_sets(model_run):
"""
Generate basic sets for a given pre-processed ``model_run``.
Parameters
----------
model_run : AttrDict
"""
sets = AttrDict()
flat_techs = model_run.techs.as_dict(flat=True)
flat_locations = model_run.locations.as_dict(flat=True)
sets.resources = set(
flatten_list(v for k, v in flat_techs.items() if '.carrier' in k))
sets.carriers = sets.resources - set(['resource'])
sets.carrier_tiers = set(
key.split('.carrier_')[1] for key in flat_techs.keys()
if '.carrier_' in key)
sets.costs = set(
k.split('costs.')[-1].split('.')[0] for k in flat_locations.keys()
if '.costs.' in k)
sets.locs = set(model_run.locations.keys())
sets.techs_non_transmission = set(k for k, v in model_run.techs.items()
if v.inheritance[-1] != 'transmission')
sets.techs_transmission_names = set(k for k, v in model_run.techs.items()
if v.inheritance[-1] == 'transmission')
# This builds the "tech:loc" expansion of transmission technologies
techs_transmission = set()
for loc_name, loc_config in model_run.locations.items():
for link_name, link_config in loc_config.get('links', {}).items():
for tech_name in link_config.techs:
techs_transmission.add('{}:{}'.format(tech_name, link_name))
sets.techs_transmission = techs_transmission
sets.techs = sets.techs_non_transmission | sets.techs_transmission_names
# this extracts location coordinate information
coordinates = set(
k.split('.')[-1] for k in flat_locations.keys()
if '.coordinates.' in k)
if coordinates:
sets.coordinates = coordinates
# `timesteps` set is built from the results of timeseries_data processing
sets.timesteps = list(model_run.timesteps.astype(str))
model_run.del_key('timesteps')
# `techlists` are strings with comma-separated techs used for grouping in
# some model-wide constraints
sets.techlists = set()
for k in model_run.model.get_key('group_share', {}).keys():
sets.techlists.add(k)
return sets
def generate_simple_sets(model_run):
"""
Generate basic sets for a given pre-processed ``model_run``.
Parameters
----------
model_run : AttrDict
"""
sets = AttrDict()
flat_techs = model_run.techs.as_dict(flat=True)
flat_locations = model_run.locations.as_dict(flat=True)
sets.resources = set(
flatten_list(v for k, v in flat_techs.items() if '.carrier' in k))
sets.carriers = sets.resources - set(['resource'])
sets.carrier_tiers = set(
key.split('.carrier_')[1] for key in flat_techs.keys()
if '.carrier_' in key)
sets.costs = set(
k.split('costs.')[-1].split('.')[0] for k in flat_locations.keys()
if '.costs.' in k)
sets.locs = set(model_run.locations.keys())
sets.techs_non_transmission = set()
tech_groups = [
'demand', 'supply', 'supply_plus', 'conversion', 'conversion_plus',
'storage'
]
for tech_group in tech_groups:
sets['techs_{}'.format(tech_group)] = set(
k for k, v in model_run.techs.items()
if v.inheritance[-1] == tech_group)
sets.techs_non_transmission.update(sets['techs_{}'.format(tech_group)])
sets.techs_transmission_names = set(k for k, v in model_run.techs.items()
if v.inheritance[-1] == 'transmission')
# This builds the "tech:loc" expansion of transmission technologies
techs_transmission = set()
for loc_name, loc_config in model_run.locations.items():
for link_name, link_config in loc_config.get('links', {}).items():
for tech_name in link_config.techs:
techs_transmission.add('{}:{}'.format(tech_name, link_name))
sets.techs_transmission = techs_transmission
sets.techs = sets.techs_non_transmission | sets.techs_transmission_names
# this extracts location coordinate information
coordinates = set(
k.split('.')[-1] for k in flat_locations.keys()
if '.coordinates.' in k)
if coordinates:
sets.coordinates = coordinates
# `timesteps` set is built from the results of timeseries_data processing
sets.timesteps = list(model_run.timesteps.astype(str))
model_run.del_key('timesteps')
# `techlists` are strings with comma-separated techs used for grouping in
# some model-wide constraints
sets.techlists = set()
for k in model_run.model.get_key('group_share', {}).keys():
sets.techlists.add(k)
# `constraint_groups` are the group names per constraint that is defined
# at a group level
sets.group_constraints = set()
group_constraints = AttrDict({
name: data
for name, data in model_run['group_constraints'].items()
if data.get("exists", True)
})
if len(group_constraints.keys()) > 0:
sets.group_constraints.update(
i.split('.')[1] for i in group_constraints.as_dict_flat().keys()
if i.split('.')[1] not in ['techs', 'locs'])
for constr in sets.group_constraints:
sets['group_names_' + constr] = set(
k for k, v in group_constraints.items() if constr in v.keys())
return sets
def process_techs(config_model):
default_palette_cycler = itertools.cycle(range(len(_DEFAULT_PALETTE)))
result = AttrDict()
errors = []
debug_comments = AttrDict()
for tech_id, tech_config in config_model.techs.items():
# If a tech specifies ``exists: false``, we skip it entirely
if not tech_config.get('exists', True):
continue
tech_result = AttrDict()
# Add inheritance chain
tech_result.inheritance = get_parents(tech_id, config_model)
# CHECK: A tech's parent must lead to one of the built-in tech_groups
builtin_tech_groups = checks.defaults_model.tech_groups.keys()
if tech_result.inheritance[-1] not in builtin_tech_groups:
errors.append(
'tech {} must inherit from a built-in tech group'.format(tech_id)
)
# Process inheritance
tech_result.essentials = AttrDict()
tech_result.constraints = AttrDict()
for parent in reversed(tech_result.inheritance):
# Does the parent group have model-wide settings?
parent_essentials = config_model.tech_groups[parent].essentials
parent_systemwide_constraints = util.get_systemwide_constraints(
config_model.tech_groups[parent]
)
for k in parent_essentials.as_dict_flat():
debug_comments.set_key(
'{}.essentials.{}'.format(tech_id, k),
'From parent tech_group `{}`'.format(parent)
)
tech_result.essentials.union(parent_essentials, allow_override=True)
tech_result.constraints.union(parent_systemwide_constraints, allow_override=True)
# Add this tech's essentials and constraints, overwriting any essentials from parents
tech_result.essentials.union(tech_config.essentials, allow_override=True)
tech_result.constraints.union(
util.get_systemwide_constraints(tech_config), allow_override=True
)
# Add allowed_constraints and required_constraints from base tech
keys_to_add = ['required_constraints', 'allowed_constraints', 'allowed_costs']
for k in keys_to_add:
tech_result[k] = config_model.tech_groups[tech_result.inheritance[-1]].get(k, [])
# CHECK: If necessary, populate carrier_in and carrier_out in essentials, but
# also break on missing carrier data
if 'carrier_in' not in tech_result.essentials:
if tech_result.inheritance[-1] in ['supply', 'supply_plus']:
tech_result.essentials.carrier_in = 'resource'
elif tech_result.inheritance[-1] in ['demand', 'transmission',
'storage']:
try:
tech_result.essentials.carrier_in = \
tech_result.essentials.carrier
debug_comments.set_key(
'{}.essentials.carrier_in'.format(tech_id),
'Set from essentials.carrier'
)
except KeyError:
errors.append(
'`carrier` or `carrier_in` must be '
'defined for {}'.format(tech_id)
)
else:
errors.append(
'`carrier_in` must be defined for {}'.format(tech_id)
)
if 'carrier_out' not in tech_result.essentials:
if tech_result.inheritance[-1] == 'demand':
tech_result.essentials.carrier_out = 'resource'
elif tech_result.inheritance[-1] in ['supply', 'supply_plus',
'transmission', 'storage']:
try:
tech_result.essentials.carrier_out = \
tech_result.essentials.carrier
except KeyError:
errors.append(
'`carrier` or `carrier_out` must be '
'defined for {}'.format(tech_id)
)
else:
errors.append(
'`carrier_out` must be defined for {}'.format(tech_id)
)
# Deal with primary carrier in/out for conversion_plus techs
if tech_result.inheritance[-1] == 'conversion_plus':
for direction in ['_in', '_out']:
carriers = set(util.flatten_list([
v for k, v in tech_result.essentials.items()
if k.startswith('carrier' + direction)
]))
primary_carrier = tech_result.essentials.get(
'primary_carrier' + direction, None
)
if primary_carrier is None and len(carriers) == 1:
tech_result.essentials['primary_carrier' + direction] = carriers.pop()
elif primary_carrier is None and len(carriers) > 1:
errors.append(
'Primary_carrier{0} must be assigned for tech `{1}` as '
'there are multiple carriers{0}'.format(direction, tech_id)
)
elif primary_carrier not in carriers:
errors.append(
'Primary_carrier{0} `{1}` not one of the available carriers'
'{0} for `{2}`'.format(direction, primary_carrier, tech_id)
)
# If necessary, pick a color for the tech, cycling through
# the hardcoded default palette
if not tech_result.essentials.get_key('color', None):
color = _DEFAULT_PALETTE[next(default_palette_cycler)]
tech_result.essentials.color = color
debug_comments.set_key(
'{}.essentials.color'.format(tech_id),
'From Calliope default palette')
result[tech_id] = tech_result
return result, debug_comments, errors
def _check_tech(model_run, tech_id, tech_config, loc_id, warnings, errors, comments):
"""
Checks individual tech/tech groups at specific locations.
NOTE: Updates `warnings` and `errors` lists in-place.
"""
if tech_id not in model_run.techs:
warnings.append(
'Tech {} was removed by setting ``exists: False`` - not checking '
'the consistency of its constraints at location {}.'.format(tech_id, loc_id)
)
return warnings, errors
required = model_run.techs[tech_id].required_constraints
allowed = model_run.techs[tech_id].allowed_constraints
allowed_costs = model_run.techs[tech_id].allowed_costs
all_defaults = list(defaults.default_tech.constraints.keys())
# Error if required constraints are not defined
for r in required:
# If it's a string, it must be defined
single_ok = isinstance(r, str) and r in tech_config.constraints
# If it's a list of strings, one of them must be defined
multiple_ok = (
isinstance(r, list) and
any([i in tech_config.constraints for i in r])
)
if not single_ok and not multiple_ok:
errors.append(
'`{}` at `{}` fails to define '
'all required constraints: {}'.format(tech_id, loc_id, required)
)
# print('{} -- {}-{}: {}, {}'.format(r, loc_id, tech_id, single_ok, multiple_ok))
# If the technology is supply_plus, check if it has storage_cap_max. If yes, it needs charge rate
if model_run.techs[tech_id].essentials.parent == 'supply_plus':
if (any(['storage_cap_' in k for k in tech_config.constraints.keys()])
and 'charge_rate' not in tech_config.constraints.keys()):
errors.append(
'`{}` at `{}` fails to define '
'charge_rate, but is using storage'.format(tech_id, loc_id, required)
)
# If a technology is defined by units (i.e. integer decision variable), it must define energy_cap_per_unit
if (any(['units_' in k for k in tech_config.constraints.keys()])
and 'energy_cap_per_unit' not in tech_config.constraints.keys()):
errors.append(
'`{}` at `{}` fails to define energy_cap_per_unit when specifying '
'technology in units_max/min/equals'.format(tech_id, loc_id, required)
)
# If a technology is defined by units & is a storage tech, it must define storage_cap_per_unit
if (any(['units_' in k for k in tech_config.constraints.keys()])
and model_run.techs[tech_id].essentials.parent in ['storage', 'supply_plus']
and any(['storage' in k for k in tech_config.constraints.keys()])
and 'storage_cap_per_unit' not in tech_config.constraints.keys()):
errors.append(
'`{}` at `{}` fails to define storage_cap_per_unit when specifying '
'technology in units_max/min/equals'.format(tech_id, loc_id, required)
)
# If a technology is defines force_resource but is not in loc_techs_finite_resource
if ('force_resource' in tech_config.constraints.keys() and
loc_id + '::' + tech_id not in model_run.sets.loc_techs_finite_resource):
warnings.append(
'`{}` at `{}` defines force_resource but not a finite resource, so '
'force_resource will not be applied'.format(tech_id, loc_id)
)
# Flatten required list and gather remaining unallowed constraints
required_f = flatten_list(required)
remaining = set(tech_config.constraints) - set(required_f) - set(allowed)
# Error if something is defined that's not allowed, but is in defaults
# Warn if something is defined that's not allowed, but is not in defaults
# (it could be a misspelling)
for k in remaining:
if k in all_defaults:
errors.append(
'`{}` at `{}` defines non-allowed '
'constraint `{}`'.format(tech_id, loc_id, k)
)
else:
warnings.append(
'`{}` at `{}` defines unrecognised '
'constraint `{}` - possibly a misspelling?'.format(tech_id, loc_id, k)
)
# Error if an `export` statement does not match the given carrier_outs
if 'export_carrier' in tech_config.constraints:
essentials = model_run.techs[tech_id].essentials
export = tech_config.constraints.export_carrier
if (export and export not in [essentials.get_key(k, '')
for k in ['carrier_out', 'carrier_out_2', 'carrier_out_3']]):
errors.append(
'`{}` at `{}` is attempting to export a carrier '
'not given as an output carrier: `{}`'.format(tech_id, loc_id, export)
)
# Error if non-allowed costs are defined
for cost_class in tech_config.get_key('costs', {}):
for k in tech_config.costs[cost_class]:
if k not in allowed_costs:
errors.append(
'`{}` at `{}` defines non-allowed '
'{} cost: `{}`'.format(tech_id, loc_id, cost_class, k)
)
return warnings, errors
def generate_simple_sets(model_run):
"""
Generate basic sets for a given pre-processed ``model_run``.
Parameters
----------
model_run : AttrDict
"""
sets = AttrDict()
flat_techs = model_run.techs.as_dict(flat=True)
flat_locations = model_run.locations.as_dict(flat=True)
sets.resources = set(flatten_list(
v for k, v in flat_techs.items()
if '.carrier' in k
))
sets.carriers = sets.resources - set(['resource'])
sets.carrier_tiers = set(
key.split('.carrier_')[1]
for key in flat_techs.keys()
if '.carrier_' in key
)
sets.costs = set(
k.split('costs.')[-1].split('.')[0]
for k in flat_locations.keys()
if '.costs.' in k)
sets.locs = set(model_run.locations.keys())
sets.techs_non_transmission = set(
k for k, v in model_run.techs.items()
if v.inheritance[-1] != 'transmission')
sets.techs_transmission_names = set(
k for k, v in model_run.techs.items()
if v.inheritance[-1] == 'transmission')
# This builds the "tech:loc" expansion of transmission technologies
techs_transmission = set()
for loc_name, loc_config in model_run.locations.items():
for link_name, link_config in loc_config.get('links', {}).items():
for tech_name in link_config.techs:
techs_transmission.add('{}:{}'.format(tech_name, link_name))
sets.techs_transmission = techs_transmission
sets.techs = sets.techs_non_transmission | sets.techs_transmission_names
# this extracts location coordinate information
coordinates = set(
k.split('.')[-1] for k in flat_locations.keys()
if '.coordinates.' in k)
if coordinates:
sets.coordinates = coordinates
# `timesteps` set is built from the results of timeseries_data processing
sets.timesteps = list(model_run.timesteps.astype(str))
model_run.del_key('timesteps')
# `techlists` are strings with comma-separated techs used for grouping in
# some model-wide constraints
sets.techlists = set()
for k in model_run.model.get_key('group_share', {}).keys():
sets.techlists.add(k)
return sets
def process_techs(config_model):
default_palette_cycler = itertools.cycle(range(len(_DEFAULT_PALETTE)))
result = AttrDict()
errors = []
debug_comments = AttrDict()
for tech_id, tech_config in config_model.techs.items():
# If a tech specifies ``exists: false``, we skip it entirely
if not tech_config.get('exists', True):
continue
tech_result = AttrDict()
# Add inheritance chain
tech_result.inheritance = get_parents(tech_id, config_model)
# CHECK: A tech's parent must lead to one of the built-in tech_groups
builtin_tech_groups = checks.defaults_model.tech_groups.keys()
if tech_result.inheritance[-1] not in builtin_tech_groups:
errors.append(
'tech {} must inherit from a built-in tech group'.format(tech_id)
)
# Process inheritance
tech_result.essentials = AttrDict()
tech_result.constraints = AttrDict()
for parent in reversed(tech_result.inheritance):
# Does the parent group have model-wide settings?
parent_essentials = config_model.tech_groups[parent].essentials
parent_systemwide_constraints = util.get_systemwide_constraints(
config_model.tech_groups[parent]
)
for k in parent_essentials.as_dict_flat():
debug_comments.set_key(
'{}.essentials.{}'.format(tech_id, k),
'From parent tech_group `{}`'.format(parent)
)
tech_result.essentials.union(parent_essentials, allow_override=True)
tech_result.constraints.union(parent_systemwide_constraints, allow_override=True)
# Add this tech's essentials and constraints, overwriting any essentials from parents
tech_result.essentials.union(tech_config.essentials, allow_override=True)
tech_result.constraints.union(
util.get_systemwide_constraints(tech_config), allow_override=True
)
# Add allowed_constraints and required_constraints from base tech
keys_to_add = ['required_constraints', 'allowed_constraints', 'allowed_costs']
for k in keys_to_add:
tech_result[k] = config_model.tech_groups[tech_result.inheritance[-1]].get(k, [])
# CHECK: If necessary, populate carrier_in and carrier_out in essentials, but
# also break on missing carrier data
if 'carrier_in' not in tech_result.essentials:
if tech_result.inheritance[-1] in ['supply', 'supply_plus']:
tech_result.essentials.carrier_in = 'resource'
elif tech_result.inheritance[-1] in ['demand', 'transmission',
'storage']:
try:
tech_result.essentials.carrier_in = \
tech_result.essentials.carrier
debug_comments.set_key(
'{}.essentials.carrier_in'.format(tech_id),
'Set from essentials.carrier'
)
except KeyError:
errors.append(
'`carrier` or `carrier_in` must be '
'defined for {}'.format(tech_id)
)
else:
errors.append(
'`carrier_in` must be defined for {}'.format(tech_id)
)
if 'carrier_out' not in tech_result.essentials:
if tech_result.inheritance[-1] == 'demand':
tech_result.essentials.carrier_out = 'resource'
elif tech_result.inheritance[-1] in ['supply', 'supply_plus',
'transmission', 'storage']:
try:
tech_result.essentials.carrier_out = \
tech_result.essentials.carrier
except KeyError:
errors.append(
'`carrier` or `carrier_out` must be '
'defined for {}'.format(tech_id)
)
else:
errors.append(
'`carrier_out` must be defined for {}'.format(tech_id)
)
# Deal with primary carrier in/out for conversion_plus techs
if tech_result.inheritance[-1] == 'conversion_plus':
for direction in ['_in', '_out']:
carriers = set(util.flatten_list([
v for k, v in tech_result.essentials.items()
if k.startswith('carrier' + direction)
]))
primary_carrier = tech_result.essentials.get(
'primary_carrier' + direction, None
)
if primary_carrier is None and len(carriers) == 1:
tech_result.essentials['primary_carrier' + direction] = carriers.pop()
elif primary_carrier is None and len(carriers) > 1:
errors.append(
'Primary_carrier{0} must be assigned for tech `{1}` as '
'there are multiple carriers{0}'.format(direction, tech_id)
)
elif primary_carrier not in carriers:
errors.append(
'Primary_carrier{0} `{1}` not one of the available carriers'
'{0} for `{2}`'.format(direction, primary_carrier, tech_id)
)
# If necessary, pick a color for the tech, cycling through
# the hardcoded default palette
if not tech_result.essentials.get_key('color', None):
color = _DEFAULT_PALETTE[next(default_palette_cycler)]
tech_result.essentials.color = color
debug_comments.set_key(
'{}.essentials.color'.format(tech_id),
'From Calliope default palette')
result[tech_id] = tech_result
return result, debug_comments, errors