def process_locations(model_config, modelrun_techs):
"""
Process locations by taking an AttrDict that may include compact keys
such as ``1,2,3``, and returning an AttrDict with:
* exactly one key per location with all of its settings
* fully resolved installed technologies for each location
* fully expanded transmission links for each location
Parameters
----------
model_config : AttrDict
modelrun_techs : AttrDict
Returns
-------
locations : AttrDict
locations_comments : AttrDict
"""
techs_in = model_config.techs.copy()
tech_groups_in = model_config.tech_groups
locations_in = model_config.locations
links_in = model_config.get("links", AttrDict())
allowed_from_file = DEFAULTS.model.file_allowed
warnings = []
errors = []
locations_comments = AttrDict()
##
# Expand compressed `loc1,loc2,loc3,loc4: ...` definitions
##
locations = AttrDict()
for key in locations_in:
if ("--" in key) or ("," in key):
key_locs = explode_locations(key)
for subkey in key_locs:
_set_loc_key(locations, subkey, locations_in[key])
else:
_set_loc_key(locations, key, locations_in[key])
##
# Kill any locations that the modeller does not want to exist
##
for loc in list(locations.keys()):
if not locations[loc].get("exists", True):
locations.del_key(loc)
##
# Process technologies
##
techs_to_delete = []
for tech_name in techs_in:
if not techs_in[tech_name].get("exists", True):
techs_to_delete.append(tech_name)
continue
# Get inheritance chain generated in process_techs()
inheritance_chain = modelrun_techs[tech_name].inheritance
# Get and save list of required_constraints from base technology
base_tech = inheritance_chain[-1]
rq = model_config.tech_groups[base_tech].required_constraints
# locations[loc_name].techs[tech_name].required_constraints = rq
techs_in[tech_name].required_constraints = rq
# Kill any techs that the modeller does not want to exist
for tech_name in techs_to_delete:
del techs_in[tech_name]
##
# Fully expand all installed technologies for the location,
# filling in any undefined parameters from defaults
##
location_techs_to_delete = []
for loc_name, loc in locations.items():
if "techs" not in loc:
# Mark this as a transmission-only node if it has not allowed
# any technologies
locations[loc_name].transmission_node = True
locations_comments.set_key(
"{}.transmission_node".format(loc_name),
"Automatically inserted: specifies that this node is "
"a transmission-only node.",
)
continue # No need to process any technologies at this node
for tech_name in loc.techs:
if tech_name in techs_to_delete:
# Techs that were removed need not be further considered
continue
if not isinstance(locations[loc_name].techs[tech_name], dict):
locations[loc_name].techs[tech_name] = AttrDict()
# Starting at top of the inheritance chain, for each level,
# check if the level has location-specific group settings
# and keep merging together the settings, overwriting as we
# go along.
tech_settings = AttrDict()
for parent in reversed(modelrun_techs[tech_name].inheritance):
# Does the parent group have model-wide settings?
tech_settings.union(tech_groups_in[parent],
allow_override=True)
# Does the parent group have location-specific settings?
if ("tech_groups" in locations[loc_name]
and parent in locations[loc_name].tech_groups):
tech_settings.union(
locations[loc_name].tech_groups[parent],
allow_override=True)
# Now overwrite with the tech's own model-wide
# and location-specific settings
tech_settings.union(techs_in[tech_name], allow_override=True)
if tech_name in locations[loc_name].techs:
tech_settings.union(locations[loc_name].techs[tech_name],
allow_override=True)
tech_settings = cleanup_undesired_keys(tech_settings)
# Resolve columns in filename if necessary
file_or_df_configs = [
i for i in tech_settings.keys_nested()
if (isinstance(tech_settings.get_key(i), str) and (
"file=" in tech_settings.get_key(i)
or "df=" in tech_settings.get_key(i)))
]
for config_key in file_or_df_configs:
config_value = tech_settings.get_key(config_key, "")
if ":" not in config_value:
config_value = "{}:{}".format(config_value, loc_name)
tech_settings.set_key(config_key, config_value)
tech_settings = check_costs_and_compute_depreciation_rates(
tech_name, loc_name, tech_settings, warnings, errors)
# Now merge the tech settings into the location-specific
# tech dict -- but if a tech specifies ``exists: false``,
# we kill it at this location
if not tech_settings.get("exists", True):
location_techs_to_delete.append("{}.techs.{}".format(
loc_name, tech_name))
else:
locations[loc_name].techs[tech_name].union(tech_settings,
allow_override=True)
for k in location_techs_to_delete:
locations.del_key(k)
# Generate all transmission links
processed_links = AttrDict()
for link in links_in:
loc_from, loc_to = [i.strip() for i in link.split(",")]
# Skip this link entirely if it has been told not to exist
if not links_in[link].get("exists", True):
continue
# Also skip this link - and warn about it - if it links to a
# now-inexistant (because removed) location
if loc_from not in locations.keys() or loc_to not in locations.keys():
warnings.append(
"Not building the link {},{} because one or both of its "
"locations have been removed from the model by setting "
"``exists: false``".format(loc_from, loc_to))
continue
processed_transmission_techs = AttrDict()
for tech_name in links_in[link].techs:
# Skip techs that have been told not to exist
# for this particular link
if not links_in[link].get_key("techs.{}.exists".format(tech_name),
True):
continue
if tech_name not in processed_transmission_techs:
tech_settings = AttrDict()
# Combine model-wide settings from all parent groups
for parent in reversed(modelrun_techs[tech_name].inheritance):
tech_settings.union(tech_groups_in[parent],
allow_override=True)
# Now overwrite with the tech's own model-wide settings
tech_settings.union(techs_in[tech_name], allow_override=True)
# Add link-specific constraint overrides
if links_in[link].techs[tech_name]:
tech_settings.union(links_in[link].techs[tech_name],
allow_override=True)
tech_settings = cleanup_undesired_keys(tech_settings)
tech_settings = process_per_distance_constraints(
tech_name,
tech_settings,
locations,
locations_comments,
loc_from,
loc_to,
)
tech_settings = check_costs_and_compute_depreciation_rates(
tech_name, link, tech_settings, warnings, errors)
processed_transmission_techs[tech_name] = tech_settings
else:
tech_settings = processed_transmission_techs[tech_name]
processed_links.set_key(
"{}.links.{}.techs.{}".format(loc_from, loc_to, tech_name),
tech_settings.copy(),
)
processed_links.set_key(
"{}.links.{}.techs.{}".format(loc_to, loc_from, tech_name),
tech_settings.copy(),
)
# If this is a one-way link, we set the constraints for energy_prod
# and energy_con accordingly on both parts of the link
if tech_settings.get_key("constraints.one_way", False):
processed_links.set_key(
"{}.links.{}.techs.{}.constraints.energy_prod".format(
loc_from, loc_to, tech_name),
False,
)
processed_links.set_key(
"{}.links.{}.techs.{}.constraints.energy_con".format(
loc_to, loc_from, tech_name),
False,
)
locations.union(processed_links, allow_override=True)
return locations, locations_comments, list(set(warnings)), list(
set(errors))
def generate_model_run(config, debug_comments, applied_overrides, scenario):
"""
Returns a processed model_run configuration AttrDict and a debug
YAML object with comments attached, ready to write to disk.
Parameters
----------
config : AttrDict
debug_comments : AttrDict
"""
model_run = AttrDict()
model_run['scenario'] = scenario
model_run['applied_overrides'] = ';'.join(applied_overrides)
# 1) Initial checks on model configuration
warnings, errors = checks.check_initial(config)
exceptions.print_warnings_and_raise_errors(warnings=warnings, errors=errors)
# 2) Fully populate techs
# Raises ModelError if necessary
model_run['techs'], debug_techs, errors = process_techs(config)
debug_comments.set_key('model_run.techs', debug_techs)
exceptions.print_warnings_and_raise_errors(errors=errors)
# 3) Fully populate tech_groups
model_run['tech_groups'] = process_tech_groups(config, model_run['techs'])
# 4) Fully populate locations
model_run['locations'], debug_locs, warnings, errors = locations.process_locations(
config, model_run['techs']
)
debug_comments.set_key('model_run.locations', debug_locs)
exceptions.print_warnings_and_raise_errors(warnings=warnings, errors=errors)
# 5) Fully populate timeseries data
# Raises ModelErrors if there are problems with timeseries data at this stage
model_run['timeseries_data'], model_run['timesteps'] = (
process_timeseries_data(config, model_run)
)
# 6) Grab additional relevant bits from run and model config
model_run['run'] = config['run']
model_run['model'] = config['model']
# 7) Initialize sets
all_sets = sets.generate_simple_sets(model_run)
all_sets.union(sets.generate_loc_tech_sets(model_run, all_sets))
all_sets = AttrDict({k: list(v) for k, v in all_sets.items()})
model_run['sets'] = all_sets
model_run['constraint_sets'] = constraint_sets.generate_constraint_sets(model_run)
# 8) Final sense-checking
final_check_comments, warnings, errors = checks.check_final(model_run)
debug_comments.union(final_check_comments)
exceptions.print_warnings_and_raise_errors(warnings=warnings, errors=errors)
# 9) Build a debug data dict with comments and the original configs
debug_data = AttrDict({
'comments': debug_comments,
'config_initial': config,
})
return model_run, debug_data
def generate_simple_sets(model_run):
"""
Generate basic sets for a given pre-processed ``model_run``.
Parameters
----------
model_run : AttrDict
"""
sets = AttrDict()
flat_techs = model_run.techs.as_dict(flat=True)
flat_locations = model_run.locations.as_dict(flat=True)
sets.resources = set(
flatten_list(v for k, v in flat_techs.items() if '.carrier' in k))
sets.carriers = sets.resources - set(['resource'])
sets.carrier_tiers = set(
key.split('.carrier_')[1] for key in flat_techs.keys()
if '.carrier_' in key)
sets.costs = set(
k.split('costs.')[-1].split('.')[0] for k in flat_locations.keys()
if '.costs.' in k)
sets.locs = set(model_run.locations.keys())
sets.techs_non_transmission = set()
tech_groups = [
'demand', 'supply', 'supply_plus', 'conversion', 'conversion_plus',
'storage'
]
for tech_group in tech_groups:
sets['techs_{}'.format(tech_group)] = set(
k for k, v in model_run.techs.items()
if v.inheritance[-1] == tech_group)
sets.techs_non_transmission.update(sets['techs_{}'.format(tech_group)])
sets.techs_transmission_names = set(k for k, v in model_run.techs.items()
if v.inheritance[-1] == 'transmission')
# This builds the "tech:loc" expansion of transmission technologies
techs_transmission = set()
for loc_name, loc_config in model_run.locations.items():
for link_name, link_config in loc_config.get('links', {}).items():
for tech_name in link_config.techs:
techs_transmission.add('{}:{}'.format(tech_name, link_name))
sets.techs_transmission = techs_transmission
sets.techs = sets.techs_non_transmission | sets.techs_transmission_names
# this extracts location coordinate information
coordinates = set(
k.split('.')[-1] for k in flat_locations.keys()
if '.coordinates.' in k)
if coordinates:
sets.coordinates = coordinates
# `timesteps` set is built from the results of timeseries_data processing
sets.timesteps = list(model_run.timesteps.astype(str))
model_run.del_key('timesteps')
# `techlists` are strings with comma-separated techs used for grouping in
# some model-wide constraints
sets.techlists = set()
for k in model_run.model.get_key('group_share', {}).keys():
sets.techlists.add(k)
# `constraint_groups` are the group names per constraint that is defined
# at a group level
sets.group_constraints = set()
group_constraints = AttrDict({
name: data
for name, data in model_run['group_constraints'].items()
if data.get("exists", True)
})
if len(group_constraints.keys()) > 0:
sets.group_constraints.update(
i.split('.')[1] for i in group_constraints.as_dict_flat().keys()
if i.split('.')[1] not in ['techs', 'locs'])
for constr in sets.group_constraints:
sets['group_names_' + constr] = set(
k for k, v in group_constraints.items() if constr in v.keys())
return sets
def convert_model_dict(in_dict, conversion_dict, state, tech_groups=None):
out_dict = AttrDict()
# process techs
if 'techs' in in_dict:
for k, v in in_dict.techs.items():
# Remove now unsupported `unmet_demand` techs
if (v.get('parent',
'') in ['unmet_demand', 'unmet_demand_as_supply_tech']
or 'unmet_demand_' in k):
out_dict.set_key('__disabled.techs.{}'.format(k), v)
# We will want to enable ``ensure_feasibility`` to replace
# ``unmet_demand``
state['ensure_feasibility'] = True
continue
new_tech_config = convert_subdict(v,
conversion_dict['tech_config'])
if 'constraints_per_distance' in v:
# Convert loss to efficiency
if 'e_loss' in v.constraints_per_distance:
v.constraints_per_distance.e_loss = 1 - v.constraints_per_distance.e_loss
new_tech_config.update(
convert_subdict(
v.constraints_per_distance,
conversion_dict['tech_constraints_per_distance_config']
))
# Costs are a little more involved -- need to get each cost class
# as a subdict and merge the results back together
new_cost_dict = AttrDict()
if 'costs' in v:
for cost_class in v.costs:
new_cost_dict[cost_class] = convert_subdict(
v.costs[cost_class],
conversion_dict['tech_costs_config'])
if 'costs_per_distance' in v:
for cost_class in v.costs_per_distance:
# FIXME update not overwrite
per_distance_config = convert_subdict(
v.costs_per_distance[cost_class],
conversion_dict['tech_costs_per_distance_config'])
if cost_class in new_cost_dict:
new_cost_dict[cost_class].union(per_distance_config)
else:
new_cost_dict[cost_class] = per_distance_config
if 'depreciation' in v:
# 'depreciation.interest.{cost_class}' goes to 'costs.{cost_class}.interest_rate'
if 'interest' in v.depreciation:
for cost_class, interest in v.depreciation.interest.items(
):
new_cost_dict.set_key(
'{}.interest_rate'.format(cost_class), interest)
# 'depreciation.lifetime' goes to 'constraints.lifetime'
if 'lifetime' in v.depreciation:
new_tech_config.set_key('constraints.lifetime',
v.depreciation.lifetime)
if new_cost_dict:
new_tech_config['costs'] = new_cost_dict
# After conversion, remove legacy _per_distance top-level entries
try:
del new_tech_config['constraints_per_distance']
del new_tech_config['costs_per_distance']
except KeyError:
pass
# Assign converted techs to either tech_groups or techs
if tech_groups and k in tech_groups:
out_key = 'tech_groups.{}'.format(k)
else:
out_key = 'techs.{}'.format(k)
out_dict.set_key(out_key, new_tech_config)
del in_dict['techs']
# process locations
if 'locations' in in_dict:
new_locations_dict = AttrDict()
for k, v in in_dict.locations.items():
new_locations_dict[k] = convert_subdict(
v, conversion_dict['location_config'])
# convert per-location constraints now in [locname].techs[techname].constraints
for k, v in new_locations_dict.items():
if 'techs' in v:
for tech, tech_dict in v.techs.items():
new_locations_dict[k].techs[tech] = convert_subdict(
tech_dict, conversion_dict['tech_config'])
# Add techs that do not specify any overrides as keys
missing_techs = set(v.get_key('__disabled.techs', [])) - set(
v.get('techs', {}).keys())
for tech in missing_techs:
new_locations_dict[k].set_key('techs.{}'.format(tech), None)
# Remove now unsupported `unmet_demand` techs
for k, v in new_locations_dict.items():
for tech in list(v.techs.keys()):
parent = v.get_key('techs.{}.parent'.format(tech), '')
if (parent in ['unmet_demand', 'unmet_demand_as_supply_tech']
or 'unmet_demand_' in tech):
new_locations_dict[k].del_key('techs.{}'.format(tech))
if '__disabled.techs' in new_locations_dict[k]:
new_locations_dict[k].get_key(
'__disabled.techs').append(tech)
else:
new_locations_dict[k].set_key('__disabled.techs',
[tech])
out_dict['locations'] = new_locations_dict
del in_dict['locations']
# process links
if 'links' in in_dict:
new_links_dict = AttrDict()
for k, v in in_dict.links.items():
for tech, tech_dict in v.items():
new_links_dict.set_key(
'{}.techs.{}'.format(k, tech),
convert_subdict(tech_dict, conversion_dict['tech_config']))
out_dict['links'] = new_links_dict
del in_dict['links']
# process metadata
if 'metadata' in in_dict:
# manually transfer location coordinates
if 'location_coordinates' in in_dict.metadata:
for k, v in in_dict.metadata.location_coordinates.items():
if isinstance(v, list): # Assume it was lat/lon
new_coords = AttrDict({'lat': v[0], 'lon': v[1]})
else:
new_coords = v
in_dict.set_key('locations.{}.coordinates'.format(k),
new_coords)
del in_dict['metadata']
# Fix up any 'resource' keys that refer to 'file' only
for k in [i for i in out_dict.keys_nested() if i.endswith('.resource')]:
if out_dict.get_key(k) == 'file':
tech = k.split('techs.')[-1].split('.')[0]
out_dict.set_key(k, 'file={}_r.csv'.format(tech))
# process remaining top-level entries
out_dict.union(convert_subdict(in_dict, conversion_dict['model_config']))
return out_dict
def process_timeseries_data(config_model, model_run):
if config_model.model.timeseries_data is None:
timeseries_data = AttrDict()
else:
timeseries_data = config_model.model.timeseries_data
def _parser(x, dtformat):
return pd.to_datetime(x, format=dtformat, exact=False)
if 'timeseries_data_path' in config_model.model:
dtformat = config_model.model['timeseries_dateformat']
# Generate the set of all files we want to read from file
location_config = model_run.locations.as_dict_flat()
model_config = config_model.model.as_dict_flat()
get_filenames = lambda config: set([
v.split('=')[1].rsplit(':', 1)[0]
for v in config.values() if 'file=' in str(v)
])
constraint_filenames = get_filenames(location_config)
cluster_filenames = get_filenames(model_config)
datetime_min = []
datetime_max = []
for file in constraint_filenames | cluster_filenames:
file_path = os.path.join(config_model.model.timeseries_data_path, file)
# load the data, without parsing the dates, to catch errors in the data
df = pd.read_csv(file_path, index_col=0)
try:
df.apply(pd.to_numeric)
except ValueError as e:
raise exceptions.ModelError(
'Error in loading data from {}. Ensure all entries are '
'numeric. Full error: {}'.format(file, e)
)
# Now parse the dates, checking for errors specific to this
try:
df.index = _parser(df.index, dtformat)
except ValueError as e:
raise exceptions.ModelError(
'Error in parsing dates in timeseries data from {}, '
'using datetime format `{}`: {}'.format(file, dtformat, e)
)
timeseries_data[file] = df
datetime_min.append(df.index[0].date())
datetime_max.append(df.index[-1].date())
# Apply time subsetting, if supplied in model_run
subset_time_config = config_model.model.subset_time
if subset_time_config is not None:
# Test parsing dates first, to make sure they fit our required subset format
try:
subset_time = _parser(subset_time_config, '%Y-%m-%d %H:%M:%S')
except ValueError as e:
raise exceptions.ModelError(
'Timeseries subset must be in ISO format (anything up to the '
'detail of `%Y-%m-%d %H:%M:%S`.\n User time subset: {}\n '
'Error caused: {}'.format(subset_time_config, e)
)
if isinstance(subset_time_config, list) and len(subset_time_config) == 2:
time_slice = slice(subset_time_config[0], subset_time_config[1])
# Don't allow slicing outside the range of input data
if (subset_time[0].date() < max(datetime_min) or
subset_time[1].date() > min(datetime_max)):
raise exceptions.ModelError(
'subset time range {} is outside the input data time range '
'[{}, {}]'.format(subset_time_config,
max(datetime_min).strftime('%Y-%m-%d'),
min(datetime_max).strftime('%Y-%m-%d'))
)
elif isinstance(subset_time_config, list):
raise exceptions.ModelError(
'Invalid subset_time value: {}'.format(subset_time_config)
)
else:
time_slice = str(subset_time_config)
for k in timeseries_data.keys():
timeseries_data[k] = timeseries_data[k].loc[time_slice, :]
if timeseries_data[k].empty:
raise exceptions.ModelError(
'The time slice {} creates an empty timeseries array for {}'
.format(time_slice, k)
)
# Ensure all timeseries have the same index
indices = [
(file, df.index) for file, df in timeseries_data.items()
if file not in cluster_filenames
]
first_file, first_index = indices[0]
for file, idx in indices[1:]:
if not first_index.equals(idx):
raise exceptions.ModelError(
'Time series indices do not match '
'between {} and {}'.format(first_file, file)
)
return timeseries_data, first_index
def process_timeseries_data(config_model, model_run):
if config_model.model.timeseries_data is None:
timeseries_data = AttrDict()
else:
timeseries_data = config_model.model.timeseries_data
if 'timeseries_data_path' in config_model.model:
dtformat = config_model.model['timeseries_dateformat']
# Generate the set of all files we want to read from file
flattened_config = model_run.locations.as_dict_flat()
csv_files = set([
v.split('=')[1].rsplit(':', 1)[0]
for v in flattened_config.values() if 'file=' in str(v)
])
for file in csv_files:
file_path = os.path.join(config_model.model.timeseries_data_path,
file)
parser = lambda x: datetime.datetime.strptime(x, dtformat)
try:
df = pd.read_csv(file_path,
index_col=0,
parse_dates=True,
date_parser=parser)
except ValueError as e:
raise exceptions.ModelError(
"Incorrect datetime format used in {}, expecting "
"`{}`, got `{}` instead"
"".format(file, dtformat, e.args[0].split("'")[1]))
timeseries_data[file] = df
datetime_range = df.index
# Apply time subsetting, if supplied in model_run
subset_time_config = config_model.model.subset_time
if subset_time_config is not None:
if isinstance(subset_time_config, list):
if len(subset_time_config) == 2:
time_slice = slice(subset_time_config[0],
subset_time_config[1])
if pd.to_datetime(
subset_time_config[0]).date() < datetime_range[0].date(
) or pd.to_datetime(subset_time_config[1]).date(
) > datetime_range[-1].date():
raise exceptions.ModelError(
'subset time range {} is outside the input data time range [{}, {}]'
.format(subset_time_config,
datetime_range[0].strftime('%Y-%m-%d'),
datetime_range[-1].strftime('%Y-%m-%d')))
else:
raise exceptions.ModelError(
'Invalid subset_time value: {}'.format(subset_time_config))
else:
time_slice = str(subset_time_config)
for k in timeseries_data.keys():
timeseries_data[k] = timeseries_data[k].loc[time_slice, :]
if timeseries_data[k].empty:
raise exceptions.ModelError(
'The time slice {} creates an empty timeseries array for {}'
.format(time_slice, k))
# Ensure all timeseries have the same index
indices = [(file, df.index) for file, df in timeseries_data.items()]
first_file, first_index = indices[0]
for file, idx in indices[1:]:
if not first_index.equals(idx):
raise exceptions.ModelError('Time series indices do not match '
'between {} and {}'.format(
first_file, file))
return timeseries_data
def convert_model_dict(in_dict, conversion_dict):
out_dict = AttrDict()
# process techs
if 'techs' in in_dict:
for k, v in in_dict.techs.items():
new_tech_config = convert_subdict(v,
conversion_dict['tech_config'])
if 'constraints_per_distance' in v:
new_tech_config.update(
convert_subdict(
v.constraints_per_distance,
conversion_dict['tech_constraints_per_distance_config']
))
# Costs are a little more involved -- need to get each cost class
# as a subdict and merge the results back together
new_cost_dict = AttrDict()
if 'costs' in v:
for cost_class in v.costs:
new_cost_dict[cost_class] = convert_subdict(
v.costs[cost_class],
conversion_dict['tech_costs_config'])
if 'costs_per_distance' in v:
for cost_class in v.costs_per_distance:
# FIXME update not overwrite
per_distance_config = convert_subdict(
v.costs_per_distance[cost_class],
conversion_dict['tech_costs_config'])
if cost_class in new_cost_dict:
new_cost_dict[cost_class].union(per_distance_config)
else:
new_cost_dict[cost_class] = per_distance_config
if 'depreciation' in v:
# 'depreciation.interest.{cost_class}' goes to 'costs.{cost_class}.interest_rate'
if 'interest' in v.depreciation:
for cost_class, interest in v.depreciation.interest.items(
):
new_cost_dict.set_key(
'{}.interest_rate'.format(cost_class), interest)
# 'depreciation.lifetime' goes to 'constraints.lifetime'
if 'lifetime' in v.depreciation:
new_tech_config.set_key('constraints.lifetime',
v.depreciation.lifetime)
if new_cost_dict:
new_tech_config['costs'] = new_cost_dict
out_dict.set_key('techs.{}'.format(k), new_tech_config)
del in_dict['techs']
# process locations
if 'locations' in in_dict:
new_locations_dict = AttrDict()
for k, v in in_dict.locations.items():
new_locations_dict[k] = convert_subdict(
v, conversion_dict['location_config'])
# convert per-location constraints now in [locname].techs[techname].constraints
for k, v in new_locations_dict.items():
if 'techs' in v:
for tech, tech_dict in v.techs.items():
new_locations_dict[k].techs[tech] = convert_subdict(
tech_dict, conversion_dict['tech_config'])
# Add techs that do not specify any overrides as keys
missing_techs = set(v.get_key('__disabled.techs', [])) - set(
v.get('techs', {}).keys())
for tech in missing_techs:
new_locations_dict[k].set_key('techs.{}'.format(tech), None)
out_dict['locations'] = new_locations_dict
del in_dict['locations']
# process metadata
if 'metadata' in in_dict:
# manually transfer location coordinates
if 'location_coordinates' in in_dict.metadata:
for k, v in in_dict.metadata.location_coordinates.items():
if isinstance(v, list): # Assume it was lat/lon
new_coords = AttrDict({'lat': v[0], 'lon': v[1]})
else:
new_coords = v
in_dict.set_key('locations.{}.coordinates'.format(k),
new_coords)
del in_dict['metadata']
# process remaining top-level entries
out_dict.union(convert_subdict(in_dict, conversion_dict['model_config']))
return out_dict
def generate_model_run(
config, timeseries_dataframes, debug_comments, applied_overrides, scenario
):
"""
Returns a processed model_run configuration AttrDict and a debug
YAML object with comments attached, ready to write to disk.
Parameters
----------
config : AttrDict
timeseries_dataframes : dict
debug_comments : AttrDict
scenario : str
"""
model_run = AttrDict()
model_run["scenario"] = scenario
model_run["applied_overrides"] = ";".join(applied_overrides)
# 1) Initial checks on model configuration
warning_messages, errors = checks.check_initial(config)
exceptions.print_warnings_and_raise_errors(warnings=warning_messages, errors=errors)
# 2) Fully populate techs
# Raises ModelError if necessary
model_run["techs"], debug_techs, errors = process_techs(config)
debug_comments.set_key("model_run.techs", debug_techs)
exceptions.print_warnings_and_raise_errors(errors=errors)
# 3) Fully populate tech_groups
model_run["tech_groups"] = process_tech_groups(config, model_run["techs"])
# 4) Fully populate locations
(
model_run["locations"],
debug_locs,
warning_messages,
errors,
) = locations.process_locations(config, model_run["techs"])
debug_comments.set_key("model_run.locations", debug_locs)
exceptions.print_warnings_and_raise_errors(warnings=warning_messages, errors=errors)
# 5) Fully populate timeseries data
# Raises ModelErrors if there are problems with timeseries data at this stage
model_run["timeseries_data"], model_run["timesteps"] = process_timeseries_data(
config, model_run, timeseries_dataframes
)
# 6) Grab additional relevant bits from run and model config
model_run["run"] = config["run"]
model_run["model"] = config["model"]
model_run["group_constraints"] = config.get("group_constraints", {})
# 7) Initialize sets
all_sets = sets.generate_simple_sets(model_run)
all_sets.union(sets.generate_loc_tech_sets(model_run, all_sets))
all_sets = AttrDict({k: list(v) for k, v in all_sets.items()})
model_run["sets"] = all_sets
model_run["constraint_sets"] = constraint_sets.generate_constraint_sets(model_run)
# 8) Final sense-checking
final_check_comments, warning_messages, errors = checks.check_final(model_run)
debug_comments.union(final_check_comments)
exceptions.print_warnings_and_raise_errors(warnings=warning_messages, errors=errors)
# 9) Build a debug data dict with comments and the original configs
debug_data = AttrDict({"comments": debug_comments, "config_initial": config,})
return model_run, debug_data
def process_timeseries_data(config_model, model_run, timeseries_dataframes):
if config_model.model.timeseries_data is None:
timeseries_data = AttrDict()
else:
timeseries_data = config_model.model.timeseries_data
def _parser(x, dtformat):
return pd.to_datetime(x, format=dtformat, exact=False)
dtformat = config_model.model["timeseries_dateformat"]
# Generate set of all files and dataframes we want to load
location_config = model_run.locations.as_dict_flat()
model_config = config_model.model.as_dict_flat()
# Find names of csv files (file=) or dataframes (df=) called in config
def get_names(datatype, config):
return set(
[
v.split("=")[1].rsplit(":", 1)[0]
for v in config.values()
if str(datatype) + "=" in str(v)
]
)
constraint_filenames = get_names("file", location_config)
cluster_filenames = get_names("file", model_config)
constraint_dfnames = get_names("df", location_config)
cluster_dfnames = get_names("df", model_config)
# Check if timeseries_dataframes is in the correct format (dict of
# pandas DataFrames)
if timeseries_dataframes is not None:
check_timeseries_dataframes(timeseries_dataframes)
datetime_min = []
datetime_max = []
# Check there is at least one timeseries present
if (
len(
constraint_filenames
| cluster_filenames
| constraint_dfnames
| cluster_dfnames
)
== 0
):
raise exceptions.ModelError(
"There is no timeseries in the model. At least one timeseries is "
"necessary to run the model."
)
# Load each timeseries into timeseries data. tskey is either a filename
# (called by file=...) or a key in timeseries_dataframes (called by df=...)
for tskey in (
constraint_filenames | cluster_filenames | constraint_dfnames | cluster_dfnames
): # Filenames or dict keys
# If tskey is a CSV path, load the CSV, else load the dataframe
if tskey in constraint_filenames | cluster_filenames:
df = load_timeseries_from_file(config_model, tskey)
elif tskey in constraint_dfnames | cluster_dfnames:
df = load_timeseries_from_dataframe(timeseries_dataframes, tskey)
try:
df.apply(pd.to_numeric)
except ValueError as e:
raise exceptions.ModelError(
"Error in loading data from {}. Ensure all entries are "
"numeric. Full error: {}".format(tskey, e)
)
# Parse the dates, checking for errors specific to this
try:
df.index = _parser(df.index, dtformat)
except ValueError as e:
raise exceptions.ModelError(
"Error in parsing dates in timeseries data from {}, "
"using datetime format `{}`: {}".format(tskey, dtformat, e)
)
timeseries_data[tskey] = df
datetime_min.append(df.index[0].date())
datetime_max.append(df.index[-1].date())
# Apply time subsetting, if supplied in model_run
subset_time_config = config_model.model.subset_time
if subset_time_config is not None:
# Test parsing dates first, to make sure they fit our required subset format
try:
subset_time = _parser(subset_time_config, "%Y-%m-%d %H:%M:%S")
except ValueError as e:
raise exceptions.ModelError(
"Timeseries subset must be in ISO format (anything up to the "
"detail of `%Y-%m-%d %H:%M:%S`.\n User time subset: {}\n "
"Error caused: {}".format(subset_time_config, e)
)
if isinstance(subset_time_config, list) and len(subset_time_config) == 2:
time_slice = slice(subset_time_config[0], subset_time_config[1])
# Don't allow slicing outside the range of input data
if subset_time[0].date() < max(datetime_min) or subset_time[1].date() > min(
datetime_max
):
raise exceptions.ModelError(
"subset time range {} is outside the input data time range "
"[{}, {}]".format(
subset_time_config,
max(datetime_min).strftime("%Y-%m-%d"),
min(datetime_max).strftime("%Y-%m-%d"),
)
)
elif isinstance(subset_time_config, list):
raise exceptions.ModelError(
"Invalid subset_time value: {}".format(subset_time_config)
)
else:
time_slice = str(subset_time_config)
for k in timeseries_data.keys():
timeseries_data[k] = timeseries_data[k].loc[time_slice, :]
if timeseries_data[k].empty:
raise exceptions.ModelError(
"The time slice {} creates an empty timeseries array for {}".format(
time_slice, k
)
)
# Ensure all timeseries have the same index
indices = [
(tskey, df.index)
for tskey, df in timeseries_data.items()
if tskey not in cluster_filenames | cluster_dfnames
]
first_tskey, first_index = indices[0]
for tskey, idx in indices[1:]:
if not first_index.equals(idx):
raise exceptions.ModelError(
"Time series indices do not match "
"between {} and {}".format(first_tskey, tskey)
)
return timeseries_data, first_index
def process_locations(model_config, modelrun_techs):
"""
Process locations by taking an AttrDict that may include compact keys
such as ``1,2,3``, and returning an AttrDict with:
* exactly one key per location with all of its settings
* fully resolved installed technologies for each location
* fully expanded transmission links for each location
Parameters
----------
model_config : AttrDict
modelrun_techs : AttrDict
Returns
-------
locations : AttrDict
locations_comments : AttrDict
"""
techs_in = model_config.techs.copy()
tech_groups_in = model_config.tech_groups
locations_in = model_config.locations
links_in = model_config.get('links', AttrDict())
allowed_from_file = defaults['file_allowed']
warnings = []
errors = []
locations_comments = AttrDict()
##
# Expand compressed `loc1,loc2,loc3,loc4: ...` definitions
##
locations = AttrDict()
for key in locations_in:
if ('--' in key) or (',' in key):
key_locs = explode_locations(key)
for subkey in key_locs:
_set_loc_key(locations, subkey, locations_in[key])
else:
_set_loc_key(locations, key, locations_in[key])
##
# Kill any locations that the modeller does not want to exist
##
for loc in list(locations.keys()):
if not locations[loc].get('exists', True):
locations.del_key(loc)
##
# Process technologies
##
techs_to_delete = []
for tech_name in techs_in:
if not techs_in[tech_name].get('exists', True):
techs_to_delete.append(tech_name)
continue
# Get inheritance chain generated in process_techs()
inheritance_chain = modelrun_techs[tech_name].inheritance
# Get and save list of required_constraints from base technology
base_tech = inheritance_chain[-1]
rq = model_config.tech_groups[base_tech].required_constraints
# locations[loc_name].techs[tech_name].required_constraints = rq
techs_in[tech_name].required_constraints = rq
# Kill any techs that the modeller does not want to exist
for tech_name in techs_to_delete:
del techs_in[tech_name]
##
# Fully expand all installed technologies for the location,
# filling in any undefined parameters from defaults
##
location_techs_to_delete = []
for loc_name, loc in locations.items():
if 'techs' not in loc:
# Mark this as a transmission-only node if it has not allowed
# any technologies
locations[loc_name].transmission_node = True
locations_comments.set_key(
'{}.transmission_node'.format(loc_name),
'Automatically inserted: specifies that this node is '
'a transmission-only node.'
)
continue # No need to process any technologies at this node
for tech_name in loc.techs:
if tech_name in techs_to_delete:
# Techs that were removed need not be further considered
continue
if not isinstance(locations[loc_name].techs[tech_name], dict):
locations[loc_name].techs[tech_name] = AttrDict()
# Starting at top of the inheritance chain, for each level,
# check if the level has location-specific group settings
# and keep merging together the settings, overwriting as we
# go along.
tech_settings = AttrDict()
for parent in reversed(modelrun_techs[tech_name].inheritance):
# Does the parent group have model-wide settings?
tech_settings.union(tech_groups_in[parent], allow_override=True)
# Does the parent group have location-specific settings?
if ('tech_groups' in locations[loc_name] and
parent in locations[loc_name].tech_groups):
tech_settings.union(
locations[loc_name].tech_groups[parent],
allow_override=True)
# Now overwrite with the tech's own model-wide
# and location-specific settings
tech_settings.union(techs_in[tech_name], allow_override=True)
if tech_name in locations[loc_name].techs:
tech_settings.union(
locations[loc_name].techs[tech_name],
allow_override=True)
tech_settings = cleanup_undesired_keys(tech_settings)
# Resolve columns in filename if necessary
file_configs = [
i for i in tech_settings.keys_nested()
if (isinstance(tech_settings.get_key(i), str) and
'file=' in tech_settings.get_key(i))
]
for config_key in file_configs:
if config_key.split('.')[-1] not in allowed_from_file:
# Allow any custom settings that end with _time_varying
# FIXME: add this to docs
if config_key.endswith('_time_varying'):
warn('Using custom constraint '
'{} with time-varying data.'.format(config_key))
else:
raise ModelError('`file=` not allowed in {}'.format(config_key))
config_value = tech_settings.get_key(config_key, '')
if ':' not in config_value:
config_value = '{}:{}'.format(config_value, loc_name)
tech_settings.set_key(config_key, config_value)
tech_settings = compute_depreciation_rates(tech_name, tech_settings, warnings, errors)
# Now merge the tech settings into the location-specific
# tech dict -- but if a tech specifies ``exists: false``,
# we kill it at this location
if not tech_settings.get('exists', True):
location_techs_to_delete.append('{}.techs.{}'.format(loc_name, tech_name))
else:
locations[loc_name].techs[tech_name].union(
tech_settings, allow_override=True
)
for k in location_techs_to_delete:
locations.del_key(k)
# Generate all transmission links
processed_links = AttrDict()
for link in links_in:
loc_from, loc_to = link.split(',')
# Skip this link entirely if it has been told not to exist
if not links_in[link].get('exists', True):
continue
# Also skip this link - and warn about it - if it links to a
# now-inexistant (because removed) location
if (loc_from not in locations.keys() or loc_to not in locations.keys()):
warnings.append(
'Not building the link {},{} because one or both of its '
'locations have been removed from the model by setting '
'``exists: false``'.format(loc_from, loc_to)
)
continue
processed_transmission_techs = AttrDict()
for tech_name in links_in[link].techs:
# Skip techs that have been told not to exist
# for this particular link
if not links_in[link].get_key('techs.{}.exists'.format(tech_name), True):
continue
if tech_name not in processed_transmission_techs:
tech_settings = AttrDict()
# Combine model-wide settings from all parent groups
for parent in reversed(modelrun_techs[tech_name].inheritance):
tech_settings.union(
tech_groups_in[parent],
allow_override=True
)
# Now overwrite with the tech's own model-wide settings
tech_settings.union(
techs_in[tech_name],
allow_override=True
)
# Add link-specific constraint overrides
if links_in[link].techs[tech_name]:
tech_settings.union(
links_in[link].techs[tech_name],
allow_override=True
)
tech_settings = cleanup_undesired_keys(tech_settings)
tech_settings = process_per_distance_constraints(tech_name, tech_settings, locations, locations_comments, loc_from, loc_to)
tech_settings = compute_depreciation_rates(tech_name, tech_settings, warnings, errors)
processed_transmission_techs[tech_name] = tech_settings
else:
tech_settings = processed_transmission_techs[tech_name]
processed_links.set_key(
'{}.links.{}.techs.{}'.format(loc_from, loc_to, tech_name),
tech_settings.copy()
)
processed_links.set_key(
'{}.links.{}.techs.{}'.format(loc_to, loc_from, tech_name),
tech_settings.copy()
)
# If this is a one-way link, we set the constraints for energy_prod
# and energy_con accordingly on both parts of the link
if tech_settings.get_key('constraints.one_way', False):
processed_links.set_key(
'{}.links.{}.techs.{}.constraints.energy_prod'.format(loc_from, loc_to, tech_name),
False)
processed_links.set_key(
'{}.links.{}.techs.{}.constraints.energy_con'.format(loc_to, loc_from, tech_name),
False)
locations.union(processed_links, allow_override=True)
return locations, locations_comments, list(set(warnings)), list(set(errors))
def generate_model_run(config, debug_comments, applied_overrides, scenario):
"""
Returns a processed model_run configuration AttrDict and a debug
YAML object with comments attached, ready to write to disk.
Parameters
----------
config : AttrDict
debug_comments : AttrDict
"""
model_run = AttrDict()
model_run['scenario'] = scenario
model_run['applied_overrides'] = ';'.join(applied_overrides)
# 1) Initial checks on model configuration
warnings, errors = checks.check_initial(config)
exceptions.print_warnings_and_raise_errors(warnings=warnings, errors=errors)
# 2) Fully populate techs
# Raises ModelError if necessary
model_run['techs'], debug_techs, errors = process_techs(config)
debug_comments.set_key('model_run.techs', debug_techs)
exceptions.print_warnings_and_raise_errors(errors=errors)
# 3) Fully populate tech_groups
model_run['tech_groups'] = process_tech_groups(config, model_run['techs'])
# 4) Fully populate locations
model_run['locations'], debug_locs, warnings, errors = locations.process_locations(
config, model_run['techs']
)
debug_comments.set_key('model_run.locations', debug_locs)
exceptions.print_warnings_and_raise_errors(warnings=warnings, errors=errors)
# 5) Fully populate timeseries data
# Raises ModelErrors if there are problems with timeseries data at this stage
model_run['timeseries_data'], model_run['timesteps'] = (
process_timeseries_data(config, model_run)
)
# 6) Grab additional relevant bits from run and model config
model_run['run'] = config['run']
model_run['model'] = config['model']
# 7) Initialize sets
all_sets = sets.generate_simple_sets(model_run)
all_sets.union(sets.generate_loc_tech_sets(model_run, all_sets))
all_sets = AttrDict({k: list(v) for k, v in all_sets.items()})
model_run['sets'] = all_sets
model_run['constraint_sets'] = constraint_sets.generate_constraint_sets(model_run)
# 8) Final sense-checking
final_check_comments, warnings, errors = checks.check_final(model_run)
debug_comments.union(final_check_comments)
exceptions.print_warnings_and_raise_errors(warnings=warnings, errors=errors)
# 9) Build a debug data dict with comments and the original configs
debug_data = AttrDict({
'comments': debug_comments,
'config_initial': config,
})
return model_run, debug_data
def process_timeseries_data(config_model, model_run):
if config_model.model.timeseries_data is None:
timeseries_data = AttrDict()
else:
timeseries_data = config_model.model.timeseries_data
def _parser(x, dtformat):
return pd.to_datetime(x, format=dtformat, exact=False)
if 'timeseries_data_path' in config_model.model:
dtformat = config_model.model['timeseries_dateformat']
# Generate the set of all files we want to read from file
location_config = model_run.locations.as_dict_flat()
model_config = config_model.model.as_dict_flat()
get_filenames = lambda config: set([
v.split('=')[1].rsplit(':', 1)[0]
for v in config.values() if 'file=' in str(v)
])
constraint_filenames = get_filenames(location_config)
cluster_filenames = get_filenames(model_config)
_assert_timeseries_available(constraint_filenames | cluster_filenames)
datetime_min = []
datetime_max = []
for file in constraint_filenames | cluster_filenames:
file_path = os.path.join(config_model.model.timeseries_data_path, file)
# load the data, without parsing the dates, to catch errors in the data
df = pd.read_csv(file_path, index_col=0)
try:
df.apply(pd.to_numeric)
except ValueError as e:
raise exceptions.ModelError(
'Error in loading data from {}. Ensure all entries are '
'numeric. Full error: {}'.format(file, e)
)
# Now parse the dates, checking for errors specific to this
try:
df.index = _parser(df.index, dtformat)
except ValueError as e:
raise exceptions.ModelError(
'Error in parsing dates in timeseries data from {}, '
'using datetime format `{}`: {}'.format(file, dtformat, e)
)
timeseries_data[file] = df
datetime_min.append(df.index[0].date())
datetime_max.append(df.index[-1].date())
# Apply time subsetting, if supplied in model_run
subset_time_config = config_model.model.subset_time
if subset_time_config is not None:
# Test parsing dates first, to make sure they fit our required subset format
try:
subset_time = _parser(subset_time_config, '%Y-%m-%d %H:%M:%S')
except ValueError as e:
raise exceptions.ModelError(
'Timeseries subset must be in ISO format (anything up to the '
'detail of `%Y-%m-%d %H:%M:%S`.\n User time subset: {}\n '
'Error caused: {}'.format(subset_time_config, e)
)
if isinstance(subset_time_config, list) and len(subset_time_config) == 2:
time_slice = slice(subset_time_config[0], subset_time_config[1])
# Don't allow slicing outside the range of input data
if (subset_time[0].date() < max(datetime_min) or
subset_time[1].date() > min(datetime_max)):
raise exceptions.ModelError(
'subset time range {} is outside the input data time range '
'[{}, {}]'.format(subset_time_config,
max(datetime_min).strftime('%Y-%m-%d'),
min(datetime_max).strftime('%Y-%m-%d'))
)
elif isinstance(subset_time_config, list):
raise exceptions.ModelError(
'Invalid subset_time value: {}'.format(subset_time_config)
)
else:
time_slice = str(subset_time_config)
for k in timeseries_data.keys():
timeseries_data[k] = timeseries_data[k].loc[time_slice, :]
if timeseries_data[k].empty:
raise exceptions.ModelError(
'The time slice {} creates an empty timeseries array for {}'
.format(time_slice, k)
)
# Ensure all timeseries have the same index
indices = [
(file, df.index) for file, df in timeseries_data.items()
if file not in cluster_filenames
]
first_file, first_index = indices[0]
for file, idx in indices[1:]:
if not first_index.equals(idx):
raise exceptions.ModelError(
'Time series indices do not match '
'between {} and {}'.format(first_file, file)
)
return timeseries_data, first_index
