def postprocess_model_results(results, model_data, timings):
"""
Adds additional post-processed result variables to
the given model results in-place. Model must have solved successfully.
Returns None.
"""
log_time(timings, 'post_process_start', comment='Postprocessing: started')
results['capacity_factor'] = capacity_factor(results, model_data)
results['systemwide_capacity_factor'] = systemwide_capacity_factor(
results, model_data)
results['systemwide_levelised_cost'] = systemwide_levelised_cost(results)
results['total_levelised_cost'] = systemwide_levelised_cost(results,
total=True)
results = clean_results(results,
model_data.attrs.get('run.zero_threshold', 0),
timings)
log_time(timings,
'post_process_end',
time_since_start=True,
comment='Postprocessing: ended')
return results
def _init_from_model_data(self, model_data):
self._model_run = None
self._debug_data = None
self._model_data = model_data
self.inputs = self._model_data.filter_by_attrs(is_result=0)
results = self._model_data.filter_by_attrs(is_result=1)
if len(results.data_vars) > 0:
self.results = results
log_time(self._timings, 'model_data_loaded', time_since_start=True)
def rerun_pyomo_model(model_data, backend_model):
"""
Rerun the Pyomo backend, perhaps after updating a parameter value,
(de)activating a constraint/objective or updating run options in the model
model_data object (e.g. `run.solver`).
Returns
-------
run_data : xarray.Dataset
Raw data from this rerun, including both inputs and results.
to filter inputs/results, use `run_data.filter_by_attrs(is_result=...)`
with 0 for inputs and 1 for results.
"""
if model_data.attrs['run.mode'] != 'plan':
raise exceptions.ModelError(
'Cannot rerun the backend in {} run mode. Only `plan` mode is '
'possible.'.format(model_data.attrs['run.mode']))
timings = {}
log_time(timings, 'model_creation')
results, backend_model = backend_run.run_plan(model_data,
timings,
run_pyomo,
build_only=False,
backend_rerun=backend_model)
for k, v in timings.items():
results.attrs['timings.' + k] = v
exceptions.ModelWarning(
'model.results will only be updated on running the model from '
'`model.run()`. We provide results of this rerun as a standalone xarray '
'Dataset')
results.attrs.update(model_data.attrs)
for key, var in results.data_vars.items():
var.attrs['is_result'] = 1
inputs = access_pyomo_model_inputs(backend_model)
for key, var in inputs.data_vars.items():
var.attrs['is_result'] = 0
results.update(inputs)
run_data = results
return run_data
def postprocess_model_results(results, model_data, timings):
"""
Adds additional post-processed result variables to
the given model results in-place. Model must have solved successfully.
Parameters
----------
results : xarray Dataset
Output from the solver backend
model_data : xarray Dataset
Calliope model data, stored as calliope.Model()._model_data
timings : dict
Calliope timing dictionary, stored as calliope.Model()._timings
Returns
-------
results : xarray Dataset
Input results Dataset, with additional DataArray variables and removed
all instances of unreasonably low numbers (set by zero_threshold)
"""
log_time(timings, 'post_process_start', comment='Postprocessing: started')
results['capacity_factor'] = capacity_factor(results, model_data)
results['systemwide_capacity_factor'] = systemwide_capacity_factor(
results, model_data)
results['systemwide_levelised_cost'] = systemwide_levelised_cost(
results, model_data)
results['total_levelised_cost'] = systemwide_levelised_cost(results,
model_data,
total=True)
results = clean_results(results,
model_data.attrs.get('run.zero_threshold', 0),
timings)
log_time(timings,
'post_process_end',
time_since_start=True,
comment='Postprocessing: ended')
return results
def _init_from_model_run(self, model_run, debug_data):
self._model_run = model_run
self._debug_data = debug_data
log_time(self._timings, 'model_run_creation')
self._model_data_original = build_model_data(model_run)
log_time(self._timings, 'model_data_original_creation')
random_seed = self._model_run.get_key('model.random_seed', None)
if random_seed:
np.random.seed(seed=random_seed)
# After setting the random seed, time clustering can take place
time_config = model_run.model.get('time', None)
if not time_config:
_model_data = self._model_data_original
else:
_model_data = apply_time_clustering(self._model_data_original,
model_run)
self._model_data = final_timedimension_processing(_model_data)
log_time(self._timings, 'model_data_creation', time_since_start=True)
for var in self._model_data.data_vars:
self._model_data[var].attrs['is_result'] = 0
self.inputs = self._model_data.filter_by_attrs(is_result=0)
def clean_results(results, zero_threshold, timings):
"""
Remove unreasonably small values (solver output can lead to floating point
errors) and remove unmet_demand if it was never used (i.e. sum = zero)
zero_threshold is a value set in model configuration. If not set, defaults
to zero (i.e. doesn't do anything). Reasonable value = 1e-12
"""
threshold_applied = []
for k, v in results.data_vars.items():
# If there are any values in the data variable which fall below the
# threshold, note the data variable name and set those values to zero
if v.where(abs(v) < zero_threshold, drop=True).sum():
threshold_applied.append(k)
with np.errstate(invalid='ignore'):
v.values[abs(v.values) < zero_threshold] = 0
v.loc[{}] = v.values
if threshold_applied:
comment = 'All values < {} set to 0 in {}'.format(
zero_threshold, ', '.join(threshold_applied))
else:
comment = 'zero threshold of {} not required'.format(zero_threshold)
log_time(timings,
'threshold_applied',
comment='Postprocessing: ' + comment)
if 'unmet_demand' in results.data_vars.keys(
) and not results.unmet_demand.sum():
log_time(
timings,
'delete_unmet_demand',
comment=
'Postprocessing: Model was feasible, deleting unmet_demand variable'
)
results = results.drop('unmet_demand')
return results
def __init__(self, config, model_data=None, *args, **kwargs):
"""
Returns a new Model from either the path to a YAML model
configuration file or a dict fully specifying the model.
Parameters
----------
config : str or dict or AttrDict
If str, must be the path to a model configuration file.
If dict or AttrDict, must fully specify the model.
model_data : Dataset, optional
Create a Model instance from a fully built model_data Dataset.
This is only used if `config` is explicitly set to None
and is primarily used to re-create a Model instance from
a model previously saved to a NetCDF file.
"""
self._timings = {}
log_time(self._timings, 'model_creation')
if isinstance(config, str):
model_run, debug_data = model_run_from_yaml(
config, *args, **kwargs)
self._init_from_model_run(model_run, debug_data)
elif isinstance(config, dict):
model_run, debug_data = model_run_from_dict(
config, *args, **kwargs)
self._init_from_model_run(model_run, debug_data)
elif model_data is not None and config is None:
self._init_from_model_data(model_data)
else:
# expected input is a string pointing to a YAML file of the run
# configuration or a dict/AttrDict in which the run and model
# configurations are defined
raise ValueError(
'Input configuration must either be a string or a dictionary.')
self.plot = plotting.ModelPlotMethods(self)
def run_plan(model_data, timings, backend, build_only, backend_rerun=False):
log_time(timings, 'run_start', comment='Backend: starting model run')
if not backend_rerun:
backend_model = backend.generate_model(model_data)
log_time(timings,
'run_backend_model_generated',
time_since_start=True,
comment='Backend: model generated')
else:
backend_model = backend_rerun
solver = model_data.attrs['run.solver']
solver_io = model_data.attrs.get('run.solver_io', None)
solver_options = model_data.attrs.get('run.solver_options', None)
save_logs = model_data.attrs.get('run.save_logs', None)
if build_only:
results = xr.Dataset()
else:
log_time(timings,
'run_solver_start',
comment='Backend: sending model to solver')
results = backend.solve_model(backend_model,
solver=solver,
solver_io=solver_io,
solver_options=solver_options,
save_logs=save_logs)
log_time(timings,
'run_solver_exit',
time_since_start=True,
comment='Backend: solver finished running')
termination = backend.load_results(backend_model, results)
log_time(timings,
'run_results_loaded',
comment='Backend: loaded results')
results = backend.get_result_array(backend_model, model_data)
results.attrs['termination_condition'] = termination
log_time(timings,
'run_solution_returned',
time_since_start=True,
comment='Backend: generated solution array')
return results, backend_model
def run_operate(model_data, timings, backend, build_only):
"""
For use when mode is 'operate', to allow the model to be built, edited, and
iteratively run within Pyomo.
"""
log_time(timings,
'run_start',
comment='Backend: starting model run in operational mode')
defaults = ruamel.yaml.load(model_data.attrs['defaults'],
Loader=ruamel.yaml.Loader)
operate_params = ['purchased'] + [
i.replace('_max', '') for i in defaults if i[-4:] == '_max'
]
# Capacity results (from plan mode) can be used as the input to operate mode
if (any(model_data.filter_by_attrs(is_result=1).data_vars)
and model_data.attrs.get('run.operation.use_cap_results', False)):
# Anything with is_result = 1 will be ignored in the Pyomo model
for varname, varvals in model_data.data_vars.items():
if varname in operate_params:
varvals.attrs['is_result'] = 1
varvals.attrs['operate_param'] = 1
else:
cap_max = xr.merge([
v.rename(k.replace('_max', ''))
for k, v in model_data.data_vars.items() if '_max' in k
])
cap_equals = xr.merge([
v.rename(k.replace('_equals', ''))
for k, v in model_data.data_vars.items() if '_equals' in k
])
caps = cap_max.update(cap_equals)
for cap in caps.data_vars.values():
cap.attrs['is_result'] = 1
cap.attrs['operate_param'] = 1
model_data.update(caps)
# Storage initial is carried over between iterations, so must be defined along with storage
if ('loc_techs_store' in model_data.dims.keys()
and 'storage_initial' not in model_data.data_vars.keys()):
model_data['storage_initial'] = (xr.DataArray(
[0 for loc_tech in model_data.loc_techs_store.values],
dims='loc_techs_store'))
model_data['storage_initial'].attrs['is_result'] = 0
exceptions.ModelWarning(
'Initial stored energy not defined, set to zero for all '
'loc::techs in loc_techs_store, for use in iterative optimisation')
# Operated units is carried over between iterations, so must be defined in a milp model
if ('loc_techs_milp' in model_data.dims.keys()
and 'operated_units' not in model_data.data_vars.keys()):
model_data['operated_units'] = (xr.DataArray(
[0 for loc_tech in model_data.loc_techs_milp.values],
dims='loc_techs_milp'))
model_data['operated_units'].attrs['is_result'] = 1
model_data['operated_units'].attrs['operate_param'] = 1
exceptions.ModelWarning(
'daily operated units not defined, set to zero for all '
'loc::techs in loc_techs_milp, for use in iterative optimisation')
comments, warnings, errors = checks.check_operate_params(model_data)
exceptions.print_warnings_and_raise_errors(warnings=warnings,
errors=errors)
# Initialize our variables
solver = model_data.attrs['run.solver']
solver_io = model_data.attrs.get('run.solver_io', None)
solver_options = model_data.attrs.get('run.solver_options', None)
save_logs = model_data.attrs.get('run.save_logs', None)
window = model_data.attrs['run.operation.window']
horizon = model_data.attrs['run.operation.horizon']
window_to_horizon = horizon - window
# get the cumulative sum of timestep resolution, to find where we hit our window and horizon
timestep_cumsum = model_data.timestep_resolution.cumsum(
'timesteps').to_pandas()
# get the timesteps at which we start and end our windows
window_ends = timestep_cumsum.where((timestep_cumsum % window == 0) | (
timestep_cumsum == timestep_cumsum[-1]))
window_starts = timestep_cumsum.where((~np.isnan(window_ends.shift(1))) | (
timestep_cumsum == timestep_cumsum[0])).dropna()
window_ends = window_ends.dropna()
horizon_ends = timestep_cumsum[timestep_cumsum.isin(window_ends.values +
window_to_horizon)]
if not any(window_starts):
raise exceptions.ModelError(
'Not enough timesteps or incorrect timestep resolution to run in '
'operational mode with an optimisation window of {}'.format(
window))
# We will only update timseries parameters
timeseries_data_vars = [
k for k, v in model_data.data_vars.items()
if 'timesteps' in v.dims and v.attrs['is_result'] == 0
]
# Loop through each window, solve over the horizon length, and add result to
# result_array we only go as far as the end of the last horizon, which may
# clip the last bit of data
result_array = []
# track whether each iteration finds an optimal solution or not
terminations = []
if build_only:
iterations = [0]
else:
iterations = range(len(window_starts))
for i in iterations:
start_timestep = window_starts.index[i]
# Build full model in first instance
if i == 0:
warmstart = False
end_timestep = horizon_ends.index[i]
timesteps = slice(start_timestep, end_timestep)
window_model_data = model_data.loc[dict(timesteps=timesteps)]
log_time(timings,
'model_gen_1',
comment='Backend: generating initial model')
backend_model = backend.generate_model(window_model_data)
# Build the full model in the last instance(s),
# where number of timesteps is less than the horizon length
elif i > len(horizon_ends) - 1:
warmstart = False
end_timestep = window_ends.index[i]
timesteps = slice(start_timestep, end_timestep)
window_model_data = model_data.loc[dict(timesteps=timesteps)]
log_time(
timings,
'model_gen_{}'.format(i + 1),
comment=(
'Backend: ite()ration {}: generating new model for '
'end of timeseries, with horizon = {} timesteps'.format(
i + 1, window_ends[i] - window_starts[i])))
backend_model = backend.generate_model(window_model_data)
# Update relevent Pyomo Params in intermediate instances
else:
warmstart = True
end_timestep = horizon_ends.index[i]
timesteps = slice(start_timestep, end_timestep)
window_model_data = model_data.loc[dict(timesteps=timesteps)]
log_time(
timings,
'model_gen_{}'.format(i + 1),
comment='Backend: iteration {}: updating model parameters'.
format(i + 1))
# Pyomo model sees the same timestamps each time, we just change the
# values associated with those timestamps
for var in timeseries_data_vars:
# New values
var_series = window_model_data[var].to_series().dropna(
).replace('inf', np.inf)
# Same timestamps
var_series.index = backend_model.__calliope_model_data__[
'data'][var].keys()
var_dict = var_series.to_dict()
# Update pyomo Param with new dictionary
for k, v in getattr(backend_model, var).items():
if k in var_dict:
v.set_value(var_dict[k])
if not build_only:
log_time(timings,
'model_run_{}'.format(i + 1),
time_since_start=True,
comment='Backend: iteration {}: sending model to solver'.
format(i + 1))
# After iteration 1, warmstart = True, which should speed up the process
# Note: Warmstart isn't possible with GLPK (dealt with later on)
_results = backend.solve_model(backend_model,
solver=solver,
solver_io=solver_io,
solver_options=solver_options,
save_logs=save_logs,
warmstart=warmstart)
log_time(timings,
'run_solver_exit_{}'.format(i + 1),
time_since_start=True,
comment='Backend: iteration {}: solver finished running'.
format(i + 1))
# xarray dataset is built for each iteration
_termination = backend.load_results(backend_model, _results)
terminations.append(_termination)
_results = backend.get_result_array(backend_model, model_data)
# We give back the actual timesteps for this iteration and take a slice
# equal to the window length
_results['timesteps'] = window_model_data.timesteps.copy()
# We always save the window data. Until the last window(s) this will crop
# the window_to_horizon timesteps. In the last window(s), optimistion will
# only be occurring over a window length anyway
_results = _results.loc[dict(
timesteps=slice(None, window_ends.index[i]))]
result_array.append(_results)
# Set up initial storage for the next iteration
if 'loc_techs_store' in model_data.dims.keys():
storage_initial = _results.storage.loc[dict(
timesteps=window_ends.index[i])]
model_data['storage_initial'].loc[{}] = storage_initial.values
for k, v in backend_model.storage_initial.items():
v.set_value(
storage_initial.to_series().dropna().to_dict()[k])
# Set up total operated units for the next iteration
if 'loc_techs_milp' in model_data.dims.keys():
operated_units = _results.operating_units.sum(
'timesteps').astype(np.int)
model_data['operated_units'].loc[{}] += operated_units.values
for k, v in backend_model.operated_units.items():
v.set_value(
operated_units.to_series().dropna().to_dict()[k])
log_time(timings,
'run_solver_exit_{}'.format(i + 1),
time_since_start=True,
comment='Backend: iteration {}: generated solution array'.
format(i + 1))
if build_only:
results = xr.Dataset()
else:
# Concatenate results over the timestep dimension to get a single
# xarray Dataset of interest
results = xr.concat(result_array, dim='timesteps')
if all(i == 'optimal' for i in terminations):
results.attrs['termination_condition'] = 'optimal'
else:
results.attrs['termination_condition'] = ','.join(terminations)
log_time(timings,
'run_solution_returned',
time_since_start=True,
comment='Backend: generated full solution array')
return results, backend_model