def _init_from_model_data(self, model_data):
if "_model_run" in model_data.attrs:
self._model_run = AttrDict.from_yaml_string(
model_data.attrs["_model_run"])
del model_data.attrs["_model_run"]
if "_debug_data" in model_data.attrs:
self._debug_data = AttrDict.from_yaml_string(
model_data.attrs["_debug_data"])
del model_data.attrs["_debug_data"]
self._model_data = model_data
self.inputs = self._model_data.filter_by_attrs(is_result=0)
self.model_config = UpdateObserverDict(
initial_yaml_string=model_data.attrs.get("model_config", "{}"),
name="model_config",
observer=self._model_data,
)
self.run_config = UpdateObserverDict(
initial_yaml_string=model_data.attrs.get("run_config", "{}"),
name="run_config",
observer=self._model_data,
)
results = self._model_data.filter_by_attrs(is_result=1)
if len(results.data_vars) > 0:
self.results = results
log_time(
logger,
self._timings,
"model_data_loaded",
comment="Model: loaded model_data",
)
def _init_from_model_data(self, model_data):
if '_model_run' in model_data.attrs:
self._model_run = AttrDict.from_yaml_string(
model_data.attrs['_model_run'])
del model_data.attrs['_model_run']
if '_debug_data' in model_data.attrs:
self._debug_data = AttrDict.from_yaml_string(
model_data.attrs['_debug_data'])
del model_data.attrs['_debug_data']
self._model_data = model_data
self.inputs = self._model_data.filter_by_attrs(is_result=0)
self.model_config = UpdateObserverDict(
initial_yaml_string=model_data.attrs.get('model_config', '{}'),
name='model_config',
observer=self._model_data)
self.run_config = UpdateObserverDict(
initial_yaml_string=model_data.attrs.get('run_config', '{}'),
name='run_config',
observer=self._model_data)
results = self._model_data.filter_by_attrs(is_result=1)
if len(results.data_vars) > 0:
self.results = results
log_time(logger,
self._timings,
'model_data_loaded',
comment='Model: loaded model_data')
def _add_model_data_methods(self):
self.inputs = self._model_data.filter_by_attrs(is_result=0)
self.results = self._model_data.filter_by_attrs(is_result=1)
self.model_config = UpdateObserverDict(
initial_yaml_string=self._model_data.attrs.get("model_config", "{}"),
name="model_config",
observer=self._model_data,
)
self.run_config = UpdateObserverDict(
initial_yaml_string=self._model_data.attrs.get("run_config", "{}"),
name="run_config",
observer=self._model_data,
)
self.subsets = UpdateObserverDict(
initial_yaml_string=self._model_data.attrs.get("subsets", "{}"),
name="subsets",
observer=self._model_data,
flat=True,
)
results = self._model_data.filter_by_attrs(is_result=1)
if len(results.data_vars) > 0:
self.results = results
log_time(
logger,
self._timings,
"model_data_loaded",
comment="Model: loaded model_data",
)
def postprocess_model_results(results, model_data, timings):
"""
Adds additional post-processed result variables to
the given model results in-place. Model must have solved successfully.
Parameters
----------
results : xarray Dataset
Output from the solver backend
model_data : xarray Dataset
Calliope model data, stored as calliope.Model()._model_data
timings : dict
Calliope timing dictionary, stored as calliope.Model()._timings
Returns
-------
results : xarray Dataset
Input results Dataset, with additional DataArray variables and removed
all instances of unreasonably low numbers (set by zero_threshold)
"""
log_time(logger,
timings,
"post_process_start",
comment="Postprocessing: started")
if model_data.attrs['scale']:
scale(model_data, lambda x: 1 / x)
results['scale'] = model_data['scale']
scale(results, lambda x: 1 / x)
run_config = AttrDict.from_yaml_string(model_data.attrs["run_config"])
results["capacity_factor"] = capacity_factor(results, model_data)
results["systemwide_capacity_factor"] = systemwide_capacity_factor(
results, model_data)
results["systemwide_levelised_cost"] = systemwide_levelised_cost(
results, model_data)
results["total_levelised_cost"] = systemwide_levelised_cost(results,
model_data,
total=True)
results = clean_results(results, run_config.get("zero_threshold", 0),
timings)
log_time(
logger,
timings,
"post_process_end",
time_since_run_start=True,
comment="Postprocessing: ended",
)
if "run_solution_returned" in timings.keys():
results.attrs["solution_time"] = (
timings["run_solution_returned"] -
timings["run_start"]).total_seconds()
results.attrs["time_finished"] = timings[
"run_solution_returned"].strftime("%Y-%m-%d %H:%M:%S")
return results
def postprocess_model_results(results, model_data, timings):
"""
Adds additional post-processed result variables to
the given model results in-place. Model must have solved successfully.
Parameters
----------
results : xarray Dataset
Output from the solver backend
model_data : xarray Dataset
Calliope model data, stored as calliope.Model()._model_data
timings : dict
Calliope timing dictionary, stored as calliope.Model()._timings
Returns
-------
results : xarray Dataset
Input results Dataset, with additional DataArray variables and removed
all instances of unreasonably low numbers (set by zero_threshold)
"""
log_time(logger,
timings,
'post_process_start',
comment='Postprocessing: started')
run_config = AttrDict.from_yaml_string(model_data.attrs['run_config'])
results['capacity_factor'] = capacity_factor(results, model_data)
results['systemwide_capacity_factor'] = systemwide_capacity_factor(
results, model_data)
results['systemwide_levelised_cost'] = systemwide_levelised_cost(
results, model_data)
results['total_levelised_cost'] = systemwide_levelised_cost(results,
model_data,
total=True)
results = clean_results(results, run_config.get('zero_threshold', 0),
timings)
log_time(logger,
timings,
'post_process_end',
time_since_run_start=True,
comment='Postprocessing: ended')
if 'run_solution_returned' in timings.keys():
results.attrs['solution_time'] = (
timings['run_solution_returned'] -
timings['run_start']).total_seconds()
results.attrs['time_finished'] = (
timings['run_solution_returned'].strftime('%Y-%m-%d %H:%M:%S'))
return results
def _init_from_model_data(self, model_data):
self._model_run = None
self._debug_data = None
self._model_data = model_data
self.inputs = self._model_data.filter_by_attrs(is_result=0)
results = self._model_data.filter_by_attrs(is_result=1)
if len(results.data_vars) > 0:
self.results = results
log_time(self._timings,
'model_data_loaded',
comment='Model: loaded model_data',
time_since_start=True)
def _init_from_model_data(self, model_data):
self._model_run = None
self._debug_data = None
self._model_data = model_data
self.inputs = self._model_data.filter_by_attrs(is_result=0)
results = self._model_data.filter_by_attrs(is_result=1)
if len(results.data_vars) > 0:
self.results = results
log_time(
self._timings, 'model_data_loaded',
comment='Model: loaded model_data',
time_since_start=True
)
def clean_results(results, zero_threshold, timings):
"""
Remove unreasonably small values (solver output can lead to floating point
errors) and remove unmet_demand if it was never used (i.e. sum = zero)
zero_threshold is a value set in model configuration. If not set, defaults
to zero (i.e. doesn't do anything). Reasonable value = 1e-12
"""
threshold_applied = []
for k, v in results.data_vars.items():
# If there are any values in the data variable which fall below the
# threshold, note the data variable name and set those values to zero
if v.where(abs(v) < zero_threshold, drop=True).sum():
threshold_applied.append(k)
with np.errstate(invalid="ignore"):
v.values[abs(v.values) < zero_threshold] = 0
v.loc[{}] = v.values
if threshold_applied:
comment = "All values < {} set to 0 in {}".format(
zero_threshold, ", ".join(threshold_applied))
else:
comment = "zero threshold of {} not required".format(zero_threshold)
log_time(logger,
timings,
"threshold_applied",
comment="Postprocessing: " + comment)
# Combine unused_supply and unmet_demand into one variable
if ("unmet_demand" in results.data_vars.keys()
or "unused_supply" in results.data_vars.keys()):
results["unmet_demand"] = results.get("unmet_demand", 0) + results.get(
"unused_supply", 0)
results = results.drop_vars("unused_supply")
if not results.unmet_demand.sum():
log_time(
logger,
timings,
"delete_unmet_demand",
comment=
"Postprocessing: Model was feasible, deleting unmet_demand variable",
)
results = results.drop_vars("unmet_demand")
return results
def rerun_pyomo_model(model_data, backend_model):
"""
Rerun the Pyomo backend, perhaps after updating a parameter value,
(de)activating a constraint/objective or updating run options in the model
model_data object (e.g. `run.solver`).
Returns
-------
run_data : xarray.Dataset
Raw data from this rerun, including both inputs and results.
to filter inputs/results, use `run_data.filter_by_attrs(is_result=...)`
with 0 for inputs and 1 for results.
"""
backend_model.__calliope_run_config = AttrDict.from_yaml_string(model_data.attrs['run_config'])
if backend_model.__calliope_run_config['mode'] != 'plan':
raise exceptions.ModelError(
'Cannot rerun the backend in {} run mode. Only `plan` mode is '
'possible.'.format(backend_model.__calliope_run_config['mode'])
)
timings = {}
log_time(logger, timings, 'model_creation')
results, backend_model = backend_run.run_plan(
model_data, timings, run_pyomo,
build_only=False, backend_rerun=backend_model
)
for k, v in timings.items():
results.attrs['timings.' + k] = v
exceptions.ModelWarning(
'model.results will only be updated on running the model from '
'`model.run()`. We provide results of this rerun as a standalone xarray '
'Dataset'
)
results.attrs.update(model_data.attrs)
for key, var in results.data_vars.items():
var.attrs['is_result'] = 1
inputs = access_pyomo_model_inputs(backend_model)
for key, var in inputs.data_vars.items():
var.attrs['is_result'] = 0
results.update(inputs)
run_data = results
return run_data
def rerun_pyomo_model(model_data, backend_model):
"""
Rerun the Pyomo backend, perhaps after updating a parameter value,
(de)activating a constraint/objective or updating run options in the model
model_data object (e.g. `run.solver`).
Returns
-------
run_data : xarray.Dataset
Raw data from this rerun, including both inputs and results.
to filter inputs/results, use `run_data.filter_by_attrs(is_result=...)`
with 0 for inputs and 1 for results.
"""
if model_data.attrs['run.mode'] != 'plan':
raise exceptions.ModelError(
'Cannot rerun the backend in {} run mode. Only `plan` mode is '
'possible.'.format(model_data.attrs['run.mode'])
)
timings = {}
log_time(timings, 'model_creation')
results, backend_model = backend_run.run_plan(
model_data, timings, run_pyomo,
build_only=False, backend_rerun=backend_model
)
for k, v in timings.items():
results.attrs['timings.' + k] = v
exceptions.ModelWarning(
'model.results will only be updated on running the model from '
'`model.run()`. We provide results of this rerun as a standalone xarray '
'Dataset'
)
results.attrs.update(model_data.attrs)
for key, var in results.data_vars.items():
var.attrs['is_result'] = 1
inputs = access_pyomo_model_inputs(backend_model)
for key, var in inputs.data_vars.items():
var.attrs['is_result'] = 0
results.update(inputs)
run_data = results
return run_data
def clean_results(results, zero_threshold, timings):
"""
Remove unreasonably small values (solver output can lead to floating point
errors) and remove unmet_demand if it was never used (i.e. sum = zero)
zero_threshold is a value set in model configuration. If not set, defaults
to zero (i.e. doesn't do anything). Reasonable value = 1e-12
"""
threshold_applied = []
for k, v in results.data_vars.items():
# If there are any values in the data variable which fall below the
# threshold, note the data variable name and set those values to zero
if v.where(abs(v) < zero_threshold, drop=True).sum():
threshold_applied.append(k)
with np.errstate(invalid='ignore'):
v.values[abs(v.values) < zero_threshold] = 0
v.loc[{}] = v.values
if threshold_applied:
comment = 'All values < {} set to 0 in {}'.format(zero_threshold, ', '.join(threshold_applied))
else:
comment = 'zero threshold of {} not required'.format(zero_threshold)
log_time(
timings, 'threshold_applied',
comment='Postprocessing: ' + comment
)
# Combine unused_supply and unmet_demand into one variable
if ('unmet_demand' in results.data_vars.keys() or
'unused_supply' in results.data_vars.keys()):
results['unmet_demand'] = (
results.get('unmet_demand', 0) + results.get('unused_supply', 0)
)
results = results.drop('unused_supply')
if not results.unmet_demand.sum():
log_time(
timings, 'delete_unmet_demand',
comment='Postprocessing: Model was feasible, deleting unmet_demand variable'
)
results = results.drop('unmet_demand')
return results
def __init__(self, config, model_data=None, debug=False, *args, **kwargs):
"""
Returns a new Model from either the path to a YAML model
configuration file or a dict fully specifying the model.
Parameters
----------
config : str or dict or AttrDict
If str, must be the path to a model configuration file.
If dict or AttrDict, must fully specify the model.
model_data : Dataset, optional
Create a Model instance from a fully built model_data Dataset.
This is only used if `config` is explicitly set to None
and is primarily used to re-create a Model instance from
a model previously saved to a NetCDF file.
"""
self._timings = {}
# try to set logging output format assuming python interactive. Will
# use CLI logging format if model called from CLI
log_time(logger,
self._timings,
"model_creation",
comment="Model: initialising")
if isinstance(config, str):
model_run, debug_data = model_run_from_yaml(
config, *args, **kwargs)
self._init_from_model_run(model_run, debug_data, debug)
elif isinstance(config, dict):
model_run, debug_data = model_run_from_dict(
config, *args, **kwargs)
self._init_from_model_run(model_run, debug_data, debug)
elif model_data is not None and config is None:
self._init_from_model_data(model_data)
else:
# expected input is a string pointing to a YAML file of the run
# configuration or a dict/AttrDict in which the run and model
# configurations are defined
raise ValueError(
"Input configuration must either be a string or a dictionary.")
self._check_future_deprecation_warnings()
self.plot = plotting.ModelPlotMethods(self)
def _init_from_model_data(self, model_data):
if "_model_run" in model_data.attrs:
self._model_run = AttrDict.from_yaml_string(model_data.attrs["_model_run"])
del model_data.attrs["_model_run"]
if "_debug_data" in model_data.attrs:
self._debug_data = AttrDict.from_yaml_string(
model_data.attrs["_debug_data"]
)
del model_data.attrs["_debug_data"]
self._model_data = model_data
self._add_model_data_methods()
log_time(
logger,
self._timings,
"model_data_loaded",
comment="Model: loaded model_data",
)
def postprocess_model_results(results, model_data, timings):
"""
Adds additional post-processed result variables to
the given model results in-place. Model must have solved successfully.
Parameters
----------
results : xarray Dataset
Output from the solver backend
model_data : xarray Dataset
Calliope model data, stored as calliope.Model()._model_data
timings : dict
Calliope timing dictionary, stored as calliope.Model()._timings
Returns
-------
results : xarray Dataset
Input results Dataset, with additional DataArray variables and removed
all instances of unreasonably low numbers (set by zero_threshold)
"""
log_time(timings, 'post_process_start', comment='Postprocessing: started')
results['capacity_factor'] = capacity_factor(results, model_data)
results['systemwide_capacity_factor'] = systemwide_capacity_factor(
results, model_data)
results['systemwide_levelised_cost'] = systemwide_levelised_cost(
results, model_data)
results['total_levelised_cost'] = systemwide_levelised_cost(results,
model_data,
total=True)
results = clean_results(results,
model_data.attrs.get('run.zero_threshold', 0),
timings)
log_time(timings,
'post_process_end',
time_since_start=True,
comment='Postprocessing: ended')
return results
def clean_results(results, zero_threshold, timings):
"""
Remove unreasonably small values (solver output can lead to floating point
errors) and remove unmet_demand if it was never used (i.e. sum = zero)
zero_threshold is a value set in model configuration. If not set, defaults
to zero (i.e. doesn't do anything). Reasonable value = 1e-12
"""
threshold_applied = []
for k, v in results.data_vars.items():
# If there are any values in the data variable which fall below the
# threshold, note the data variable name and set those values to zero
if v.where(abs(v) < zero_threshold, drop=True).sum():
threshold_applied.append(k)
with np.errstate(invalid='ignore'):
v.values[abs(v.values) < zero_threshold] = 0
v.loc[{}] = v.values
if threshold_applied:
comment = 'All values < {} set to 0 in {}'.format(
zero_threshold, ', '.join(threshold_applied))
else:
comment = 'zero threshold of {} not required'.format(zero_threshold)
log_time(timings,
'threshold_applied',
comment='Postprocessing: ' + comment)
if 'unmet_demand' in results.data_vars.keys(
) and not results.unmet_demand.sum():
log_time(
timings,
'delete_unmet_demand',
comment=
'Postprocessing: Model was feasible, deleting unmet_demand variable'
)
results = results.drop('unmet_demand')
return results
def _init_from_model_run(self, model_run, debug_data):
self._model_run = model_run
self._debug_data = debug_data
log_time(self._timings,
'model_run_creation',
comment='Model: preprocessing stage 1 (model_run)')
self._model_data_original = build_model_data(model_run)
log_time(self._timings,
'model_data_original_creation',
comment='Model: preprocessing stage 2 (model_data)')
random_seed = self._model_run.get_key('model.random_seed', None)
if random_seed:
np.random.seed(seed=random_seed)
# After setting the random seed, time clustering can take place
time_config = model_run.model.get('time', None)
if not time_config:
_model_data = self._model_data_original
else:
_model_data = apply_time_clustering(self._model_data_original,
model_run)
self._model_data = final_timedimension_processing(_model_data)
log_time(self._timings,
'model_data_creation',
comment='Model: preprocessing complete',
time_since_start=True)
for var in self._model_data.data_vars:
self._model_data[var].attrs['is_result'] = 0
self.inputs = self._model_data.filter_by_attrs(is_result=0)
def _init_from_model_run(self, model_run, debug_data):
self._model_run = model_run
self._debug_data = debug_data
log_time(self._timings, 'model_run_creation', comment='Model: preprocessing stage 1 (model_run)')
self._model_data_original = build_model_data(model_run)
log_time(self._timings, 'model_data_original_creation', comment='Model: preprocessing stage 2 (model_data)')
random_seed = self._model_run.get_key('model.random_seed', None)
if random_seed:
np.random.seed(seed=random_seed)
# After setting the random seed, time clustering can take place
time_config = model_run.model.get('time', None)
if not time_config:
_model_data = self._model_data_original
else:
_model_data = apply_time_clustering(
self._model_data_original, model_run
)
self._model_data = final_timedimension_processing(_model_data)
log_time(
self._timings, 'model_data_creation',
comment='Model: preprocessing complete',
time_since_start=True
)
for var in self._model_data.data_vars:
self._model_data[var].attrs['is_result'] = 0
self.inputs = self._model_data.filter_by_attrs(is_result=0)
def __init__(self, config, model_data=None, *args, **kwargs):
"""
Returns a new Model from either the path to a YAML model
configuration file or a dict fully specifying the model.
Parameters
----------
config : str or dict or AttrDict
If str, must be the path to a model configuration file.
If dict or AttrDict, must fully specify the model.
model_data : Dataset, optional
Create a Model instance from a fully built model_data Dataset.
This is only used if `config` is explicitly set to None
and is primarily used to re-create a Model instance from
a model previously saved to a NetCDF file.
"""
self._timings = {}
# try to set logging output format assuming python interactive. Will
# use CLI logging format if model called from CLI
log_time(self._timings, 'model_creation', comment='Model: initialising')
if isinstance(config, str):
model_run, debug_data = model_run_from_yaml(config, *args, **kwargs)
self._init_from_model_run(model_run, debug_data)
elif isinstance(config, dict):
model_run, debug_data = model_run_from_dict(config, *args, **kwargs)
self._init_from_model_run(model_run, debug_data)
elif model_data is not None and config is None:
self._init_from_model_data(model_data)
else:
# expected input is a string pointing to a YAML file of the run
# configuration or a dict/AttrDict in which the run and model
# configurations are defined
raise ValueError(
'Input configuration must either be a string or a dictionary.'
)
check_future_deprecation_warnings(self._model_run, self._model_data)
self.plot = plotting.ModelPlotMethods(self)
def postprocess_model_results(results, model_data, timings):
"""
Adds additional post-processed result variables to
the given model results in-place. Model must have solved successfully.
Parameters
----------
results : xarray Dataset
Output from the solver backend
model_data : xarray Dataset
Calliope model data, stored as calliope.Model()._model_data
timings : dict
Calliope timing dictionary, stored as calliope.Model()._timings
Returns
-------
results : xarray Dataset
Input results Dataset, with additional DataArray variables and removed
all instances of unreasonably low numbers (set by zero_threshold)
"""
log_time(
timings, 'post_process_start',
comment='Postprocessing: started'
)
results['capacity_factor'] = capacity_factor(results, model_data)
results['systemwide_capacity_factor'] = systemwide_capacity_factor(results, model_data)
results['systemwide_levelised_cost'] = systemwide_levelised_cost(results, model_data)
results['total_levelised_cost'] = systemwide_levelised_cost(results, model_data, total=True)
results = clean_results(results, model_data.attrs.get('run.zero_threshold', 0), timings)
log_time(
timings, 'post_process_end', time_since_start=True,
comment='Postprocessing: ended'
)
return results
def _init_from_model_run(self, model_run, debug_data, debug):
self._model_run = model_run
log_time(
logger,
self._timings,
"model_run_creation",
comment="Model: preprocessing stage 1 (model_run)",
)
model_data_factory = ModelDataFactory(model_run)
(
model_data_pre_clustering,
model_data,
data_pre_time,
stripped_keys,
) = model_data_factory()
self._model_data_pre_clustering = model_data_pre_clustering
self._model_data = model_data
if debug:
self._debug_data = debug_data
self._model_data_pre_time = data_pre_time
self._model_data_stripped_keys = stripped_keys
self.inputs = self._model_data.filter_by_attrs(is_result=0)
log_time(
logger,
self._timings,
"model_data_original_creation",
comment="Model: preprocessing stage 2 (model_data)",
)
# Ensure model and run attributes of _model_data update themselves
model_config = {
k: v
for k, v in model_run.get("model", {}).items()
if k != "file_allowed"
}
self.model_config = UpdateObserverDict(initial_dict=model_config,
name="model_config",
observer=self._model_data)
self.run_config = UpdateObserverDict(
initial_dict=model_run.get("run", {}),
name="run_config",
observer=self._model_data,
)
self.subsets = UpdateObserverDict(
initial_dict=model_run.get("subsets").as_dict_flat(),
name="subsets",
observer=self._model_data,
)
log_time(
logger,
self._timings,
"model_data_creation",
comment="Model: preprocessing complete",
)
def _init_from_model_run(self, model_run, debug_data):
self._model_run = model_run
self._debug_data = debug_data
log_time(
logger,
self._timings,
"model_run_creation",
comment="Model: preprocessing stage 1 (model_run)",
)
self._model_data_original = build_model_data(model_run)
log_time(
logger,
self._timings,
"model_data_original_creation",
comment="Model: preprocessing stage 2 (model_data)",
)
random_seed = self._model_run.get_key("model.random_seed", None)
if random_seed:
np.random.seed(seed=random_seed)
# After setting the random seed, time clustering can take place
time_config = model_run.model.get("time", None)
if not time_config:
_model_data = self._model_data_original
else:
_model_data = apply_time_clustering(self._model_data_original,
model_run)
self._model_data = final_timedimension_processing(_model_data)
log_time(
logger,
self._timings,
"model_data_creation",
comment="Model: preprocessing complete",
)
# Ensure model and run attributes of _model_data update themselves
for var in self._model_data.data_vars:
self._model_data[var].attrs["is_result"] = 0
self.inputs = self._model_data.filter_by_attrs(is_result=0)
model_config = {
k: v
for k, v in model_run.get("model", {}).items()
if k != "file_allowed"
}
self.model_config = UpdateObserverDict(initial_dict=model_config,
name="model_config",
observer=self._model_data)
self.run_config = UpdateObserverDict(
initial_dict=model_run.get("run", {}),
name="run_config",
observer=self._model_data,
)
def test_timing_log(self):
timings = {'model_creation': datetime.datetime.now()}
# TODO: capture logging output and check that comment is in string
log_time(timings, 'test', comment='test_comment', level='info')
assert isinstance(timings['test'], datetime.datetime)
log_time(timings, 'test2', comment=None, level='info')
assert isinstance(timings['test2'], datetime.datetime)
# TODO: capture logging output and check that time_since_start is in the string
log_time(timings,
'test',
comment=None,
level='info',
time_since_start=True)
def test_timing_log(self):
timings = {"model_creation": datetime.datetime.now()}
logger = logging.getLogger("calliope.testlogger")
# TODO: capture logging output and check that comment is in string
log_time(logger, timings, "test", comment="test_comment", level="info")
assert isinstance(timings["test"], datetime.datetime)
log_time(logger, timings, "test2", comment=None, level="info")
assert isinstance(timings["test2"], datetime.datetime)
# TODO: capture logging output and check that time_since_run_start is in the string
log_time(
logger,
timings,
"test",
comment=None,
level="info",
time_since_run_start=True,
)
def run_spores(
model_data,
run_config,
timings,
interface,
backend,
build_only,
backend_rerun=False,
opt=None,
):
"""
For use when mode is 'spores', to allow the model to be built, edited, and
iteratively run within Pyomo, modifying, at each iteration, the score of
each location-technology combination in such a way to generate
Spatially explicit Practically Optimal RESults (SPORES).
"""
log_time(
logger,
timings,
"run_start",
comment="Backend: starting model run in SPORES mode",
)
def _cap_loc_score_default(results, subset=None):
if subset is None:
subset = {}
# Define default scoring function, based on integer scoring method
# TODO: make it possible to point to a custom function instead of this one
cap_loc_score = xr.where(results.energy_cap > 1e-3, 100, 0)
return cap_loc_score.to_series()[subset]
# Define function to update "spores_score" after each iteration of the method
def _update_spores_score(backend_model, cap_loc_score):
print("Updating node-technology spores scores")
cap_loc_score_dict = cap_loc_score.to_dict()
interface.update_pyomo_param(backend_model, opt, "cost_energy_cap",
cap_loc_score_dict)
def _warn_on_infeasibility():
return exceptions.warn(
"Infeasible SPORE detected. Please check your model configuration. "
"No more SPORES will be generated.")
def _get_updated_spores_inputs(backend_model):
inputs = interface.access_pyomo_model_inputs(backend_model)
inputs_to_keep = ["cost_energy_cap"]
return inputs[inputs_to_keep]
def _combine_spores_results_and_inputs(backend_model,
results,
spore_num,
model_data=None):
inputs_to_keep = _get_updated_spores_inputs(backend_model)
for var_data in results.data_vars.values():
if "is_result" not in var_data.attrs.keys():
var_data.attrs["is_result"] = 1
if model_data is not None:
datasets = [inputs_to_keep, model_data, results]
else:
datasets = [inputs_to_keep, results]
new_ds = xr.combine_by_coords(datasets,
compat="override",
combine_attrs="no_conflicts")
return new_ds.assign_coords(spores=("spores", [spore_num]))
def _add_results_to_list(backend_model, spores_results, results,
spore_num):
results_to_add = _combine_spores_results_and_inputs(
backend_model, results, spore_num)
spores_results[spore_num] = results_to_add
def _save_spore(backend_model, results, spore_num, model_data=None):
_path = spores_config["save_per_spore_path"].format(spore_num)
new_ds = _combine_spores_results_and_inputs(backend_model, results,
spore_num, model_data)
print(f"Saving SPORE {spore_num} to {_path}")
io.save_netcdf(new_ds, _path)
def _update_slack_cost_constraint(backend_model):
slack_costs = model_data.group_cost_max.loc[{
"group_names_cost_max":
spores_config["slack_cost_group"]
}].dropna("costs")
interface.update_pyomo_param(
backend_model,
opt,
"group_cost_max",
{(_cost_class, spores_config["slack_cost_group"]):
results.cost.loc[{
"costs": _cost_class
}].sum().item() * (1 + spores_config["slack"])
for _cost_class in slack_costs.costs.values},
)
def _update_to_spores_objective(backend_model):
interface.update_pyomo_param(
backend_model,
opt,
"objective_cost_class",
spores_config["objective_cost_class"],
)
def _initialise_backend_model():
if backend_rerun:
kwargs = {"backend_rerun": backend_rerun, "opt": opt}
else:
kwargs = {}
if spores_config["skip_cost_op"]:
print(
"Skipping cost optimal run and using model_data to initialise SPORES directly"
)
return run_plan(model_data,
run_config,
timings,
backend,
build_only=True,
**kwargs)
else:
# Run once for the 'cost-optimal' solution
return run_plan(
model_data,
run_config,
timings,
backend,
build_only,
persistent=False,
**kwargs,
)
def _initialise_spores_number():
if "spores" in model_data.coords and spores_config["skip_cost_op"]:
return model_data.spores.max().item()
else:
return 0
def _error_on_malformed_input():
if backend_rerun:
try:
backend_rerun.obj()
except ValueError: # model has not yet been run
pass
else:
raise exceptions.ModelError(
"Cannot run SPORES if the backend model already has a solution. "
"Consider using the `build_only` optional `run()` argument to avoid this."
)
if "spores" in model_data.filter_by_attrs(is_result=0).squeeze().dims:
raise exceptions.ModelError(
"Cannot run SPORES with a SPORES dimension in any input (e.g. `cost_energy_cap`)."
)
_error_on_malformed_input()
if backend_rerun:
model_data = _combine_spores_results_and_inputs(
backend_rerun, xr.Dataset(), 0, model_data=model_data) #
spores_config = run_config["spores_options"]
if "spores" in model_data.dims and model_data.spores.size == 1:
model_data = model_data.squeeze("spores")
init_spores_scores = (model_data.cost_energy_cap.loc[{
"costs": [spores_config["score_cost_class"]]
}].to_series().dropna())
spores_results = {}
results, backend_model, opt = _initialise_backend_model()
if build_only:
return results, backend_model, opt
init_spore = _initialise_spores_number()
if spores_config["skip_cost_op"]:
cumulative_spores_scores = init_spores_scores.copy()
_update_to_spores_objective(backend_model)
elif results.attrs["termination_condition"] in ["optimal", "feasible"]:
results.attrs["objective_function_value"] = backend_model.obj()
if spores_config["save_per_spore"] is True:
_save_spore(backend_model,
results,
init_spore,
model_data=model_data)
# Storing results and scores in the specific dictionaries
_add_results_to_list(backend_model, spores_results, results, 0)
cumulative_spores_scores = init_spores_scores + _cap_loc_score_default(
results, init_spores_scores.index.droplevel("costs"))
# Set group constraint "cost_max" equal to slacked cost
_update_slack_cost_constraint(backend_model)
_update_to_spores_objective(backend_model)
# Update "spores_score" based on previous iteration
_update_spores_score(backend_model, cumulative_spores_scores)
log_time(
logger,
timings,
"run_solution_returned",
time_since_run_start=True,
comment=
"Backend: generated solution array for the cost-optimal case",
)
else:
_warn_on_infeasibility()
return results, backend_model, opt
# Iterate over the number of SPORES requested by the user
for _spore in range(init_spore + 1, spores_config["spores_number"] + 1):
print(f"Running SPORES {_spore}")
if opt is not None and "persistent" in opt.type:
opt = interface.regenerate_persistent_pyomo_solver(
backend_model,
opt,
obj=True,
constraints={
"cost_investment_constraint":
cumulative_spores_scores.index,
"cost_constraint": cumulative_spores_scores.index,
},
)
else:
opt = None
results, backend_model, opt = run_plan(
model_data,
run_config,
timings,
backend,
build_only=False,
backend_rerun=backend_model,
allow_warmstart=False,
persistent=True,
opt=opt,
)
if results.attrs["termination_condition"] in ["optimal", "feasible"]:
results.attrs["objective_function_value"] = backend_model.obj()
if spores_config["save_per_spore"] is True:
_save_spore(backend_model, results, _spore)
# Storing results and scores in the specific dictionaries
_add_results_to_list(backend_model, spores_results, results,
_spore)
print(
f"Updating capacity scores from {cumulative_spores_scores.sum()}..."
)
cumulative_spores_scores += _cap_loc_score_default(
results, init_spores_scores.index.droplevel("costs"))
print(f"... to {cumulative_spores_scores.sum()}")
# Update "spores_score" based on previous iteration
_update_spores_score(backend_model, cumulative_spores_scores)
else:
_warn_on_infeasibility()
break
log_time(
logger,
timings,
"run_solution_returned",
time_since_run_start=True,
comment=f"Backend: generated solution array for the SPORE {_spore}",
)
results = xr.concat(spores_results.values(),
dim=pd.Index(spores_results.keys(), name="spores"))
return results, backend_model, opt
def run_plan(model_data, timings, backend, build_only, backend_rerun=False):
log_time(logger,
timings,
'run_start',
comment='Backend: starting model run')
if not backend_rerun:
backend_model = backend.generate_model(model_data)
log_time(logger,
timings,
'run_backend_model_generated',
time_since_run_start=True,
comment='Backend: model generated')
else:
backend_model = backend_rerun
run_config = backend_model.__calliope_run_config
solver = run_config['solver']
solver_io = run_config.get('solver_io', None)
solver_options = run_config.get('solver_options', None)
save_logs = run_config.get('save_logs', None)
if build_only:
results = xr.Dataset()
else:
log_time(logger,
timings,
'run_solver_start',
comment='Backend: sending model to solver')
results = backend.solve_model(backend_model,
solver=solver,
solver_io=solver_io,
solver_options=solver_options,
save_logs=save_logs)
log_time(logger,
timings,
'run_solver_exit',
time_since_run_start=True,
comment='Backend: solver finished running')
termination = backend.load_results(backend_model, results)
log_time(logger,
timings,
'run_results_loaded',
comment='Backend: loaded results')
results = backend.get_result_array(backend_model, model_data)
results.attrs['termination_condition'] = termination
if results.attrs['termination_condition'] in ['optimal', 'feasible']:
results.attrs['objective_function_value'] = backend_model.obj()
log_time(logger,
timings,
'run_solution_returned',
time_since_run_start=True,
comment='Backend: generated solution array')
return results, backend_model
def run_operate(model_data, timings, backend, build_only):
"""
For use when mode is 'operate', to allow the model to be built, edited, and
iteratively run within Pyomo.
"""
log_time(logger,
timings,
'run_start',
comment='Backend: starting model run in operational mode')
defaults = AttrDict.from_yaml_string(model_data.attrs['defaults'])
run_config = AttrDict.from_yaml_string(model_data.attrs['run_config'])
operate_params = ['purchased'] + [
i.replace('_max', '') for i in defaults if i[-4:] == '_max'
]
# Capacity results (from plan mode) can be used as the input to operate mode
if (any(model_data.filter_by_attrs(is_result=1).data_vars)
and run_config.get('operation.use_cap_results', False)):
# Anything with is_result = 1 will be ignored in the Pyomo model
for varname, varvals in model_data.data_vars.items():
if varname in operate_params:
varvals.attrs['is_result'] = 1
varvals.attrs['operate_param'] = 1
else:
cap_max = xr.merge([
v.rename(k.replace('_max', ''))
for k, v in model_data.data_vars.items() if '_max' in k
])
cap_equals = xr.merge([
v.rename(k.replace('_equals', ''))
for k, v in model_data.data_vars.items() if '_equals' in k
])
caps = cap_max.update(cap_equals)
for cap in caps.data_vars.values():
cap.attrs['is_result'] = 1
cap.attrs['operate_param'] = 1
model_data.update(caps)
# Storage initial is carried over between iterations, so must be defined along with storage
if ('loc_techs_store' in model_data.dims.keys()
and 'storage_initial' not in model_data.data_vars.keys()):
model_data['storage_initial'] = (xr.DataArray(
[0 for loc_tech in model_data.loc_techs_store.values],
dims='loc_techs_store'))
model_data['storage_initial'].attrs['is_result'] = 0
exceptions.warn(
'Initial stored energy not defined, set to zero for all '
'loc::techs in loc_techs_store, for use in iterative optimisation')
# Operated units is carried over between iterations, so must be defined in a milp model
if ('loc_techs_milp' in model_data.dims.keys()
and 'operated_units' not in model_data.data_vars.keys()):
model_data['operated_units'] = (xr.DataArray(
[0 for loc_tech in model_data.loc_techs_milp.values],
dims='loc_techs_milp'))
model_data['operated_units'].attrs['is_result'] = 1
model_data['operated_units'].attrs['operate_param'] = 1
exceptions.warn(
'daily operated units not defined, set to zero for all '
'loc::techs in loc_techs_milp, for use in iterative optimisation')
comments, warnings, errors = checks.check_operate_params(model_data)
exceptions.print_warnings_and_raise_errors(warnings=warnings,
errors=errors)
# Initialize our variables
solver = run_config['solver']
solver_io = run_config.get('solver_io', None)
solver_options = run_config.get('solver_options', None)
save_logs = run_config.get('save_logs', None)
window = run_config['operation']['window']
horizon = run_config['operation']['horizon']
window_to_horizon = horizon - window
# get the cumulative sum of timestep resolution, to find where we hit our window and horizon
timestep_cumsum = model_data.timestep_resolution.cumsum(
'timesteps').to_pandas()
# get the timesteps at which we start and end our windows
window_ends = timestep_cumsum.where((timestep_cumsum % window == 0) | (
timestep_cumsum == timestep_cumsum[-1]))
window_starts = timestep_cumsum.where((~np.isnan(window_ends.shift(1))) | (
timestep_cumsum == timestep_cumsum[0])).dropna()
window_ends = window_ends.dropna()
horizon_ends = timestep_cumsum[timestep_cumsum.isin(window_ends.values +
window_to_horizon)]
if not any(window_starts):
raise exceptions.ModelError(
'Not enough timesteps or incorrect timestep resolution to run in '
'operational mode with an optimisation window of {}'.format(
window))
# We will only update timseries parameters
timeseries_data_vars = [
k for k, v in model_data.data_vars.items()
if 'timesteps' in v.dims and v.attrs['is_result'] == 0
]
# Loop through each window, solve over the horizon length, and add result to
# result_array we only go as far as the end of the last horizon, which may
# clip the last bit of data
result_array = []
# track whether each iteration finds an optimal solution or not
terminations = []
if build_only:
iterations = [0]
else:
iterations = range(len(window_starts))
for i in iterations:
start_timestep = window_starts.index[i]
# Build full model in first instance
if i == 0:
warmstart = False
end_timestep = horizon_ends.index[i]
timesteps = slice(start_timestep, end_timestep)
window_model_data = model_data.loc[dict(timesteps=timesteps)]
log_time(logger,
timings,
'model_gen_1',
comment='Backend: generating initial model')
backend_model = backend.generate_model(window_model_data)
# Build the full model in the last instance(s),
# where number of timesteps is less than the horizon length
elif i > len(horizon_ends) - 1:
warmstart = False
end_timestep = window_ends.index[i]
timesteps = slice(start_timestep, end_timestep)
window_model_data = model_data.loc[dict(timesteps=timesteps)]
log_time(
logger,
timings,
'model_gen_{}'.format(i + 1),
comment=(
'Backend: iteration {}: generating new model for '
'end of timeseries, with horizon = {} timesteps'.format(
i + 1, window_ends[i] - window_starts[i])))
backend_model = backend.generate_model(window_model_data)
# Update relevent Pyomo Params in intermediate instances
else:
warmstart = True
end_timestep = horizon_ends.index[i]
timesteps = slice(start_timestep, end_timestep)
window_model_data = model_data.loc[dict(timesteps=timesteps)]
log_time(
logger,
timings,
'model_gen_{}'.format(i + 1),
comment='Backend: iteration {}: updating model parameters'.
format(i + 1))
# Pyomo model sees the same timestamps each time, we just change the
# values associated with those timestamps
for var in timeseries_data_vars:
# New values
var_series = window_model_data[var].to_series().dropna(
).replace('inf', np.inf)
# Same timestamps
var_series.index = backend_model.__calliope_model_data['data'][
var].keys()
var_dict = var_series.to_dict()
# Update pyomo Param with new dictionary
getattr(backend_model, var).store_values(var_dict)
if not build_only:
log_time(logger,
timings,
'model_run_{}'.format(i + 1),
time_since_run_start=True,
comment='Backend: iteration {}: sending model to solver'.
format(i + 1))
# After iteration 1, warmstart = True, which should speed up the process
# Note: Warmstart isn't possible with GLPK (dealt with later on)
_results = backend.solve_model(
backend_model,
solver=solver,
solver_io=solver_io,
solver_options=solver_options,
save_logs=save_logs,
warmstart=warmstart,
)
log_time(logger,
timings,
'run_solver_exit_{}'.format(i + 1),
time_since_run_start=True,
comment='Backend: iteration {}: solver finished running'.
format(i + 1))
# xarray dataset is built for each iteration
_termination = backend.load_results(backend_model, _results)
terminations.append(_termination)
_results = backend.get_result_array(backend_model, model_data)
# We give back the actual timesteps for this iteration and take a slice
# equal to the window length
_results['timesteps'] = window_model_data.timesteps.copy()
# We always save the window data. Until the last window(s) this will crop
# the window_to_horizon timesteps. In the last window(s), optimistion will
# only be occurring over a window length anyway
_results = _results.loc[dict(
timesteps=slice(None, window_ends.index[i]))]
result_array.append(_results)
# Set up initial storage for the next iteration
if 'loc_techs_store' in model_data.dims.keys():
storage_initial = _results.storage.loc[{
'timesteps':
window_ends.index[i]
}].drop('timesteps')
model_data['storage_initial'].loc[
storage_initial.coords] = storage_initial.values
backend_model.storage_initial.store_values(
storage_initial.to_series().dropna().to_dict())
# Set up total operated units for the next iteration
if 'loc_techs_milp' in model_data.dims.keys():
operated_units = _results.operating_units.sum(
'timesteps').astype(np.int)
model_data['operated_units'].loc[{}] += operated_units.values
backend_model.operated_units.store_values(
operated_units.to_series().dropna().to_dict())
log_time(logger,
timings,
'run_solver_exit_{}'.format(i + 1),
time_since_run_start=True,
comment='Backend: iteration {}: generated solution array'.
format(i + 1))
if build_only:
results = xr.Dataset()
else:
# Concatenate results over the timestep dimension to get a single
# xarray Dataset of interest
results = xr.concat(result_array, dim='timesteps')
if all(i == 'optimal' for i in terminations):
results.attrs['termination_condition'] = 'optimal'
elif all(i in ['optimal', 'feasible'] for i in terminations):
results.attrs['termination_condition'] = 'feasible'
else:
results.attrs['termination_condition'] = ','.join(terminations)
log_time(logger,
timings,
'run_solution_returned',
time_since_run_start=True,
comment='Backend: generated full solution array')
return results, backend_model
def run_operate(model_data, timings, backend, build_only):
"""
For use when mode is 'operate', to allow the model to be built, edited, and
iteratively run within Pyomo.
"""
log_time(
logger,
timings,
"run_start",
comment="Backend: starting model run in operational mode",
)
defaults = UpdateObserverDict(
initial_yaml_string=model_data.attrs["defaults"],
name="defaults",
observer=model_data,
)
run_config = UpdateObserverDict(
initial_yaml_string=model_data.attrs["run_config"],
name="run_config",
observer=model_data,
)
# New param defaults = old maximum param defaults (e.g. energy_cap gets default from energy_cap_max)
operate_params = {
k.replace("_max", ""): v
for k, v in defaults.items() if k.endswith("_max")
}
operate_params[
"purchased"] = 0 # no _max to work from here, so we hardcode a default
defaults.update(operate_params)
# Capacity results (from plan mode) can be used as the input to operate mode
if any(model_data.filter_by_attrs(
is_result=1).data_vars) and run_config.get(
"operation.use_cap_results", False):
# Anything with is_result = 1 will be ignored in the Pyomo model
for varname, varvals in model_data.data_vars.items():
if varname in operate_params.keys():
varvals.attrs["is_result"] = 1
varvals.attrs["operate_param"] = 1
else:
cap_max = xr.merge([
v.rename(k.replace("_max", ""))
for k, v in model_data.data_vars.items() if "_max" in k
])
cap_equals = xr.merge([
v.rename(k.replace("_equals", ""))
for k, v in model_data.data_vars.items() if "_equals" in k
])
caps = cap_max.update(cap_equals)
for cap in caps.data_vars.values():
cap.attrs["is_result"] = 1
cap.attrs["operate_param"] = 1
model_data.update(caps)
comments, warnings, errors = checks.check_operate_params(model_data)
exceptions.print_warnings_and_raise_errors(warnings=warnings,
errors=errors)
# Initialize our variables
solver = run_config["solver"]
solver_io = run_config.get("solver_io", None)
solver_options = run_config.get("solver_options", None)
save_logs = run_config.get("save_logs", None)
window = run_config["operation"]["window"]
horizon = run_config["operation"]["horizon"]
window_to_horizon = horizon - window
# get the cumulative sum of timestep resolution, to find where we hit our window and horizon
timestep_cumsum = model_data.timestep_resolution.cumsum(
"timesteps").to_pandas()
# get the timesteps at which we start and end our windows
window_ends = timestep_cumsum.where((timestep_cumsum % window == 0) | (
timestep_cumsum == timestep_cumsum[-1]))
window_starts = timestep_cumsum.where((~np.isnan(window_ends.shift(1))) | (
timestep_cumsum == timestep_cumsum[0])).dropna()
window_ends = window_ends.dropna()
horizon_ends = timestep_cumsum[timestep_cumsum.isin(window_ends.values +
window_to_horizon)]
if not any(window_starts):
raise exceptions.ModelError(
"Not enough timesteps or incorrect timestep resolution to run in "
"operational mode with an optimisation window of {}".format(
window))
# We will only update timseries parameters
timeseries_data_vars = [
k for k, v in model_data.data_vars.items()
if "timesteps" in v.dims and v.attrs["is_result"] == 0
]
# Loop through each window, solve over the horizon length, and add result to
# result_array we only go as far as the end of the last horizon, which may
# clip the last bit of data
result_array = []
# track whether each iteration finds an optimal solution or not
terminations = []
if build_only:
iterations = [0]
else:
iterations = range(len(window_starts))
for i in iterations:
start_timestep = window_starts.index[i]
# Build full model in first instance
if i == 0:
warmstart = False
end_timestep = horizon_ends.index[i]
timesteps = slice(start_timestep, end_timestep)
window_model_data = model_data.loc[dict(timesteps=timesteps)]
log_time(
logger,
timings,
"model_gen_1",
comment="Backend: generating initial model",
)
backend_model = backend.generate_model(window_model_data)
# Build the full model in the last instance(s),
# where number of timesteps is less than the horizon length
elif i > len(horizon_ends) - 1:
warmstart = False
end_timestep = window_ends.index[i]
timesteps = slice(start_timestep, end_timestep)
window_model_data = model_data.loc[dict(timesteps=timesteps)]
log_time(
logger,
timings,
"model_gen_{}".format(i + 1),
comment=(
"Backend: iteration {}: generating new model for "
"end of timeseries, with horizon = {} timesteps".format(
i + 1, window_ends[i] - window_starts[i])),
)
backend_model = backend.generate_model(window_model_data)
# Update relevent Pyomo Params in intermediate instances
else:
warmstart = True
end_timestep = horizon_ends.index[i]
timesteps = slice(start_timestep, end_timestep)
window_model_data = model_data.loc[dict(timesteps=timesteps)]
log_time(
logger,
timings,
"model_gen_{}".format(i + 1),
comment="Backend: iteration {}: updating model parameters".
format(i + 1),
)
# Pyomo model sees the same timestamps each time, we just change the
# values associated with those timestamps
for var in timeseries_data_vars:
# New values
var_series = (
window_model_data[var].to_series().dropna().replace(
"inf", np.inf))
# Same timestamps
var_series.index = backend_model.__calliope_model_data["data"][
var].keys()
var_dict = var_series.to_dict()
# Update pyomo Param with new dictionary
getattr(backend_model, var).store_values(var_dict)
if not build_only:
log_time(
logger,
timings,
"model_run_{}".format(i + 1),
time_since_run_start=True,
comment="Backend: iteration {}: sending model to solver".
format(i + 1),
)
# After iteration 1, warmstart = True, which should speed up the process
# Note: Warmstart isn't possible with GLPK (dealt with later on)
_results = backend.solve_model(
backend_model,
solver=solver,
solver_io=solver_io,
solver_options=solver_options,
save_logs=save_logs,
warmstart=warmstart,
)
log_time(
logger,
timings,
"run_solver_exit_{}".format(i + 1),
time_since_run_start=True,
comment="Backend: iteration {}: solver finished running".
format(i + 1),
)
# xarray dataset is built for each iteration
_termination = backend.load_results(backend_model, _results)
terminations.append(_termination)
_results = backend.get_result_array(backend_model, model_data)
# We give back the actual timesteps for this iteration and take a slice
# equal to the window length
_results["timesteps"] = window_model_data.timesteps.copy()
# We always save the window data. Until the last window(s) this will crop
# the window_to_horizon timesteps. In the last window(s), optimistion will
# only be occurring over a window length anyway
_results = _results.loc[dict(
timesteps=slice(None, window_ends.index[i]))]
result_array.append(_results)
# Set up initial storage for the next iteration
if "loc_techs_store" in model_data.dims.keys():
storage_initial = _results.storage.loc[{
"timesteps":
window_ends.index[i]
}].drop("timesteps")
model_data["storage_initial"].loc[
storage_initial.coords] = storage_initial.values
backend_model.storage_initial.store_values(
storage_initial.to_series().dropna().to_dict())
# Set up total operated units for the next iteration
if "loc_techs_milp" in model_data.dims.keys():
operated_units = _results.operating_units.sum(
"timesteps").astype(np.int)
model_data["operated_units"].loc[{}] += operated_units.values
backend_model.operated_units.store_values(
operated_units.to_series().dropna().to_dict())
log_time(
logger,
timings,
"run_solver_exit_{}".format(i + 1),
time_since_run_start=True,
comment="Backend: iteration {}: generated solution array".
format(i + 1),
)
if build_only:
results = xr.Dataset()
else:
# Concatenate results over the timestep dimension to get a single
# xarray Dataset of interest
results = xr.concat(result_array, dim="timesteps")
if all(i == "optimal" for i in terminations):
results.attrs["termination_condition"] = "optimal"
elif all(i in ["optimal", "feasible"] for i in terminations):
results.attrs["termination_condition"] = "feasible"
else:
results.attrs["termination_condition"] = ",".join(terminations)
log_time(
logger,
timings,
"run_solution_returned",
time_since_run_start=True,
comment="Backend: generated full solution array",
)
return results, backend_model
def run_plan(model_data, timings, backend, build_only, backend_rerun=False):
log_time(timings, 'run_start', comment='Backend: starting model run')
if not backend_rerun:
backend_model = backend.generate_model(model_data)
log_time(
timings, 'run_backend_model_generated', time_since_start=True,
comment='Backend: model generated'
)
else:
backend_model = backend_rerun
solver = model_data.attrs['run.solver']
solver_io = model_data.attrs.get('run.solver_io', None)
solver_options = {
k.split('.')[-1]: v
for k, v in model_data.attrs.items() if '.solver_options.' in k
}
save_logs = model_data.attrs.get('run.save_logs', None)
if build_only:
results = xr.Dataset()
else:
log_time(
timings, 'run_solver_start',
comment='Backend: sending model to solver'
)
results = backend.solve_model(
backend_model, solver=solver,
solver_io=solver_io, solver_options=solver_options, save_logs=save_logs
)
log_time(
timings, 'run_solver_exit', time_since_start=True,
comment='Backend: solver finished running'
)
termination = backend.load_results(backend_model, results)
log_time(
timings, 'run_results_loaded',
comment='Backend: loaded results'
)
results = backend.get_result_array(backend_model, model_data)
results.attrs['termination_condition'] = termination
log_time(
timings, 'run_solution_returned', time_since_start=True,
comment='Backend: generated solution array'
)
return results, backend_model
def run_plan(model_data, timings, backend, build_only, backend_rerun=False):
log_time(timings, 'run_start', comment='Backend: starting model run')
if not backend_rerun:
backend_model = backend.generate_model(model_data)
log_time(timings,
'run_backend_model_generated',
time_since_start=True,
comment='Backend: model generated')
else:
backend_model = backend_rerun
solver = model_data.attrs['run.solver']
solver_io = model_data.attrs.get('run.solver_io', None)
solver_options = {
k.split('.')[-1]: v
for k, v in model_data.attrs.items() if '.solver_options.' in k
}
save_logs = model_data.attrs.get('run.save_logs', None)
if build_only:
results = xr.Dataset()
else:
log_time(timings,
'run_solver_start',
comment='Backend: sending model to solver')
results = backend.solve_model(backend_model,
solver=solver,
solver_io=solver_io,
solver_options=solver_options,
save_logs=save_logs)
log_time(timings,
'run_solver_exit',
time_since_start=True,
comment='Backend: solver finished running')
termination = backend.load_results(backend_model, results)
log_time(timings,
'run_results_loaded',
comment='Backend: loaded results')
results = backend.get_result_array(backend_model, model_data)
results.attrs['termination_condition'] = termination
log_time(timings,
'run_solution_returned',
time_since_start=True,
comment='Backend: generated solution array')
return results, backend_model
def rerun_pyomo_model(model_data, run_config, backend_model):
"""
Rerun the Pyomo backend, perhaps after updating a parameter value,
(de)activating a constraint/objective or updating run options in the model
model_data object (e.g. `run.solver`).
Returns
-------
new_model : calliope.Model
New calliope model, including both inputs and results, but no backend interface.
"""
backend_model.__calliope_run_config = run_config
if run_config["mode"] != "plan":
raise exceptions.ModelError(
"Cannot rerun the backend in {} run mode. Only `plan` mode is "
"possible.".format(run_config["mode"]))
timings = {}
log_time(logger, timings, "model_creation")
results, backend_model = backend_run.run_plan(
model_data,
run_config,
timings,
run_pyomo,
build_only=False,
backend_rerun=backend_model,
)
inputs = access_pyomo_model_inputs(backend_model)
# Add additional post-processed result variables to results
if results.attrs.get("termination_condition",
None) in ["optimal", "feasible"]:
results = postprocess_model_results(results,
model_data.reindex(results.coords),
timings)
for key, var in results.data_vars.items():
var.attrs["is_result"] = 1
for key, var in inputs.data_vars.items():
var.attrs["is_result"] = 0
new_model_data = xr.merge((results, inputs))
new_model_data.attrs.update(model_data.attrs)
new_model_data.attrs.update(results.attrs)
# Only add coordinates from the original model_data that don't already exist
new_coords = [
i for i in model_data.coords.keys()
if i not in new_model_data.coords.keys()
]
new_model_data = new_model_data.update(model_data[new_coords])
# Reorganise the coordinates so that model data and new model data share
# the same order of items in each dimension
new_model_data = new_model_data.reindex(model_data.coords)
exceptions.warn(
"The results of rerunning the backend model are only available within "
"the Calliope model returned by this function call.")
new_calliope_model = calliope.Model(config=None, model_data=new_model_data)
new_calliope_model._timings = timings
return new_calliope_model
def run_spores(model_data, timings, interface, backend, build_only):
"""
For use when mode is 'spores', to allow the model to be built, edited, and
iteratively run within Pyomo, modifying, at each iteration, the score of
each loc::tech in such a way to generate Spatially explicit Practically Optimal
RESults (SPORES).
"""
log_time(
logger,
timings,
"run_start",
comment="Backend: starting model run in SPORES mode",
)
run_config = UpdateObserverDict(
initial_yaml_string=model_data.attrs["run_config"],
name="run_config",
observer=model_data,
)
backend_model = backend.generate_model(model_data)
log_time(
logger,
timings,
"run_backend_model_generated",
time_since_run_start=True,
comment="Backend: model generated",
)
n_spores = run_config["spores_options"]["spores_number"]
slack = run_config["spores_options"]["slack"]
spores_score = run_config["spores_options"]["score_cost_class"]
slack_group = run_config["spores_options"]["slack_cost_group"]
# Define default scoring function, based on integer scoring method
# TODO: make the function to run optional
def _cap_loc_score_default(results, subset=None):
if subset is None:
subset = {}
cap_loc_score = split_loc_techs(results["energy_cap"]).loc[subset]
cap_loc_score = cap_loc_score.where(cap_loc_score > 1e-3, other=0)
cap_loc_score = cap_loc_score.where(cap_loc_score == 0, other=100)
return cap_loc_score.to_pandas()
# Define function to update "spores_score" after each iteration of the method
def _update_spores_score(backend_model, cap_loc_score):
loc_tech_score_dict = {
(spores_score, "{}::{}".format(i, j)): k
for (i, j), k in cap_loc_score.stack().items()
if "{}::{}".format(i, j) in model_data.loc_techs_investment_cost
}
interface.update_pyomo_param(backend_model, "cost_energy_cap",
loc_tech_score_dict)
def _warn_on_infeasibility():
return exceptions.warn(
"Infeasible SPORE detected. Please check your model configuration. "
"No more SPORES will be generated.")
# Run once for the 'cost-optimal' solution
results, backend_model = run_plan(model_data, timings, backend, build_only)
if build_only:
return results, backend_model # We have what we need, so break out of the loop
if results.attrs["termination_condition"] in ["optimal", "feasible"]:
results.attrs["objective_function_value"] = backend_model.obj()
# Storing results and scores in the specific dictionaries
spores_list = [results]
cum_scores = _cap_loc_score_default(results)
# Set group constraint "cost_max" equal to slacked cost
slack_costs = model_data.group_cost_max.loc[{
"group_names_cost_max":
slack_group
}].dropna("costs")
interface.update_pyomo_param(
backend_model,
"group_cost_max",
{(_cost_class, slack_group): results.cost.loc[{
"costs": _cost_class
}].sum().item() * (1 + slack)
for _cost_class in slack_costs.costs.values},
)
# Modify objective function weights: spores_score -> 1, all others -> 0
interface.update_pyomo_param(
backend_model,
"objective_cost_class",
{
spores_score: 1,
**{i: 0
for i in slack_costs.costs.values}
},
)
# Update "spores_score" based on previous iteration
_update_spores_score(backend_model, cum_scores)
else:
_warn_on_infeasibility()
return results, backend_model
log_time(
logger,
timings,
"run_solution_returned",
time_since_run_start=True,
comment="Backend: generated solution array for the cost-optimal case",
)
# Iterate over the number of SPORES requested by the user
for _spore in range(0, n_spores):
results, backend_model = run_plan(model_data,
timings,
backend,
build_only,
backend_rerun=backend_model)
if results.attrs["termination_condition"] in ["optimal", "feasible"]:
results.attrs["objective_function_value"] = backend_model.obj()
# Storing results and scores in the specific dictionaries
spores_list.append(results)
cum_scores += _cap_loc_score_default(results)
# Update "spores_score" based on previous iteration
_update_spores_score(backend_model, cum_scores)
else:
_warn_on_infeasibility()
break
log_time(
logger,
timings,
"run_solution_returned",
time_since_run_start=True,
comment=
"Backend: generated solution array for the cost-optimal case",
)
# TODO: make this function work with the spores dimension,
# so that postprocessing can take place in core/model.py, as with run_plan and run_operate
results = xr.concat(spores_list, dim="spores")
return results, backend_model
def run_plan(
model_data,
run_config,
timings,
backend,
build_only,
backend_rerun=False,
allow_warmstart=False,
persistent=True,
opt=None,
):
log_time(logger,
timings,
"run_start",
comment="Backend: starting model run")
warmstart = False
if not backend_rerun:
backend_model = backend.generate_model(model_data)
log_time(
logger,
timings,
"run_backend_model_generated",
time_since_run_start=True,
comment="Backend: model generated",
)
else:
backend_model = backend_rerun
if allow_warmstart:
warmstart = True
run_config = UpdateObserverDict(
initial_yaml_string=model_data.attrs["run_config"],
name="run_config",
observer=model_data,
)
solver = run_config["solver"]
solver_io = run_config.get("solver_io", None)
solver_options = run_config.get("solver_options", None)
save_logs = run_config.get("save_logs", None)
if build_only:
results = xr.Dataset()
else:
if "persistent" in solver and persistent is False:
exceptions.warn(
f"The chosen solver, `{solver}` will not be used in this run. "
f"`{solver.replace('_persistent', '')}` will be used instead.")
solver = solver.replace("_persistent", "")
log_time(
logger,
timings,
"run_solver_start",
comment="Backend: sending model to solver",
)
backend_results, opt = backend.solve_model(
backend_model,
solver=solver,
solver_io=solver_io,
solver_options=solver_options,
save_logs=save_logs,
warmstart=warmstart,
opt=opt,
)
log_time(
logger,
timings,
"run_solver_exit",
time_since_run_start=True,
comment="Backend: solver finished running",
)
termination = backend.load_results(backend_model, backend_results, opt)
log_time(logger,
timings,
"run_results_loaded",
comment="Backend: loaded results")
if termination in ["optimal", "feasible"]:
results = backend.get_result_array(backend_model, model_data)
results.attrs["termination_condition"] = termination
if "persistent" in opt.name and persistent is True:
results.attrs["objective_function_value"] = opt.get_model_attr(
"ObjVal")
else:
results.attrs["objective_function_value"] = backend_model.obj()
else:
results = xr.Dataset(attrs={"termination_condition": termination})
log_time(
logger,
timings,
"run_solution_returned",
time_since_run_start=True,
comment="Backend: generated solution array",
)
return results, backend_model, opt
def run_plan(model_data, timings, backend, build_only, backend_rerun=False):
log_time(logger,
timings,
"run_start",
comment="Backend: starting model run")
if not backend_rerun:
backend_model = backend.generate_model(model_data)
log_time(
logger,
timings,
"run_backend_model_generated",
time_since_run_start=True,
comment="Backend: model generated",
)
else:
backend_model = backend_rerun
run_config = backend_model.__calliope_run_config
solver = run_config["solver"]
solver_io = run_config.get("solver_io", None)
solver_options = run_config.get("solver_options", None)
save_logs = run_config.get("save_logs", None)
if build_only:
results = xr.Dataset()
else:
log_time(
logger,
timings,
"run_solver_start",
comment="Backend: sending model to solver",
)
results = backend.solve_model(
backend_model,
solver=solver,
solver_io=solver_io,
solver_options=solver_options,
save_logs=save_logs,
)
log_time(
logger,
timings,
"run_solver_exit",
time_since_run_start=True,
comment="Backend: solver finished running",
)
termination = backend.load_results(backend_model, results)
log_time(logger,
timings,
"run_results_loaded",
comment="Backend: loaded results")
results = backend.get_result_array(backend_model, model_data)
results.attrs["termination_condition"] = termination
if results.attrs["termination_condition"] in ["optimal", "feasible"]:
results.attrs["objective_function_value"] = backend_model.obj()
log_time(
logger,
timings,
"run_solution_returned",
time_since_run_start=True,
comment="Backend: generated solution array",
)
return results, backend_model
def run_operate(model_data, timings, backend, build_only):
"""
For use when mode is 'operate', to allow the model to be built, edited, and
iteratively run within Pyomo.
"""
log_time(timings, 'run_start',
comment='Backend: starting model run in operational mode')
defaults = ruamel.yaml.load(model_data.attrs['defaults'], Loader=ruamel.yaml.Loader)
operate_params = ['purchased'] + [
i.replace('_max', '') for i in defaults if i[-4:] == '_max'
]
# Capacity results (from plan mode) can be used as the input to operate mode
if (any(model_data.filter_by_attrs(is_result=1).data_vars) and
model_data.attrs.get('run.operation.use_cap_results', False)):
# Anything with is_result = 1 will be ignored in the Pyomo model
for varname, varvals in model_data.data_vars.items():
if varname in operate_params:
varvals.attrs['is_result'] = 1
varvals.attrs['operate_param'] = 1
else:
cap_max = xr.merge([
v.rename(k.replace('_max', ''))
for k, v in model_data.data_vars.items() if '_max' in k
])
cap_equals = xr.merge([
v.rename(k.replace('_equals', ''))
for k, v in model_data.data_vars.items() if '_equals' in k
])
caps = cap_max.update(cap_equals)
for cap in caps.data_vars.values():
cap.attrs['is_result'] = 1
cap.attrs['operate_param'] = 1
model_data.update(caps)
# Storage initial is carried over between iterations, so must be defined along with storage
if ('loc_techs_store' in model_data.dims.keys() and
'storage_initial' not in model_data.data_vars.keys()):
model_data['storage_initial'] = (
xr.DataArray([0 for loc_tech in model_data.loc_techs_store.values],
dims='loc_techs_store')
)
model_data['storage_initial'].attrs['is_result'] = 0
exceptions.ModelWarning(
'Initial stored energy not defined, set to zero for all '
'loc::techs in loc_techs_store, for use in iterative optimisation'
)
# Operated units is carried over between iterations, so must be defined in a milp model
if ('loc_techs_milp' in model_data.dims.keys() and
'operated_units' not in model_data.data_vars.keys()):
model_data['operated_units'] = (
xr.DataArray([0 for loc_tech in model_data.loc_techs_milp.values],
dims='loc_techs_milp')
)
model_data['operated_units'].attrs['is_result'] = 1
model_data['operated_units'].attrs['operate_param'] = 1
exceptions.ModelWarning(
'daily operated units not defined, set to zero for all '
'loc::techs in loc_techs_milp, for use in iterative optimisation'
)
comments, warnings, errors = checks.check_operate_params(model_data)
exceptions.print_warnings_and_raise_errors(warnings=warnings, errors=errors)
# Initialize our variables
solver = model_data.attrs['run.solver']
solver_io = model_data.attrs.get('run.solver_io', None)
solver_options = model_data.attrs.get('run.solver_options', None)
save_logs = model_data.attrs.get('run.save_logs', None)
window = model_data.attrs['run.operation.window']
horizon = model_data.attrs['run.operation.horizon']
window_to_horizon = horizon - window
# get the cumulative sum of timestep resolution, to find where we hit our window and horizon
timestep_cumsum = model_data.timestep_resolution.cumsum('timesteps').to_pandas()
# get the timesteps at which we start and end our windows
window_ends = timestep_cumsum.where(
(timestep_cumsum % window == 0) | (timestep_cumsum == timestep_cumsum[-1])
)
window_starts = timestep_cumsum.where(
(~np.isnan(window_ends.shift(1))) | (timestep_cumsum == timestep_cumsum[0])
).dropna()
window_ends = window_ends.dropna()
horizon_ends = timestep_cumsum[timestep_cumsum.isin(window_ends.values + window_to_horizon)]
if not any(window_starts):
raise exceptions.ModelError(
'Not enough timesteps or incorrect timestep resolution to run in '
'operational mode with an optimisation window of {}'.format(window)
)
# We will only update timseries parameters
timeseries_data_vars = [
k for k, v in model_data.data_vars.items() if 'timesteps' in v.dims
and v.attrs['is_result'] == 0
]
# Loop through each window, solve over the horizon length, and add result to
# result_array we only go as far as the end of the last horizon, which may
# clip the last bit of data
result_array = []
# track whether each iteration finds an optimal solution or not
terminations = []
if build_only:
iterations = [0]
else:
iterations = range(len(window_starts))
for i in iterations:
start_timestep = window_starts.index[i]
# Build full model in first instance
if i == 0:
warmstart = False
end_timestep = horizon_ends.index[i]
timesteps = slice(start_timestep, end_timestep)
window_model_data = model_data.loc[dict(timesteps=timesteps)]
log_time(
timings, 'model_gen_1',
comment='Backend: generating initial model'
)
backend_model = backend.generate_model(window_model_data)
# Build the full model in the last instance(s),
# where number of timesteps is less than the horizon length
elif i > len(horizon_ends) - 1:
warmstart = False
end_timestep = window_ends.index[i]
timesteps = slice(start_timestep, end_timestep)
window_model_data = model_data.loc[dict(timesteps=timesteps)]
log_time(
timings, 'model_gen_{}'.format(i + 1),
comment=(
'Backend: iteration {}: generating new model for '
'end of timeseries, with horizon = {} timesteps'
.format(i + 1, window_ends[i] - window_starts[i])
)
)
backend_model = backend.generate_model(window_model_data)
# Update relevent Pyomo Params in intermediate instances
else:
warmstart = True
end_timestep = horizon_ends.index[i]
timesteps = slice(start_timestep, end_timestep)
window_model_data = model_data.loc[dict(timesteps=timesteps)]
log_time(
timings, 'model_gen_{}'.format(i + 1),
comment='Backend: iteration {}: updating model parameters'.format(i + 1)
)
# Pyomo model sees the same timestamps each time, we just change the
# values associated with those timestamps
for var in timeseries_data_vars:
# New values
var_series = window_model_data[var].to_series().dropna().replace('inf', np.inf)
# Same timestamps
var_series.index = backend_model.__calliope_model_data__['data'][var].keys()
var_dict = var_series.to_dict()
# Update pyomo Param with new dictionary
for k, v in getattr(backend_model, var).items():
if k in var_dict:
v.set_value(var_dict[k])
if not build_only:
log_time(
timings, 'model_run_{}'.format(i + 1), time_since_start=True,
comment='Backend: iteration {}: sending model to solver'.format(i + 1)
)
# After iteration 1, warmstart = True, which should speed up the process
# Note: Warmstart isn't possible with GLPK (dealt with later on)
_results = backend.solve_model(
backend_model, solver=solver, solver_io=solver_io,
solver_options=solver_options, save_logs=save_logs, warmstart=warmstart,
)
log_time(
timings, 'run_solver_exit_{}'.format(i + 1), time_since_start=True,
comment='Backend: iteration {}: solver finished running'.format(i + 1)
)
# xarray dataset is built for each iteration
_termination = backend.load_results(backend_model, _results)
terminations.append(_termination)
_results = backend.get_result_array(backend_model, model_data)
# We give back the actual timesteps for this iteration and take a slice
# equal to the window length
_results['timesteps'] = window_model_data.timesteps.copy()
# We always save the window data. Until the last window(s) this will crop
# the window_to_horizon timesteps. In the last window(s), optimistion will
# only be occurring over a window length anyway
_results = _results.loc[dict(timesteps=slice(None, window_ends.index[i]))]
result_array.append(_results)
# Set up initial storage for the next iteration
if 'loc_techs_store' in model_data.dims.keys():
storage_initial = _results.storage.loc[{'timesteps': window_ends.index[i]}].drop('timesteps')
model_data['storage_initial'].loc[storage_initial.coords] = storage_initial.values
for k, v in backend_model.storage_initial.items():
v.set_value(storage_initial.to_series().dropna().to_dict()[k])
# Set up total operated units for the next iteration
if 'loc_techs_milp' in model_data.dims.keys():
operated_units = _results.operating_units.sum('timesteps').astype(np.int)
model_data['operated_units'].loc[{}] += operated_units.values
for k, v in backend_model.operated_units.items():
v.set_value(operated_units.to_series().dropna().to_dict()[k])
log_time(
timings, 'run_solver_exit_{}'.format(i + 1), time_since_start=True,
comment='Backend: iteration {}: generated solution array'.format(i + 1)
)
if build_only:
results = xr.Dataset()
else:
# Concatenate results over the timestep dimension to get a single
# xarray Dataset of interest
results = xr.concat(result_array, dim='timesteps')
if all(i == 'optimal' for i in terminations):
results.attrs['termination_condition'] = 'optimal'
else:
results.attrs['termination_condition'] = ','.join(terminations)
log_time(
timings, 'run_solution_returned', time_since_start=True,
comment='Backend: generated full solution array'
)
return results, backend_model