class Model(object):
"""
A Calliope Model.
"""
def __init__(self, config, model_data=None, debug=False, *args, **kwargs):
"""
Returns a new Model from either the path to a YAML model
configuration file or a dict fully specifying the model.
Parameters
----------
config : str or dict or AttrDict
If str, must be the path to a model configuration file.
If dict or AttrDict, must fully specify the model.
model_data : Dataset, optional
Create a Model instance from a fully built model_data Dataset.
This is only used if `config` is explicitly set to None
and is primarily used to re-create a Model instance from
a model previously saved to a NetCDF file.
"""
self._timings = {}
# try to set logging output format assuming python interactive. Will
# use CLI logging format if model called from CLI
log_time(logger,
self._timings,
"model_creation",
comment="Model: initialising")
if isinstance(config, str):
model_run, debug_data = model_run_from_yaml(
config, *args, **kwargs)
self._init_from_model_run(model_run, debug_data, debug)
elif isinstance(config, dict):
model_run, debug_data = model_run_from_dict(
config, *args, **kwargs)
self._init_from_model_run(model_run, debug_data, debug)
elif model_data is not None and config is None:
self._init_from_model_data(model_data)
else:
# expected input is a string pointing to a YAML file of the run
# configuration or a dict/AttrDict in which the run and model
# configurations are defined
raise ValueError(
"Input configuration must either be a string or a dictionary.")
self._check_future_deprecation_warnings()
self.plot = plotting.ModelPlotMethods(self)
def _init_from_model_run(self, model_run, debug_data, debug):
self._model_run = model_run
log_time(
logger,
self._timings,
"model_run_creation",
comment="Model: preprocessing stage 1 (model_run)",
)
model_data_factory = ModelDataFactory(model_run)
(
model_data_pre_clustering,
model_data,
data_pre_time,
stripped_keys,
) = model_data_factory()
self._model_data_pre_clustering = model_data_pre_clustering
self._model_data = model_data
if debug:
self._debug_data = debug_data
self._model_data_pre_time = data_pre_time
self._model_data_stripped_keys = stripped_keys
self.inputs = self._model_data.filter_by_attrs(is_result=0)
log_time(
logger,
self._timings,
"model_data_original_creation",
comment="Model: preprocessing stage 2 (model_data)",
)
# Ensure model and run attributes of _model_data update themselves
model_config = {
k: v
for k, v in model_run.get("model", {}).items()
if k != "file_allowed"
}
self.model_config = UpdateObserverDict(initial_dict=model_config,
name="model_config",
observer=self._model_data)
self.run_config = UpdateObserverDict(
initial_dict=model_run.get("run", {}),
name="run_config",
observer=self._model_data,
)
self.subsets = UpdateObserverDict(
initial_dict=model_run.get("subsets").as_dict_flat(),
name="subsets",
observer=self._model_data,
)
log_time(
logger,
self._timings,
"model_data_creation",
comment="Model: preprocessing complete",
)
def _init_from_model_data(self, model_data):
if "_model_run" in model_data.attrs:
self._model_run = AttrDict.from_yaml_string(
model_data.attrs["_model_run"])
del model_data.attrs["_model_run"]
if "_debug_data" in model_data.attrs:
self._debug_data = AttrDict.from_yaml_string(
model_data.attrs["_debug_data"])
del model_data.attrs["_debug_data"]
self._model_data = model_data
self.inputs = self._model_data.filter_by_attrs(is_result=0)
self.model_config = UpdateObserverDict(
initial_yaml_string=model_data.attrs.get("model_config", "{}"),
name="model_config",
observer=self._model_data,
)
self.run_config = UpdateObserverDict(
initial_yaml_string=model_data.attrs.get("run_config", "{}"),
name="run_config",
observer=self._model_data,
)
self.subsets = UpdateObserverDict(
initial_yaml_string=model_data.attrs.get("subsets", "{}"),
name="subsets",
observer=self._model_data,
flat=True,
)
results = self._model_data.filter_by_attrs(is_result=1)
if len(results.data_vars) > 0:
self.results = results
log_time(
logger,
self._timings,
"model_data_loaded",
comment="Model: loaded model_data",
)
def run(self, force_rerun=False, **kwargs):
"""
Run the model. If ``force_rerun`` is True, any existing results
will be overwritten.
Additional kwargs are passed to the backend.
"""
# Check that results exist and are non-empty
if hasattr(self,
"results") and self.results.data_vars and not force_rerun:
raise exceptions.ModelError(
"This model object already has results. "
"Use model.run(force_rerun=True) to force"
"the results to be overwritten with a new run.")
if (self.run_config["mode"] == "operate"
and not self._model_data.attrs["allow_operate_mode"]):
raise exceptions.ModelError(
"Unable to run this model in operational mode, probably because "
"there exist non-uniform timesteps (e.g. from time masking)")
results, self._backend_model, interface = run_backend(
self._model_data, self._timings, **kwargs)
# Add additional post-processed result variables to results
if results.attrs.get("termination_condition",
None) in ["optimal", "feasible"]:
results = postprocess_results.postprocess_model_results(
results, self._model_data, self._timings)
for var in results.data_vars:
results[var].attrs["is_result"] = 1
self._model_data.update(results)
self._model_data.attrs.update(results.attrs)
self.results = self._model_data.filter_by_attrs(is_result=1)
self.backend = interface(self)
def get_formatted_array(self, var):
"""
Return an xr.DataArray with nodes, techs, and carriers as
separate dimensions.
Parameters
----------
var : str
Decision variable for which to return a DataArray.
"""
warnings.warn(
"get_formatted_array() is deprecated and will be removed in a "
"future version. Use `model.results.variable` instead.",
DeprecationWarning,
)
if var not in self._model_data.data_vars:
raise KeyError("Variable {} not in Model data".format(var))
return self._model_data[var]
def to_netcdf(self, path):
"""
Save complete model data (inputs and, if available, results)
to a NetCDF file at the given ``path``.
"""
io.save_netcdf(self._model_data, path, model=self)
def to_csv(self, path, dropna=True):
"""
Save complete model data (inputs and, if available, results)
as a set of CSV files to the given ``path``.
Parameters
----------
dropna : bool, optional
If True (default), NaN values are dropped when saving,
resulting in significantly smaller CSV files.
"""
io.save_csv(self._model_data, path, dropna)
def to_lp(self, path):
"""
Save built model to LP format at the given ``path``. If the backend
model has not been built yet, it is built prior to saving.
"""
io.save_lp(self, path)
def info(self):
info_strings = []
model_name = self.model_config.get("name", "None")
info_strings.append("Model name: {}".format(model_name))
msize = "{nodes} nodes, {techs} technologies, {times} timesteps".format(
nodes=len(self._model_data.coords.get("nodes", [])),
techs=(
len(self._model_data.coords.get("techs_non_transmission",
[])) +
len(self._model_data.coords.get("techs_transmission_names",
[]))),
times=len(self._model_data.coords.get("timesteps", [])),
)
info_strings.append("Model size: {}".format(msize))
return "\n".join(info_strings)
def _check_future_deprecation_warnings(self):
"""
def run_operate(model_data, timings, backend, build_only):
"""
For use when mode is 'operate', to allow the model to be built, edited, and
iteratively run within Pyomo.
"""
log_time(
logger,
timings,
"run_start",
comment="Backend: starting model run in operational mode",
)
defaults = UpdateObserverDict(
initial_yaml_string=model_data.attrs["defaults"],
name="defaults",
observer=model_data,
)
run_config = UpdateObserverDict(
initial_yaml_string=model_data.attrs["run_config"],
name="run_config",
observer=model_data,
)
# New param defaults = old maximum param defaults (e.g. energy_cap gets default from energy_cap_max)
operate_params = {
k.replace("_max", ""): v
for k, v in defaults.items() if k.endswith("_max")
}
operate_params[
"purchased"] = 0 # no _max to work from here, so we hardcode a default
defaults.update(operate_params)
# Capacity results (from plan mode) can be used as the input to operate mode
if any(model_data.filter_by_attrs(
is_result=1).data_vars) and run_config.get(
"operation.use_cap_results", False):
# Anything with is_result = 1 will be ignored in the Pyomo model
for varname, varvals in model_data.data_vars.items():
if varname in operate_params.keys():
varvals.attrs["is_result"] = 1
varvals.attrs["operate_param"] = 1
else:
cap_max = xr.merge([
v.rename(k.replace("_max", ""))
for k, v in model_data.data_vars.items() if "_max" in k
])
cap_equals = xr.merge([
v.rename(k.replace("_equals", ""))
for k, v in model_data.data_vars.items() if "_equals" in k
])
caps = cap_max.update(cap_equals)
for cap in caps.data_vars.values():
cap.attrs["is_result"] = 1
cap.attrs["operate_param"] = 1
model_data.update(caps)
comments, warnings, errors = checks.check_operate_params(model_data)
exceptions.print_warnings_and_raise_errors(warnings=warnings,
errors=errors)
# Initialize our variables
solver = run_config["solver"]
solver_io = run_config.get("solver_io", None)
solver_options = run_config.get("solver_options", None)
save_logs = run_config.get("save_logs", None)
window = run_config["operation"]["window"]
horizon = run_config["operation"]["horizon"]
window_to_horizon = horizon - window
# get the cumulative sum of timestep resolution, to find where we hit our window and horizon
timestep_cumsum = model_data.timestep_resolution.cumsum(
"timesteps").to_pandas()
# get the timesteps at which we start and end our windows
window_ends = timestep_cumsum.where((timestep_cumsum % window == 0) | (
timestep_cumsum == timestep_cumsum[-1]))
window_starts = timestep_cumsum.where((~np.isnan(window_ends.shift(1))) | (
timestep_cumsum == timestep_cumsum[0])).dropna()
window_ends = window_ends.dropna()
horizon_ends = timestep_cumsum[timestep_cumsum.isin(window_ends.values +
window_to_horizon)]
if not any(window_starts):
raise exceptions.ModelError(
"Not enough timesteps or incorrect timestep resolution to run in "
"operational mode with an optimisation window of {}".format(
window))
# We will only update timseries parameters
timeseries_data_vars = [
k for k, v in model_data.data_vars.items()
if "timesteps" in v.dims and v.attrs["is_result"] == 0
]
# Loop through each window, solve over the horizon length, and add result to
# result_array we only go as far as the end of the last horizon, which may
# clip the last bit of data
result_array = []
# track whether each iteration finds an optimal solution or not
terminations = []
if build_only:
iterations = [0]
else:
iterations = range(len(window_starts))
for i in iterations:
start_timestep = window_starts.index[i]
# Build full model in first instance
if i == 0:
warmstart = False
end_timestep = horizon_ends.index[i]
timesteps = slice(start_timestep, end_timestep)
window_model_data = model_data.loc[dict(timesteps=timesteps)]
log_time(
logger,
timings,
"model_gen_1",
comment="Backend: generating initial model",
)
backend_model = backend.generate_model(window_model_data)
# Build the full model in the last instance(s),
# where number of timesteps is less than the horizon length
elif i > len(horizon_ends) - 1:
warmstart = False
end_timestep = window_ends.index[i]
timesteps = slice(start_timestep, end_timestep)
window_model_data = model_data.loc[dict(timesteps=timesteps)]
log_time(
logger,
timings,
"model_gen_{}".format(i + 1),
comment=(
"Backend: iteration {}: generating new model for "
"end of timeseries, with horizon = {} timesteps".format(
i + 1, window_ends[i] - window_starts[i])),
)
backend_model = backend.generate_model(window_model_data)
# Update relevent Pyomo Params in intermediate instances
else:
warmstart = True
end_timestep = horizon_ends.index[i]
timesteps = slice(start_timestep, end_timestep)
window_model_data = model_data.loc[dict(timesteps=timesteps)]
log_time(
logger,
timings,
"model_gen_{}".format(i + 1),
comment="Backend: iteration {}: updating model parameters".
format(i + 1),
)
# Pyomo model sees the same timestamps each time, we just change the
# values associated with those timestamps
for var in timeseries_data_vars:
# New values
var_series = (
window_model_data[var].to_series().dropna().replace(
"inf", np.inf))
# Same timestamps
var_series.index = backend_model.__calliope_model_data["data"][
var].keys()
var_dict = var_series.to_dict()
# Update pyomo Param with new dictionary
getattr(backend_model, var).store_values(var_dict)
if not build_only:
log_time(
logger,
timings,
"model_run_{}".format(i + 1),
time_since_run_start=True,
comment="Backend: iteration {}: sending model to solver".
format(i + 1),
)
# After iteration 1, warmstart = True, which should speed up the process
# Note: Warmstart isn't possible with GLPK (dealt with later on)
_results = backend.solve_model(
backend_model,
solver=solver,
solver_io=solver_io,
solver_options=solver_options,
save_logs=save_logs,
warmstart=warmstart,
)
log_time(
logger,
timings,
"run_solver_exit_{}".format(i + 1),
time_since_run_start=True,
comment="Backend: iteration {}: solver finished running".
format(i + 1),
)
# xarray dataset is built for each iteration
_termination = backend.load_results(backend_model, _results)
terminations.append(_termination)
_results = backend.get_result_array(backend_model, model_data)
# We give back the actual timesteps for this iteration and take a slice
# equal to the window length
_results["timesteps"] = window_model_data.timesteps.copy()
# We always save the window data. Until the last window(s) this will crop
# the window_to_horizon timesteps. In the last window(s), optimistion will
# only be occurring over a window length anyway
_results = _results.loc[dict(
timesteps=slice(None, window_ends.index[i]))]
result_array.append(_results)
# Set up initial storage for the next iteration
if "loc_techs_store" in model_data.dims.keys():
storage_initial = _results.storage.loc[{
"timesteps":
window_ends.index[i]
}].drop("timesteps")
model_data["storage_initial"].loc[
storage_initial.coords] = storage_initial.values
backend_model.storage_initial.store_values(
storage_initial.to_series().dropna().to_dict())
# Set up total operated units for the next iteration
if "loc_techs_milp" in model_data.dims.keys():
operated_units = _results.operating_units.sum(
"timesteps").astype(np.int)
model_data["operated_units"].loc[{}] += operated_units.values
backend_model.operated_units.store_values(
operated_units.to_series().dropna().to_dict())
log_time(
logger,
timings,
"run_solver_exit_{}".format(i + 1),
time_since_run_start=True,
comment="Backend: iteration {}: generated solution array".
format(i + 1),
)
if build_only:
results = xr.Dataset()
else:
# Concatenate results over the timestep dimension to get a single
# xarray Dataset of interest
results = xr.concat(result_array, dim="timesteps")
if all(i == "optimal" for i in terminations):
results.attrs["termination_condition"] = "optimal"
elif all(i in ["optimal", "feasible"] for i in terminations):
results.attrs["termination_condition"] = "feasible"
else:
results.attrs["termination_condition"] = ",".join(terminations)
log_time(
logger,
timings,
"run_solution_returned",
time_since_run_start=True,
comment="Backend: generated full solution array",
)
return results, backend_model
def check_operate_params(model_data):
"""
if model mode = `operate`, check for clashes in capacity constraints.
In this mode, all capacity constraints are set to parameters in the backend,
so can easily lead to model infeasibility if not checked.
Returns
-------
comments : AttrDict
debug output
warnings : list
possible problems that do not prevent the model run
from continuing
errors : list
serious issues that should raise a ModelError
"""
defaults = UpdateObserverDict(
initial_yaml_string=model_data.attrs["defaults"],
name="defaults",
observer=model_data,
)
run_config = UpdateObserverDict(
initial_yaml_string=model_data.attrs["run_config"],
name="run_config",
observer=model_data,
)
warnings, errors = [], []
comments = AttrDict()
def _get_param(loc_tech, var):
if (_is_in(loc_tech, var)
and not model_data[var].loc[loc_tech].isnull().any()):
param = model_data[var].loc[loc_tech].values
else:
param = defaults[var]
return param
def _is_in(loc_tech, set_or_var):
try:
model_data[set_or_var].loc[loc_tech]
return True
except (KeyError, AttributeError):
return False
def _set_inf_and_warn(loc_tech, var, warnings, warning_text):
if np.isinf(model_data[var].loc[loc_tech].item()):
return (np.inf, warnings)
elif model_data[var].loc[loc_tech].isnull().item():
var_name = model_data[var].loc[loc_tech] = np.inf
return (var_name, warnings)
else:
var_name = model_data[var].loc[loc_tech] = np.inf
warnings.append(warning_text)
return var_name, warnings
# Storage initial is carried over between iterations, so must be defined along with storage
if ("loc_techs_store" in model_data.dims.keys()
and "storage_initial" not in model_data.data_vars.keys()):
model_data["storage_initial"] = xr.DataArray(
[0.0 for loc_tech in model_data.loc_techs_store.values],
dims="loc_techs_store",
)
model_data["storage_initial"].attrs["is_result"] = 0.0
warnings.append(
"Initial stored energy not defined, set to zero for all "
"loc::techs in loc_techs_store, for use in iterative optimisation")
# Operated units is carried over between iterations, so must be defined in a milp model
if ("loc_techs_milp" in model_data.dims.keys()
and "operated_units" not in model_data.data_vars.keys()):
model_data["operated_units"] = xr.DataArray(
[0 for loc_tech in model_data.loc_techs_milp.values],
dims="loc_techs_milp",
)
model_data["operated_units"].attrs["is_result"] = 1
model_data["operated_units"].attrs["operate_param"] = 1
warnings.append(
"daily operated units not defined, set to zero for all "
"loc::techs in loc_techs_milp, for use in iterative optimisation")
for loc_tech in model_data.loc_techs.values:
energy_cap = model_data.energy_cap.loc[loc_tech].item()
# Must have energy_cap defined for all relevant techs in the model
if (np.isinf(energy_cap) or np.isnan(energy_cap)) and (
_get_param(loc_tech, "energy_cap_min_use") or
(_get_param(loc_tech, "force_resource")
and _get_param(loc_tech, "resource_unit") == "energy_per_cap")):
errors.append(
"Operate mode: User must define a finite energy_cap (via "
"energy_cap_equals or energy_cap_max) for {}".format(loc_tech))
elif _is_in(loc_tech, "loc_techs_finite_resource"):
# Cannot have infinite resource area if linking resource and area (resource_unit = energy_per_area)
if (_is_in(loc_tech, "loc_techs_area")
and model_data.resource_unit.loc[loc_tech].item()
== "energy_per_area"):
if _is_in(loc_tech, "resource_area"):
area = model_data.resource_area.loc[loc_tech].item()
else:
area = None
if pd.isnull(area) or np.isinf(area):
errors.append(
"Operate mode: User must define a finite resource_area "
"(via resource_area_equals or resource_area_max) for {}, "
"as available resource is linked to resource_area "
"(resource_unit = `energy_per_area`)".format(loc_tech))
# force resource overrides capacity constraints, so set capacity constraints to infinity
if _get_param(loc_tech, "force_resource"):
if not _is_in(loc_tech, "loc_techs_store"):
# set resource_area to inf if the resource is linked to energy_cap using energy_per_cap
if (model_data.resource_unit.loc[loc_tech].item() ==
"energy_per_cap"):
if _is_in(loc_tech, "resource_area"):
resource_area, warnings = _set_inf_and_warn(
loc_tech,
"resource_area",
warnings,
"Resource area constraint removed from {} as "
"force_resource is applied and resource is linked "
"to energy flow using `energy_per_cap`".format(
loc_tech),
)
# set energy_cap to inf if the resource is linked to resource_area using energy_per_area
elif (model_data.resource_unit.loc[loc_tech].item() ==
"energy_per_area"):
energy_cap, warnings = _set_inf_and_warn(
loc_tech,
"energy_cap",
warnings,
"Energy capacity constraint removed from {} as "
"force_resource is applied and resource is linked "
"to energy flow using `energy_per_area`".format(
loc_tech),
)
# set both energy_cap and resource_area to inf if the resource is not linked to anything
elif (model_data.resource_unit.loc[loc_tech].item() ==
"energy"):
if _is_in(loc_tech, "resource_area"):
resource_area, warnings = _set_inf_and_warn(
loc_tech,
"resource_area",
warnings,
"Resource area constraint removed from {} as "
"force_resource is applied and resource is not linked "
"to energy flow (resource_unit = `energy`)".
format(loc_tech),
)
energy_cap, warnings = _set_inf_and_warn(
loc_tech,
"energy_cap",
warnings,
"Energy capacity constraint removed from {} as "
"force_resource is applied and resource is not linked "
"to energy flow (resource_unit = `energy`)".format(
loc_tech),
)
if _is_in(loc_tech, "resource_cap"):
resource_cap, warnings = _set_inf_and_warn(
loc_tech,
"resource_cap",
warnings,
"Resource capacity constraint removed from {} as "
"force_resource is applied".format(loc_tech),
)
if _is_in(loc_tech, "loc_techs_store"):
storage_cap = model_data.storage_cap.loc[loc_tech].item()
if not pd.isnull(storage_cap) and not pd.isnull(energy_cap):
if _get_param(loc_tech, "charge_rate") is not False:
charge_rate = (
model_data["charge_rate"].loc[loc_tech].item())
if storage_cap * charge_rate < energy_cap:
errors.append(
"fixed storage capacity * charge_rate is not larger "
"than fixed energy capacity for loc::tech {}".
format(loc_tech))
if (_get_param(loc_tech, "energy_cap_per_storage_cap_max")
is not False):
energy_cap_per_storage_cap_max = (
model_data["energy_cap_per_storage_cap_max"].
loc[loc_tech].item())
if (storage_cap * energy_cap_per_storage_cap_max <
energy_cap):
errors.append(
"fixed storage capacity * energy_cap_per_storage_cap_max is not larger "
"than fixed energy capacity for loc::tech {}".
format(loc_tech))
if (_get_param(loc_tech, "energy_cap_per_storage_cap_min")
is not False):
energy_cap_per_storage_cap_min = (
model_data["energy_cap_per_storage_cap_min"].
loc[loc_tech].item())
if (storage_cap * energy_cap_per_storage_cap_min >
energy_cap):
errors.append(
"fixed storage capacity * energy_cap_per_storage_cap_min is not smaller "
"than fixed energy capacity for loc::tech {}".
format(loc_tech))
# Must define a resource capacity to ensure the Pyomo param is created
# for it. But we just create an array of infs, so the capacity has no effect
if ("resource_cap" not in model_data.data_vars.keys()
and "loc_techs_supply_plus" in model_data.dims.keys()):
model_data["resource_cap"] = xr.DataArray(
[np.inf for i in model_data.loc_techs_supply_plus.values],
dims="loc_techs_supply_plus",
)
model_data["resource_cap"].attrs["is_result"] = 1
model_data["resource_cap"].attrs["operate_param"] = 1
warnings.append(
"Resource capacity constraint defined and set to infinity "
"for all supply_plus techs")
window = run_config.get("operation", {}).get("window", None)
horizon = run_config.get("operation", {}).get("horizon", None)
if not window or not horizon:
errors.append(
"Operational mode requires a timestep window and horizon to be "
"defined under run.operation")
elif horizon < window:
errors.append(
"Iteration horizon must be larger than iteration window, "
"for operational mode")
# Cyclic storage isn't really valid in operate mode, so we ignore it, using
# initial_storage instead (allowing us to pass storage between operation windows)
if run_config.get("cyclic_storage", True):
warnings.append(
"Storage cannot be cyclic in operate run mode, setting "
"`run.cyclic_storage` to False for this run")
run_config["cyclic_storage"] = False
if "group_demand_share_per_timestep_decision" in model_data.data_vars:
warnings.append(
"`demand_share_per_timestep_decision` group constraints cannot be "
"used in operate mode, so will not be built.")
del model_data["group_demand_share_per_timestep_decision"]
## TODO: this comes from preprocess checks so might not work out of the box here
# Check if we're allowed to use operate mode
if "allow_operate_mode" not in model_data.attrs.keys():
daily_timesteps = [
model_data.timestep_resolution.loc[i].values for i in np.unique(
model_data.timesteps.to_index().strftime("%Y-%m-%d"))
]
if not np.all(daily_timesteps == daily_timesteps[0]):
model_data.attrs["allow_operate_mode"] = 0
warnings.append(
"Operational mode requires the same timestep resolution profile "
"to be emulated on each date")
else:
model_data.attrs["allow_operate_mode"] = 1
return comments, warnings, errors
def check_operate_params(model_data):
"""
if model mode = `operate`, check for clashes in capacity constraints.
In this mode, all capacity constraints are set to parameters in the backend,
so can easily lead to model infeasibility if not checked.
Returns
-------
comments : AttrDict
debug output
warnings : list
possible problems that do not prevent the model run
from continuing
errors : list
serious issues that should raise a ModelError
"""
defaults = UpdateObserverDict(
initial_yaml_string=model_data.attrs['defaults'],
name='defaults',
observer=model_data)
run_config = UpdateObserverDict(
initial_yaml_string=model_data.attrs['run_config'],
name='run_config',
observer=model_data)
warnings, errors = [], []
comments = AttrDict()
def _get_param(loc_tech, var):
if _is_in(loc_tech, var) and not any((pd.isnull(
(model_data[var].loc[loc_tech].values, )), )):
param = model_data[var].loc[loc_tech].values
else:
param = defaults[var]
return param
def _is_in(loc_tech, set_or_var):
try:
model_data[set_or_var].loc[loc_tech]
return True
except (KeyError, AttributeError):
return False
for loc_tech in model_data.loc_techs.values:
energy_cap = model_data.energy_cap.loc[loc_tech].item()
# Must have energy_cap defined for all relevant techs in the model
if (pd.isnull(energy_cap) or np.isinf(energy_cap)) and not _is_in(
loc_tech, 'force_resource'):
errors.append(
'Operate mode: User must define a finite energy_cap (via '
'energy_cap_equals or energy_cap_max) for {}'.format(loc_tech))
elif _is_in(loc_tech, 'loc_techs_finite_resource'):
# force resource overrides capacity constraints, so set capacity constraints to infinity
if _is_in(loc_tech, 'force_resource'):
if not _is_in(loc_tech, 'loc_techs_store'):
energy_cap = model_data.energy_cap.loc[loc_tech] = np.inf
warnings.append(
'Energy capacity constraint removed from {} as '
'force_resource is applied'.format(loc_tech))
if _is_in(loc_tech, 'resource_cap'):
print(loc_tech,
model_data.resource_cap.loc_techs_supply_plus)
model_data.resource_cap.loc[loc_tech] = np.inf
warnings.append(
'Resource capacity constraint removed from {} as '
'force_resource is applied'.format(loc_tech))
# Cannot have infinite resource area (physically impossible)
if _is_in(loc_tech, 'loc_techs_area'):
area = model_data.resource_area.loc[loc_tech].item()
if pd.isnull(area) or np.isinf(area):
errors.append(
'Operate mode: User must define a finite resource_area '
'(via resource_area_equals or resource_area_max) for {}, '
'as a finite available resource is considered'.format(
loc_tech))
# Cannot have consumed resource being higher than energy_cap, as
# constraints will clash. Doesn't affect supply_plus techs with a
# storage buffer prior to carrier production.
elif not _is_in(loc_tech, 'loc_techs_store'):
resource_scale = _get_param(loc_tech, 'resource_scale')
energy_cap_scale = _get_param(loc_tech, 'energy_cap_scale')
resource_eff = _get_param(loc_tech, 'resource_eff')
energy_eff = _get_param(loc_tech, 'energy_eff')
resource = model_data.resource.loc[loc_tech].values
if (energy_cap is not None and any(
resource * resource_scale * resource_eff > energy_cap *
energy_cap_scale * energy_eff)):
errors.append(
'Operate mode: resource is forced to be higher than '
'fixed energy cap for `{}`'.format(loc_tech))
if _is_in(loc_tech, 'loc_techs_store'):
if _is_in(loc_tech, 'charge_rate'):
storage_cap = model_data.storage_cap.loc[loc_tech].item()
if storage_cap and energy_cap:
charge_rate = model_data['charge_rate'].loc[loc_tech].item(
)
if storage_cap * charge_rate < energy_cap:
errors.append(
'fixed storage capacity * charge rate is not larger '
'than fixed energy capacity for loc::tech {}'.
format(loc_tech))
if _is_in(loc_tech, 'loc_techs_store'):
if _is_in(loc_tech, 'energy_cap_per_storage_cap_max'):
storage_cap = model_data.storage_cap.loc[loc_tech].item()
if storage_cap and energy_cap:
energy_cap_per_storage_cap_max = model_data[
'energy_cap_per_storage_cap_max'].loc[loc_tech].item()
if storage_cap * energy_cap_per_storage_cap_max < energy_cap:
errors.append(
'fixed storage capacity * energy_cap_per_storage_cap_max is not larger '
'than fixed energy capacity for loc::tech {}'.
format(loc_tech))
elif _is_in(loc_tech, 'energy_cap_per_storage_cap_min'):
storage_cap = model_data.storage_cap.loc[loc_tech].item()
if storage_cap and energy_cap:
energy_cap_per_storage_cap_min = model_data[
'energy_cap_per_storage_cap_min'].loc[loc_tech].item()
if storage_cap * energy_cap_per_storage_cap_min > energy_cap:
errors.append(
'fixed storage capacity * energy_cap_per_storage_cap_min is not smaller '
'than fixed energy capacity for loc::tech {}'.
format(loc_tech))
# Must define a resource capacity to ensure the Pyomo param is created
# for it. But we just create an array of infs, so the capacity has no effect
if ('resource_cap' not in model_data.data_vars.keys()
and 'loc_techs_supply_plus' in model_data.dims.keys()):
model_data['resource_cap'] = xr.DataArray(
[np.inf for i in model_data.loc_techs_supply_plus.values],
dims='loc_techs_supply_plus')
model_data['resource_cap'].attrs['is_result'] = 1
model_data['resource_cap'].attrs['operate_param'] = 1
warnings.append(
'Resource capacity constraint defined and set to infinity '
'for all supply_plus techs')
window = run_config.get('operation', {}).get('window', None)
horizon = run_config.get('operation', {}).get('horizon', None)
if not window or not horizon:
errors.append(
'Operational mode requires a timestep window and horizon to be '
'defined under run.operation')
elif horizon < window:
errors.append(
'Iteration horizon must be larger than iteration window, '
'for operational mode')
# Cyclic storage isn't really valid in operate mode, so we ignore it, using
# initial_storage instead (allowing us to pass storage between operation windows)
if run_config.get('cyclic_storage', True):
warnings.append(
'Storage cannot be cyclic in operate run mode, setting '
'`run.cyclic_storage` to False for this run')
run_config['cyclic_storage'] = False
if 'group_demand_share_per_timestep_decision' in model_data.data_vars:
warnings.append(
'`demand_share_per_timestep_decision` group constraints cannot be '
'used in operate mode, so will not be built.')
del model_data['group_demand_share_per_timestep_decision']
return comments, warnings, errors
class Model(object):
"""
A Calliope Model.
"""
def __init__(self, config, model_data=None, *args, **kwargs):
"""
Returns a new Model from either the path to a YAML model
configuration file or a dict fully specifying the model.
Parameters
----------
config : str or dict or AttrDict
If str, must be the path to a model configuration file.
If dict or AttrDict, must fully specify the model.
model_data : Dataset, optional
Create a Model instance from a fully built model_data Dataset.
This is only used if `config` is explicitly set to None
and is primarily used to re-create a Model instance from
a model previously saved to a NetCDF file.
"""
self._timings = {}
# try to set logging output format assuming python interactive. Will
# use CLI logging format if model called from CLI
log_time(logger,
self._timings,
'model_creation',
comment='Model: initialising')
if isinstance(config, str):
model_run, debug_data = model_run_from_yaml(
config, *args, **kwargs)
self._init_from_model_run(model_run, debug_data)
elif isinstance(config, dict):
model_run, debug_data = model_run_from_dict(
config, *args, **kwargs)
self._init_from_model_run(model_run, debug_data)
elif model_data is not None and config is None:
self._init_from_model_data(model_data)
else:
# expected input is a string pointing to a YAML file of the run
# configuration or a dict/AttrDict in which the run and model
# configurations are defined
raise ValueError(
'Input configuration must either be a string or a dictionary.')
self._check_future_deprecation_warnings()
self.plot = plotting.ModelPlotMethods(self)
def _init_from_model_run(self, model_run, debug_data):
self._model_run = model_run
self._debug_data = debug_data
log_time(logger,
self._timings,
'model_run_creation',
comment='Model: preprocessing stage 1 (model_run)')
self._model_data_original = build_model_data(model_run)
log_time(logger,
self._timings,
'model_data_original_creation',
comment='Model: preprocessing stage 2 (model_data)')
random_seed = self._model_run.get_key('model.random_seed', None)
if random_seed:
np.random.seed(seed=random_seed)
# After setting the random seed, time clustering can take place
time_config = model_run.model.get('time', None)
if not time_config:
_model_data = self._model_data_original
else:
_model_data = apply_time_clustering(self._model_data_original,
model_run)
self._model_data = final_timedimension_processing(_model_data)
log_time(logger,
self._timings,
'model_data_creation',
comment='Model: preprocessing complete')
# Ensure model and run attributes of _model_data update themselves
for var in self._model_data.data_vars:
self._model_data[var].attrs['is_result'] = 0
self.inputs = self._model_data.filter_by_attrs(is_result=0)
model_config = {
k: v
for k, v in model_run.get('model', {}).items()
if k != 'file_allowed'
}
self.model_config = UpdateObserverDict(initial_dict=model_config,
name='model_config',
observer=self._model_data)
self.run_config = UpdateObserverDict(initial_dict=model_run.get(
'run', {}),
name='run_config',
observer=self._model_data)
def _init_from_model_data(self, model_data):
if '_model_run' in model_data.attrs:
self._model_run = AttrDict.from_yaml_string(
model_data.attrs['_model_run'])
del model_data.attrs['_model_run']
if '_debug_data' in model_data.attrs:
self._debug_data = AttrDict.from_yaml_string(
model_data.attrs['_debug_data'])
del model_data.attrs['_debug_data']
self._model_data = model_data
self.inputs = self._model_data.filter_by_attrs(is_result=0)
self.model_config = UpdateObserverDict(
initial_yaml_string=model_data.attrs.get('model_config', '{}'),
name='model_config',
observer=self._model_data)
self.run_config = UpdateObserverDict(
initial_yaml_string=model_data.attrs.get('run_config', '{}'),
name='run_config',
observer=self._model_data)
results = self._model_data.filter_by_attrs(is_result=1)
if len(results.data_vars) > 0:
self.results = results
log_time(logger,
self._timings,
'model_data_loaded',
comment='Model: loaded model_data')
def save_commented_model_yaml(self, path):
"""
Save a fully built and commented version of the model to a YAML file
at the given ``path``. Comments in the file indicate where values
were overridden. This is Calliope's internal representation of
a model directly before the model_data xarray.Dataset is built,
and can be useful for debugging possible issues in the model
formulation.
"""
if not self._model_run or not self._debug_data:
raise KeyError(
'This model does not have the fully built model attached, '
'so `save_commented_model_yaml` is not available. Likely '
'reason is that the model was built with a verion of Calliope '
'prior to 0.6.5.')
yaml = ruamel_yaml.YAML()
model_run_debug = self._model_run.copy()
try:
del model_run_debug['timeseries_data'] # Can't be serialised!
except KeyError:
# Possible that timeseries_data is already gone if the model
# was read from a NetCDF file
pass
# Turn sets in model_run into lists for YAML serialization
for k, v in model_run_debug.sets.items():
model_run_debug.sets[k] = list(v)
debug_comments = self._debug_data['comments']
stream = StringIO()
yaml.dump(model_run_debug.as_dict(), stream=stream)
debug_yaml = yaml.load(stream.getvalue())
for k in debug_comments.model_run.keys_nested():
v = debug_comments.model_run.get_key(k)
if v:
keys = k.split('.')
apply_to_dict(debug_yaml, keys[:-1], 'yaml_add_eol_comment',
(v, keys[-1]))
dumper = ruamel_yaml.dumper.RoundTripDumper
dumper.ignore_aliases = lambda self, data: True
with open(path, 'w') as f:
ruamel_yaml.dump(debug_yaml,
stream=f,
Dumper=dumper,
default_flow_style=False)
def run(self, force_rerun=False, **kwargs):
"""
Run the model. If ``force_rerun`` is True, any existing results
will be overwritten.
Additional kwargs are passed to the backend.
"""
# Check that results exist and are non-empty
if hasattr(self,
'results') and self.results.data_vars and not force_rerun:
raise exceptions.ModelError(
'This model object already has results. '
'Use model.run(force_rerun=True) to force'
'the results to be overwritten with a new run.')
if (self.run_config['mode'] == 'operate'
and not self._model_data.attrs['allow_operate_mode']):
raise exceptions.ModelError(
'Unable to run this model in operational mode, probably because '
'there exist non-uniform timesteps (e.g. from time masking)')
results, self._backend_model, interface = run_backend(
self._model_data, self._timings, **kwargs)
# Add additional post-processed result variables to results
if results.attrs.get('termination_condition', None) == 'optimal':
results = postprocess.postprocess_model_results(
results, self._model_data, self._timings)
for var in results.data_vars:
results[var].attrs['is_result'] = 1
self._model_data.update(results)
self._model_data.attrs.update(results.attrs)
if 'run_solution_returned' in self._timings.keys():
self._model_data.attrs['solution_time'] = (
self._timings['run_solution_returned'] -
self._timings['run_start']).total_seconds()
self._model_data.attrs['time_finished'] = (
self._timings['run_solution_returned'].strftime(
'%Y-%m-%d %H:%M:%S'))
self.results = self._model_data.filter_by_attrs(is_result=1)
self.backend = interface(self)
def get_formatted_array(self, var, index_format='index'):
"""
Return an xr.DataArray with locs, techs, and carriers as
separate dimensions.
Parameters
----------
var : str
Decision variable for which to return a DataArray.
index_format : str, default = 'index'
'index' to return the `loc_tech(_carrier)` dimensions as individual
indexes, 'multiindex' to return them as a MultiIndex. The latter
has the benefit of having a smaller memory footprint, but you cannot
undertake dimension specific operations (e.g. formatted_array.sum('locs'))
"""
if var not in self._model_data.data_vars:
raise KeyError("Variable {} not in Model data".format(var))
if index_format not in ['index', 'multiindex']:
raise ValueError(
"Argument 'index_format' must be one of 'index' or 'multiindex'"
)
elif index_format == 'index':
return_as = 'DataArray'
elif index_format == 'multiindex':
return_as = 'MultiIndex DataArray'
return split_loc_techs(self._model_data[var], return_as=return_as)
def to_netcdf(self, path):
"""
Save complete model data (inputs and, if available, results)
to a NetCDF file at the given ``path``.
"""
io.save_netcdf(self._model_data, path, model=self)
def to_csv(self, path, dropna=True):
"""
Save complete model data (inputs and, if available, results)
as a set of CSV files to the given ``path``.
Parameters
----------
dropna : bool, optional
If True (default), NaN values are dropped when saving,
resulting in significantly smaller CSV files.
"""
io.save_csv(self._model_data, path, dropna)
def to_lp(self, path):
"""
Save built model to LP format at the given ``path``. If the backend
model has not been built yet, it is built prior to saving.
"""
io.save_lp(self, path)
def info(self):
info_strings = []
model_name = self.model_config.get('name', 'None')
info_strings.append('Model name: {}'.format(model_name))
msize = '{locs} locations, {techs} technologies, {times} timesteps'.format(
locs=len(self._model_data.coords.get('locs', [])),
techs=(
len(self._model_data.coords.get('techs_non_transmission',
[])) +
len(self._model_data.coords.get('techs_transmission_names',
[]))),
times=len(self._model_data.coords.get('timesteps', [])))
info_strings.append('Model size: {}'.format(msize))
return '\n'.join(info_strings)
def _check_future_deprecation_warnings(self):
"""
Method for all FutureWarnings and DeprecationWarnings. Comment above each
warning should specify Calliope version in which it was added, and the
version in which it should be updated/removed.
"""
# Warning that group_share constraints will removed in 0.7.0 #
# Added in 0.6.4-dev, to be removed in v0.7.0-dev
if any('group_share_' in i for i in self._model_data.data_vars.keys()):
warnings.warn(
'`group_share` constraints will be removed in v0.7.0 -- '
'use the new model-wide constraints instead.', FutureWarning)
# Warning that charge rate will be removed in 0.7.0
# Added in 0.6.4-dev, to be removed in 0.7.0-dev
# Rename charge rate to energy_cap_per_storage_cap_max
if self._model_data is not None and "charge_rate" in self._model_data:
warnings.warn(
'`charge_rate` is renamed to `energy_cap_per_storage_cap_max` '
'and will be removed in v0.7.0.', FutureWarning)
def check_operate_params(model_data):
"""
if model mode = `operate`, check for clashes in capacity constraints.
In this mode, all capacity constraints are set to parameters in the backend,
so can easily lead to model infeasibility if not checked.
Returns
-------
comments : AttrDict
debug output
warnings : list
possible problems that do not prevent the model run
from continuing
errors : list
serious issues that should raise a ModelError
"""
defaults = UpdateObserverDict(
initial_yaml_string=model_data.attrs["defaults"],
name="defaults",
observer=model_data,
flat=True,
)
run_config = UpdateObserverDict(
initial_yaml_string=model_data.attrs["run_config"],
name="run_config",
observer=model_data,
)
warnings, errors = [], []
comments = AttrDict()
def _is_missing(var):
if var not in model_data.data_vars.keys():
return True
else:
return model_data[var].isnull()
# Storage initial is carried over between iterations, so must be defined along with storage
if (model_data.include_storage == 1).any():
if "storage_initial" not in model_data.data_vars.keys():
model_data["storage_initial"] = model_data.include_storage.astype(
float)
elif ((model_data.include_storage == 1)
& _is_missing("storage_initial")).any():
model_data.storage_initial = model_data.storage_initial.fillna(
(model_data.include_storage == 1).astype(float))
model_data["storage_initial"].attrs["is_result"] = 0.0
warnings.append(
"Initial stored energy not defined, set to zero for all "
"storage technologies, for use in iterative optimisation")
if ((~_is_missing("storage_cap")) &
(~_is_missing("energy_cap"))).any():
if ((model_data.storage_cap *
model_data.get("energy_cap_per_storage_cap_max", np.inf)) <
model_data.energy_cap).any():
errors.append(
"fixed storage capacity * energy_cap_per_storage_cap_max is not larger "
"than fixed energy capacity for some storage technologies")
if ((model_data.storage_cap *
model_data.get("energy_cap_per_storage_cap_min", 0)) >
model_data.energy_cap).any():
errors.append(
"fixed storage capacity * energy_cap_per_storage_cap_min is not smaller "
"than fixed energy capacity for some technologies")
# Operated units is carried over between iterations, so must be defined in a milp model
if (model_data.cap_method == "integer").any():
if "operated_units" not in model_data.data_vars.keys():
model_data["operated_units"] = (
model_data.cap_method == "integer").astype(float)
elif ((model_data.cap_method == "integer")
& _is_missing("operated_units")).any():
model_data.operated_units = model_data.operated_units.fillna(
(model_data.cap_method == "integer").astype(float))
model_data["operated_units"].attrs["is_result"] = 1
model_data["operated_units"].attrs["operate_param"] = 1
warnings.append(
"daily operated units not defined, set to zero for all technologies defining an integer capacity method, for use in iterative optimisation"
)
if (_is_missing("energy_cap")
& (~_is_missing("energy_cap_min_use")
| (~_is_missing("force_resource")
& (model_data.resource_unit == "energy_per_cap")))).any():
errors.append(
"Operate mode: User must define a finite energy_cap (via energy_cap_equals "
"or energy_cap_max) if using force_resource or energy_cap_min_use")
if (~_is_missing("resource")
& (_is_missing("resource_area")
& (model_data.resource_unit == "energy_per_area"))).any():
errors.append("Operate mode: User must define a finite resource_area "
"(via resource_area_equals or resource_area_max) "
"if available resource is linked to resource_area "
"(resource_unit = `energy_per_area`)")
if ("resource_area" in model_data.data_vars
and (model_data.resource_unit == "energy_per_cap").any()):
model_data["resource_area"] = model_data.resource_area.where(~(
(model_data.force_resource == 1)
& (model_data.resource_unit == "energy_per_cap")))
warnings.append(
"Resource area constraint removed from technologies with "
"force_resource applied and resource linked "
"to energy flow using `energy_per_cap`")
if ("energy_cap" in model_data.data_vars
and (model_data.resource_unit == "energy_per_area").any()):
model_data["energy_cap"] = model_data.energy_cap.where(~(
(model_data.force_resource == 1)
& (model_data.resource_unit == "energy_per_area")))
warnings.append(
"Energy capacity constraint removed from technologies with "
"force_resource applied and resource linked "
"to energy flow using `energy_per_area`")
if (model_data.resource_unit == "energy").any():
if "energy_cap" in model_data.data_vars:
model_data["energy_cap"] = model_data.energy_cap.where(~(
(model_data.force_resource == 1)
& (model_data.resource_unit == "energy")))
warnings.append(
"Energy capacity constraint removed from technologies with "
"force_resource applied and resource not linked "
"to energy flow (resource_unit = `energy`)")
if "resource_area" in model_data.data_vars:
model_data["resource_area"] = model_data.resource_area.where(~(
(model_data.force_resource == 1)
& (model_data.resource_unit == "energy")))
warnings.append(
"Energy capacity constraint removed from technologies with "
"force_resource applied and resource not linked "
"to energy flow (resource_unit = `energy`)")
if ("resource_cap" in model_data.data_vars
and (model_data.force_resource == 1).any()):
model_data["resource_cap"] = model_data.resource_cap.where(
model_data.force_resource == 0)
warnings.append(
"Resource capacity constraint removed from technologies with "
"force_resource applied.")
# Must define a resource capacity to ensure the Pyomo param is created
# for it. But we just create an array of infs, so the capacity has no effect
# TODO: implement this in the masks
window = run_config.get("operation", {}).get("window", None)
horizon = run_config.get("operation", {}).get("horizon", None)
if not window or not horizon:
errors.append(
"Operational mode requires a timestep window and horizon to be "
"defined under run.operation")
elif horizon < window:
errors.append(
"Iteration horizon must be larger than iteration window, "
"for operational mode")
# Cyclic storage isn't really valid in operate mode, so we ignore it, using
# initial_storage instead (allowing us to pass storage between operation windows)
if run_config.get("cyclic_storage", True):
warnings.append(
"Storage cannot be cyclic in operate run mode, setting "
"`run.cyclic_storage` to False for this run")
run_config["cyclic_storage"] = False
return comments, warnings, errors
def run_plan(
model_data,
run_config,
timings,
backend,
build_only,
backend_rerun=False,
allow_warmstart=False,
persistent=True,
opt=None,
):
log_time(logger,
timings,
"run_start",
comment="Backend: starting model run")
warmstart = False
if not backend_rerun:
backend_model = backend.generate_model(model_data)
log_time(
logger,
timings,
"run_backend_model_generated",
time_since_run_start=True,
comment="Backend: model generated",
)
else:
backend_model = backend_rerun
if allow_warmstart:
warmstart = True
run_config = UpdateObserverDict(
initial_yaml_string=model_data.attrs["run_config"],
name="run_config",
observer=model_data,
)
solver = run_config["solver"]
solver_io = run_config.get("solver_io", None)
solver_options = run_config.get("solver_options", None)
save_logs = run_config.get("save_logs", None)
if build_only:
results = xr.Dataset()
else:
if "persistent" in solver and persistent is False:
exceptions.warn(
f"The chosen solver, `{solver}` will not be used in this run. "
f"`{solver.replace('_persistent', '')}` will be used instead.")
solver = solver.replace("_persistent", "")
log_time(
logger,
timings,
"run_solver_start",
comment="Backend: sending model to solver",
)
backend_results, opt = backend.solve_model(
backend_model,
solver=solver,
solver_io=solver_io,
solver_options=solver_options,
save_logs=save_logs,
warmstart=warmstart,
opt=opt,
)
log_time(
logger,
timings,
"run_solver_exit",
time_since_run_start=True,
comment="Backend: solver finished running",
)
termination = backend.load_results(backend_model, backend_results, opt)
log_time(logger,
timings,
"run_results_loaded",
comment="Backend: loaded results")
if termination in ["optimal", "feasible"]:
results = backend.get_result_array(backend_model, model_data)
results.attrs["termination_condition"] = termination
if "persistent" in opt.name and persistent is True:
results.attrs["objective_function_value"] = opt.get_model_attr(
"ObjVal")
else:
results.attrs["objective_function_value"] = backend_model.obj()
else:
results = xr.Dataset(attrs={"termination_condition": termination})
log_time(
logger,
timings,
"run_solution_returned",
time_since_run_start=True,
comment="Backend: generated solution array",
)
return results, backend_model, opt