def test_import_must_be_list(self):
yaml_string = """
import: 'somefile.yaml'
"""
with pytest.raises(ValueError) as excinfo:
AttrDict.from_yaml_string(yaml_string, resolve_imports=True)
assert check_error_or_warning(excinfo, "`import` must be a list.")
def test_exporting_unspecified_carrier(self):
"""
User can only define an export carrier if it is defined in
['carrier_out', 'carrier_out_2', 'carrier_out_3']
"""
override_supply = lambda param: AttrDict.from_yaml_string(
"techs.test_supply_elec.constraints.export_carrier: {}".format(param)
)
override_converison_plus = lambda param: AttrDict.from_yaml_string(
"techs.test_conversion_plus.constraints.export_carrier: {}".format(param)
)
# should fail: exporting `heat` not allowed for electricity supply tech
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override_supply('heat'), scenario='simple_supply,one_day')
# should fail: exporting `random` not allowed for conversion_plus tech
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override_converison_plus('random'), scenario='simple_conversion_plus,one_day')
# should pass: exporting electricity for supply tech
build_model(override_dict=override_supply('electricity'), scenario='simple_supply,one_day')
# should pass: exporting heat for conversion tech
build_model(override_dict=override_converison_plus('heat'), scenario='simple_conversion_plus,one_day')
def test_exporting_unspecified_carrier(self):
"""
User can only define an export carrier if it is defined in
['carrier_out', 'carrier_out_2', 'carrier_out_3']
"""
override_supply = lambda param: AttrDict.from_yaml_string(
"techs.test_supply_elec.constraints.export_carrier: {}".format(
param))
override_converison_plus = lambda param: AttrDict.from_yaml_string(
"techs.test_conversion_plus.constraints.export_carrier: {}".format(
param))
# should fail: exporting `heat` not allowed for electricity supply tech
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override_supply('heat'),
scenario='simple_supply,one_day')
# should fail: exporting `random` not allowed for conversion_plus tech
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override_converison_plus('random'),
scenario='simple_conversion_plus,one_day')
# should pass: exporting electricity for supply tech
build_model(override_dict=override_supply('electricity'),
scenario='simple_supply,one_day')
# should pass: exporting heat for conversion tech
build_model(override_dict=override_converison_plus('heat'),
scenario='simple_conversion_plus,one_day')
def test_missing_required_constraints(self):
"""
A technology within an abstract base technology must define a subset of
hardcoded constraints in order to function
"""
# should fail: missing one of ['energy_cap_max', 'energy_cap_equals', 'energy_cap_per_unit']
override_supply1 = AttrDict.from_yaml_string("""
techs:
supply_missing_constraint:
essentials:
parent: supply
carrier: electricity
name: supply missing constraint
constraints:
resource_area_max: 10
locations.1.techs.supply_missing_constraint:
""")
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override_supply1,
override_groups='simple_supply,one_day')
# should pass: giving one of ['energy_cap_max', 'energy_cap_equals', 'energy_cap_per_unit']
override_supply2 = AttrDict.from_yaml_string("""
techs:
supply_missing_constraint:
essentials:
parent: supply
carrier: electricity
name: supply missing constraint
constraints.energy_cap_max: 10
locations.1.techs.supply_missing_constraint:
""")
build_model(override_dict=override_supply2,
override_groups='simple_supply,one_day')
def _init_from_model_data(self, model_data):
if "_model_run" in model_data.attrs:
self._model_run = AttrDict.from_yaml_string(
model_data.attrs["_model_run"])
del model_data.attrs["_model_run"]
if "_debug_data" in model_data.attrs:
self._debug_data = AttrDict.from_yaml_string(
model_data.attrs["_debug_data"])
del model_data.attrs["_debug_data"]
self._model_data = model_data
self.inputs = self._model_data.filter_by_attrs(is_result=0)
self.model_config = UpdateObserverDict(
initial_yaml_string=model_data.attrs.get("model_config", "{}"),
name="model_config",
observer=self._model_data,
)
self.run_config = UpdateObserverDict(
initial_yaml_string=model_data.attrs.get("run_config", "{}"),
name="run_config",
observer=self._model_data,
)
results = self._model_data.filter_by_attrs(is_result=1)
if len(results.data_vars) > 0:
self.results = results
log_time(
logger,
self._timings,
"model_data_loaded",
comment="Model: loaded model_data",
)
def test_resource_as_carrier(self):
"""
No carrier in technology or technology group can be called `resource`
"""
override1 = AttrDict.from_yaml_string("""
techs:
test_supply_elec:
essentials:
name: Supply tech
carrier: resource
parent: supply
""")
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override1,
override_groups='simple_supply,one_day')
override2 = AttrDict.from_yaml_string("""
tech_groups:
test_supply_group:
essentials:
name: Supply tech
carrier: resource
parent: supply
techs.test_supply_elec.essentials.parent: test_supply_group
""")
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override2,
override_groups='simple_supply,one_day')
def _init_from_model_data(self, model_data):
if '_model_run' in model_data.attrs:
self._model_run = AttrDict.from_yaml_string(
model_data.attrs['_model_run'])
del model_data.attrs['_model_run']
if '_debug_data' in model_data.attrs:
self._debug_data = AttrDict.from_yaml_string(
model_data.attrs['_debug_data'])
del model_data.attrs['_debug_data']
self._model_data = model_data
self.inputs = self._model_data.filter_by_attrs(is_result=0)
self.model_config = UpdateObserverDict(
initial_yaml_string=model_data.attrs.get('model_config', '{}'),
name='model_config',
observer=self._model_data)
self.run_config = UpdateObserverDict(
initial_yaml_string=model_data.attrs.get('run_config', '{}'),
name='run_config',
observer=self._model_data)
results = self._model_data.filter_by_attrs(is_result=1)
if len(results.data_vars) > 0:
self.results = results
log_time(logger,
self._timings,
'model_data_loaded',
comment='Model: loaded model_data')
def test_resource_as_carrier(self):
"""
No carrier in technology or technology group can be called `resource`
"""
override1 = AttrDict.from_yaml_string(
"""
techs:
test_supply_elec:
essentials:
name: Supply tech
carrier: resource
parent: supply
"""
)
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override1, scenario='simple_supply,one_day')
override2 = AttrDict.from_yaml_string(
"""
tech_groups:
test_supply_group:
essentials:
name: Supply tech
carrier: resource
parent: supply
techs.test_supply_elec.essentials.parent: test_supply_group
"""
)
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override2, scenario='simple_supply,one_day')
def test_parser_error(self):
with pytest.raises(ruamel_yaml.YAMLError):
AttrDict.from_yaml_string("""
foo: bar
baz: 1
- foobar
bar: baz
""")
def build_params(model_data, backend_model):
# "Parameters"
backend_model.__calliope_defaults = AttrDict.from_yaml_string(
model_data.attrs["defaults"])
backend_model.__calliope_run_config = AttrDict.from_yaml_string(
model_data.attrs["run_config"])
for k, v in model_data.data_vars.items():
if v.attrs["is_result"] == 0 or (
v.attrs.get("operate_param", 0) == 1
and backend_model.__calliope_run_config["mode"] == "operate"):
with pd.option_context("mode.use_inf_as_na", True):
_kwargs = {
"initialize": v.to_series().dropna().to_dict(),
"mutable": True,
"within": getattr(po, get_domain(v)),
}
if not pd.isnull(backend_model.__calliope_defaults.get(k, None)):
_kwargs["default"] = backend_model.__calliope_defaults[k]
dims = [getattr(backend_model, i) for i in v.dims]
if hasattr(backend_model, k):
logger.debug(
f"The parameter {k} is already an attribute of the Pyomo model."
"It will be prepended with `calliope_` for differentiatation."
)
k = f"calliope_{k}"
setattr(backend_model, k, po.Param(*dims, **_kwargs))
for option_name, option_val in backend_model.__calliope_run_config[
"objective_options"].items():
if option_name == "cost_class":
# TODO: shouldn't require filtering out unused costs (this should be caught by typedconfig?)
objective_cost_class = {
k: v
for k, v in option_val.items() if k in backend_model.costs
}
backend_model.objective_cost_class = po.Param(
backend_model.costs,
initialize=objective_cost_class,
mutable=True,
within=po.Reals,
)
else:
setattr(backend_model, "objective_" + option_name, option_val)
backend_model.bigM = po.Param(
initialize=backend_model.__calliope_run_config.get("bigM", 1e10),
mutable=True,
within=po.NonNegativeReals,
)
def test_import_preserves_comments(self, yaml_file):
with tempfile.TemporaryDirectory() as tempdir:
imported_file = os.path.join(tempdir, 'test_import.yaml')
imported_yaml = """
somekey: 1
anotherkey: 2 # anotherkey's comment
"""
with open(imported_file, 'w') as f:
f.write(imported_yaml)
yaml_string = """
import:
- {}
foo:
bar: 1 # Comment on bar
baz: 2
3:
4: 5
""".format(imported_file)
d = AttrDict.from_yaml_string(yaml_string, resolve_imports=True)
assert 'anotherkey' in d.__dict_comments__
assert d.get_comments(
'anotherkey')['inline'] == "# anotherkey's comment\n"
def test_incorrect_location_coordinates(self):
"""
Either all or no locations must have `coordinates` defined and, if all
defined, they must be in the same coordinate system (lat/lon or x/y)
"""
override = lambda param0, param1: AttrDict.from_yaml_string("""
locations:
0.coordinates: {}
1.coordinates: {}
""".format(param0, param1))
cartesian0 = {'x': 0, 'y': 1}
cartesian1 = {'x': 1, 'y': 1}
geographic0 = {'lat': 0, 'lon': 1}
geographic1 = {'lat': 1, 'lon': 1}
# should fail: cannot have locations in one place and not in another
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override(cartesian0, 'null'),
override_groups='simple_storage,one_day')
# should fail: cannot have cartesian coordinates in one place and geographic in another
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override(cartesian0, geographic1),
override_groups='simple_storage,one_day')
# should pass: cartesian coordinates in both places
build_model(override_dict=override(cartesian0, cartesian1),
override_groups='simple_storage,one_day')
# should pass: geographic coordinates in both places
build_model(override_dict=override(geographic0, geographic1),
override_groups='simple_storage,one_day')
def test_allowed_time_varying_constraints_storage(self):
"""
`file=` is only allowed on a hardcoded list of constraints, unless
`_time_varying` is appended to the constraint (i.e. user input)
"""
allowed_constraints_no_file = list(
set(defaults_model.tech_groups.storage.allowed_constraints).
difference(defaults.file_allowed))
allowed_constraints_file = list(
set(defaults_model.tech_groups.storage.allowed_constraints).
intersection(defaults.file_allowed))
override = lambda param: AttrDict.from_yaml_string(
"techs.test_storage.constraints.{}: file=binary_one_day.csv".
format(param))
# should fail: Cannot have `file=` on the following constraints
for param in allowed_constraints_no_file:
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override(param),
override_groups='simple_storage,one_day')
# should pass: can have `file=` on the following constraints
for param in allowed_constraints_file:
build_model(override_dict=override(param),
override_groups='simple_storage,one_day')
def test_allowed_time_varying_constraints(self):
"""
`file=` is only allowed on a hardcoded list of constraints, unless
`_time_varying` is appended to the constraint (i.e. user input)
"""
allowed_constraints_no_file = list(
set(defaults_model.tech_groups.storage.allowed_constraints)
.difference(defaults.file_allowed)
)
allowed_constraints_file = list(
set(defaults_model.tech_groups.storage.allowed_constraints)
.intersection(defaults.file_allowed)
)
override = lambda param: AttrDict.from_yaml_string(
"techs.test_storage.constraints.{}: file=binary_one_day.csv".format(param)
)
# should fail: Cannot have `file=` on the following constraints
for param in allowed_constraints_no_file:
with pytest.raises(exceptions.ModelError) as errors:
build_model(override_dict=override(param), scenario='simple_storage,one_day')
assert check_error_or_warning(
errors,
'Cannot load `{}` from file for configuration'.format(param)
)
# should pass: can have `file=` on the following constraints
for param in allowed_constraints_file:
build_model(override_dict=override(param), scenario='simple_storage,one_day')
def test_incorrect_subset_time(self):
"""
If subset_time is a list, it must have two entries (start_time, end_time)
If subset_time is not a list, it should successfully subset on the given
string/integer
"""
override = lambda param: AttrDict.from_yaml_string(
"model.subset_time: {}".format(param)
)
# should fail: one string in list
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override(['2005-01']), scenario='simple_supply')
# should fail: three strings in list
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override(['2005-01-01', '2005-01-02', '2005-01-03']), scenario='simple_supply')
# should pass: two string in list as slice
model = build_model(override_dict=override(['2005-01-01', '2005-01-07']), scenario='simple_supply')
assert all(model.inputs.timesteps.to_index() == pd.date_range('2005-01', '2005-01-07 23:00:00', freq='H'))
# should pass: one integer/string
model = build_model(override_dict=override('2005-01'), scenario='simple_supply')
assert all(model.inputs.timesteps.to_index() == pd.date_range('2005-01', '2005-01-31 23:00:00', freq='H'))
# should fail: time subset out of range of input data
with pytest.raises(KeyError):
build_model(override_dict=override('2005-03'), scenario='simple_supply')
# should fail: time subset out of range of input data
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override(['2005-02-01', '2005-02-05']), scenario='simple_supply')
def run(model_data, timings, build_only=False):
"""
Parameters
----------
model_data : xarray.Dataset
Pre-processed dataset of Calliope model data.
timings : dict
Stores timings of various stages of model processing.
build_only : bool, optional
If True, the backend only constructs its in-memory representation
of the problem rather than solving it. Used for debugging and
testing.
"""
BACKEND = {'pyomo': run_pyomo}
INTERFACE = {'pyomo': pyomo_interface}
run_config = AttrDict.from_yaml_string(model_data.attrs['run_config'])
if run_config['mode'] == 'plan':
results, backend = run_plan(model_data,
timings,
backend=BACKEND[run_config.backend],
build_only=build_only)
elif run_config['mode'] == 'operate':
results, backend = run_operate(model_data,
timings,
backend=BACKEND[run_config.backend],
build_only=build_only)
return results, backend, INTERFACE[
run_config.backend].BackendInterfaceMethods
def test_valid_scenarios(self):
"""
Test that valid scenario definition raises no error and results in applied scenario.
"""
override = AttrDict.from_yaml_string("""
scenarios:
scenario_1: ['one', 'two']
overrides:
one:
techs.test_supply_gas.constraints.energy_cap_max: 20
two:
techs.test_supply_elec.constraints.energy_cap_max: 20
locations:
0:
techs:
test_supply_gas:
test_supply_elec:
test_demand_elec:
""")
model = build_model(override_dict=override, scenario='scenario_1')
assert model._model_run.locations[
'0'].techs.test_supply_gas.constraints.energy_cap_max == 20
assert model._model_run.locations[
'0'].techs.test_supply_elec.constraints.energy_cap_max == 20
def test_union_replacement(self, attr_dict):
d = attr_dict
d_new = AttrDict.from_yaml_string("""
c: {_REPLACE_: foo}
""")
d.union(d_new, allow_override=True, allow_replacement=True)
assert d.c == "foo"
def test_incorrect_subset_time(self):
"""
If subset_time is a list, it must have two entries (start_time, end_time)
If subset_time is not a list, it should successfully subset on the given
string/integer
"""
override = lambda param: AttrDict.from_yaml_string(
"model.subset_time: {}".format(param)
)
# should fail: one string in list
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override(['2005-01']), override_groups='simple_supply')
# should fail: three strings in list
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override(['2005-01-01', '2005-01-02', '2005-01-03']), override_groups='simple_supply')
# should pass: two string in list as slice
model = build_model(override_dict=override(['2005-01-01', '2005-01-07']), override_groups='simple_supply')
assert all(model.inputs.timesteps.to_index() == pd.date_range('2005-01', '2005-01-07 23:00:00', freq='H'))
# should pass: one integer/string
model = build_model(override_dict=override('2005-01'), override_groups='simple_supply')
assert all(model.inputs.timesteps.to_index() == pd.date_range('2005-01', '2005-01-31 23:00:00', freq='H'))
# should fail: time subset out of range of input data
with pytest.raises(KeyError):
build_model(override_dict=override('2005-03'), override_groups='simple_supply')
# should fail: time subset out of range of input data
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override(['2005-02-01', '2005-02-05']), override_groups='simple_supply')
def test_valid_scenarios(self):
"""
Test that valid scenario definition raises no error and results in applied scenario.
"""
override = AttrDict.from_yaml_string(
"""
scenarios:
scenario_1: ['one', 'two']
overrides:
one:
techs.test_supply_gas.constraints.energy_cap_max: 20
two:
techs.test_supply_elec.constraints.energy_cap_max: 20
locations:
0:
techs:
test_supply_gas:
test_supply_elec:
test_demand_elec:
"""
)
model = build_model(override_dict=override, scenario='scenario_1')
assert model._model_run.locations['0'].techs.test_supply_gas.constraints.energy_cap_max == 20
assert model._model_run.locations['0'].techs.test_supply_elec.constraints.energy_cap_max == 20
def test_nested_import(self, yaml_file):
with tempfile.TemporaryDirectory() as tempdir:
imported_file = os.path.join(tempdir, "test_import.yaml")
imported_yaml = """
somekey: 1
anotherkey: 2
"""
with open(imported_file, "w") as f:
f.write(imported_yaml)
yaml_string = """
foobar:
import:
- {}
foo:
bar: 1
baz: 2
3:
4: 5
""".format(imported_file)
d = AttrDict.from_yaml_string(yaml_string,
resolve_imports="foobar")
assert "foobar.somekey" in d.keys_nested()
assert d.get_key("foobar.anotherkey") == 2
def postprocess_model_results(results, model_data, timings):
"""
Adds additional post-processed result variables to
the given model results in-place. Model must have solved successfully.
Parameters
----------
results : xarray Dataset
Output from the solver backend
model_data : xarray Dataset
Calliope model data, stored as calliope.Model()._model_data
timings : dict
Calliope timing dictionary, stored as calliope.Model()._timings
Returns
-------
results : xarray Dataset
Input results Dataset, with additional DataArray variables and removed
all instances of unreasonably low numbers (set by zero_threshold)
"""
log_time(logger,
timings,
"post_process_start",
comment="Postprocessing: started")
if model_data.attrs['scale']:
scale(model_data, lambda x: 1 / x)
results['scale'] = model_data['scale']
scale(results, lambda x: 1 / x)
run_config = AttrDict.from_yaml_string(model_data.attrs["run_config"])
results["capacity_factor"] = capacity_factor(results, model_data)
results["systemwide_capacity_factor"] = systemwide_capacity_factor(
results, model_data)
results["systemwide_levelised_cost"] = systemwide_levelised_cost(
results, model_data)
results["total_levelised_cost"] = systemwide_levelised_cost(results,
model_data,
total=True)
results = clean_results(results, run_config.get("zero_threshold", 0),
timings)
log_time(
logger,
timings,
"post_process_end",
time_since_run_start=True,
comment="Postprocessing: ended",
)
if "run_solution_returned" in timings.keys():
results.attrs["solution_time"] = (
timings["run_solution_returned"] -
timings["run_start"]).total_seconds()
results.attrs["time_finished"] = timings[
"run_solution_returned"].strftime("%Y-%m-%d %H:%M:%S")
return results
def test_order_of_subdicts(self):
d = AttrDict.from_yaml_string("""
A.B.C: 10
A.B:
E: 20
""")
assert d.A.B.C == 10
assert d.A.B.E == 20
def test_do_not_resolve_imports(self):
yaml_string = """
import: ['somefile.yaml']
"""
d = AttrDict.from_yaml_string(yaml_string, resolve_imports=False)
# Should not raise an error about a missing file, as we ask for
# imports not to be resolved
assert d["import"] == ["somefile.yaml"]
def test_milp_constraints(self):
"""
If `units` is defined, but not `energy_cap_per_unit`, throw an error
"""
# should fail: no energy_cap_per_unit
override1 = AttrDict.from_yaml_string("techs.test_supply_elec.constraints.units_max: 4")
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override1, scenario='simple_supply,one_day')
# should pass: energy_cap_per_unit given
override2 = AttrDict.from_yaml_string("""
techs.test_supply_elec.constraints:
units_max: 4
energy_cap_per_unit: 5
""")
build_model(override_dict=override2, scenario='simple_supply,one_day')
def test_incorrect_date_format(self):
"""
Test the date parser catches a different date format from file than
user input/default (inc. if it is just one line of a file that is incorrect)
"""
# should pass: changing datetime format from default
override1 = AttrDict.from_yaml_string("""
model.timeseries_dateformat: "%d/%m/%Y %H:%M:%S"
techs.test_demand_heat.constraints.resource: file=demand_heat_diff_dateformat.csv
techs.test_demand_elec.constraints.resource: file=demand_heat_diff_dateformat.csv
""")
model = build_model(override_dict=override1,
override_groups='simple_conversion')
assert all(model.inputs.timesteps.to_index() == pd.date_range(
'2005-01', '2005-02-01 23:00:00', freq='H'))
# should fail: wrong dateformat input for one file
override2 = AttrDict.from_yaml_string("""
techs.test_demand_heat.constraints.resource: file=demand_heat_diff_dateformat.csv
""")
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override2,
override_groups='simple_conversion')
# should fail: wrong dateformat input for all files
override3 = AttrDict.from_yaml_string("""
model.timeseries_dateformat: "%d/%m/%Y %H:%M:%S"
""")
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override3,
override_groups='simple_supply')
# should fail: one value wrong in file
override4 = AttrDict.from_yaml_string("""
techs.test_demand_heat.constraints.resource: file=demand_heat_wrong_dateformat.csv
""")
# check in output error that it points to: 07/01/2005 10:00:00
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override4,
override_groups='simple_conversion')
def test_milp_constraints(self):
"""
If `units` is defined, but not `energy_cap_per_unit`, throw an error
"""
# should fail: no energy_cap_per_unit
override1 = AttrDict.from_yaml_string("techs.test_supply_elec.constraints.units_max: 4")
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override1, override_groups='simple_supply,one_day')
# should pass: energy_cap_per_unit given
override2 = AttrDict.from_yaml_string("""
techs.test_supply_elec.constraints:
units_max: 4
energy_cap_per_unit: 5
""")
build_model(override_dict=override2, override_groups='simple_supply,one_day')
def test_tech_as_parent(self):
"""
All technologies and technology groups must specify a parent
"""
override1 = AttrDict.from_yaml_string("""
techs.test_supply_tech_parent:
essentials:
name: Supply tech
carrier: gas
parent: test_supply_elec
constraints:
energy_cap_max: 10
resource: .inf
locations.1.test_supply_tech_parent:
""")
with pytest.raises(exceptions.ModelError) as error:
build_model(override_dict=override1,
scenario='simple_supply,one_day')
check_error_or_warning(
error,
'tech `test_supply_tech_parent` has another tech as a parent')
override2 = AttrDict.from_yaml_string("""
tech_groups.test_supply_group:
essentials:
carrier: gas
parent: test_supply_elec
constraints:
energy_cap_max: 10
resource: .inf
techs.test_supply_tech_parent.essentials:
name: Supply tech
parent: test_supply_group
locations.1.test_supply_tech_parent:
""")
with pytest.raises(exceptions.ModelError) as error:
build_model(override_dict=override2,
scenario='simple_supply,one_day')
check_error_or_warning(
error, 'tech_group `test_supply_group` has a tech as a parent')
def test_tech_as_parent(self):
"""
All technologies and technology groups must specify a parent
"""
override1 = AttrDict.from_yaml_string(
"""
techs.test_supply_tech_parent:
essentials:
name: Supply tech
carrier: gas
parent: test_supply_elec
constraints:
energy_cap_max: 10
resource: .inf
locations.1.test_supply_tech_parent:
"""
)
with pytest.raises(exceptions.ModelError) as error:
build_model(override_dict=override1, scenario='simple_supply,one_day')
check_error_or_warning(error, 'tech `test_supply_tech_parent` has another tech as a parent')
override2 = AttrDict.from_yaml_string(
"""
tech_groups.test_supply_group:
essentials:
carrier: gas
parent: test_supply_elec
constraints:
energy_cap_max: 10
resource: .inf
techs.test_supply_tech_parent.essentials:
name: Supply tech
parent: test_supply_group
locations.1.test_supply_tech_parent:
"""
)
with pytest.raises(exceptions.ModelError) as error:
build_model(override_dict=override2, scenario='simple_supply,one_day')
check_error_or_warning(error, 'tech_group `test_supply_group` has a tech as a parent')
def _init_from_model_data(self, model_data):
if "_model_run" in model_data.attrs:
self._model_run = AttrDict.from_yaml_string(model_data.attrs["_model_run"])
del model_data.attrs["_model_run"]
if "_debug_data" in model_data.attrs:
self._debug_data = AttrDict.from_yaml_string(
model_data.attrs["_debug_data"]
)
del model_data.attrs["_debug_data"]
self._model_data = model_data
self._add_model_data_methods()
log_time(
logger,
self._timings,
"model_data_loaded",
comment="Model: loaded model_data",
)
def test_union_preserves_comments(self, yaml_file):
d = AttrDict.from_yaml(yaml_file)
d_new = AttrDict.from_yaml_string("""
test: 1 # And a comment
somekey:
bar:
baz: 2 # Another comment
""")
d.union(d_new)
assert d.get_comments(
'somekey.bar.baz')['inline'] == '# Another comment\n'
def test_unknown_carrier_tier(self):
"""
User can only use 'carrier_' + ['in', 'out', 'in_2', 'out_2', 'in_3',
'out_3', 'ratios']
"""
override1 = AttrDict.from_yaml_string("""
techs.test_supply_elec.essentials.carrier_1: power
""")
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override1,
override_groups='simple_supply,one_day')
override2 = AttrDict.from_yaml_string("""
techs.test_conversion_plus.essentials.carrier_out_4: power
""")
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override2,
override_groups='simple_conversion_plus,one_day')
def test_defining_non_allowed_costs(self):
"""
A technology within an abstract base technology can only define a subset
of hardcoded costs, anything else will not be implemented, so are
not allowed for that technology. This includes misspellings
"""
# should fail: storage_cap_max not allowed for supply tech
override = AttrDict.from_yaml_string("""
techs.test_supply_elec.costs.monetary.storage_cap: 10
""")
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override,
override_groups='simple_supply,one_day')
# should fail: om_prod not allowed for demand tech
override = AttrDict.from_yaml_string("""
techs.test_demand_elec.costs.monetary.om_prod: 10
""")
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override,
override_groups='simple_supply,one_day')
def __init__(self, initial_dict, initial_yaml_string, on_changed=None):
if initial_yaml_string is not None:
initial_dict = AttrDict.from_yaml_string(initial_yaml_string).as_dict()
super().__init__(initial_dict)
self.on_changed = on_changed
for k, v in initial_dict.items():
if isinstance(v, dict):
super().__setitem__(k, ObservedDict(v, None, on_changed=self.notify))
self.notify()
def get_result_array(backend_model, model_data):
"""
From a Pyomo model object, extract decision variable data and return it as
an xarray Dataset. Any rogue input parameters that are constructed inside
the backend (instead of being passed by calliope.Model().inputs) are also
added to calliope.Model()._model_data in-place.
"""
subsets_config = AttrDict.from_yaml_string(model_data.attrs["subsets"])
def _get_dim_order(foreach):
return tuple([i for i in model_data.dims.keys() if i in foreach])
all_variables = {
i.name: get_var(
backend_model,
i.name,
dims=_get_dim_order(subsets_config.variables[i.name].foreach),
)
for i in backend_model.component_objects(ctype=po.Var)
}
# Add in expressions, which are combinations of variables (e.g. costs)
all_variables.update({
i.name: get_var(
backend_model,
i.name,
dims=_get_dim_order(subsets_config.expressions[i.name].foreach),
expr=True,
)
for i in backend_model.component_objects(ctype=po.Expression)
})
# Get any parameters that did not appear in the user's model.inputs Dataset
all_params = {
i.name: get_var(backend_model, i.name, expr=True)
for i in backend_model.component_objects(
ctype=po.base.param.IndexedParam)
if i.name not in model_data.data_vars.keys()
and "objective_" not in i.name
}
results = string_to_datetime(
backend_model, reorganise_xarray_dimensions(xr.Dataset(all_variables)))
if all_params:
additional_inputs = reorganise_xarray_dimensions(
xr.Dataset(all_params))
for var in additional_inputs.data_vars:
additional_inputs[var].attrs["is_result"] = 0
model_data.update(additional_inputs)
model_data = string_to_datetime(backend_model, model_data)
results = string_to_datetime(backend_model, results)
return results
def postprocess_model_results(results, model_data, timings):
"""
Adds additional post-processed result variables to
the given model results in-place. Model must have solved successfully.
Parameters
----------
results : xarray Dataset
Output from the solver backend
model_data : xarray Dataset
Calliope model data, stored as calliope.Model()._model_data
timings : dict
Calliope timing dictionary, stored as calliope.Model()._timings
Returns
-------
results : xarray Dataset
Input results Dataset, with additional DataArray variables and removed
all instances of unreasonably low numbers (set by zero_threshold)
"""
log_time(logger,
timings,
'post_process_start',
comment='Postprocessing: started')
run_config = AttrDict.from_yaml_string(model_data.attrs['run_config'])
results['capacity_factor'] = capacity_factor(results, model_data)
results['systemwide_capacity_factor'] = systemwide_capacity_factor(
results, model_data)
results['systemwide_levelised_cost'] = systemwide_levelised_cost(
results, model_data)
results['total_levelised_cost'] = systemwide_levelised_cost(results,
model_data,
total=True)
results = clean_results(results, run_config.get('zero_threshold', 0),
timings)
log_time(logger,
timings,
'post_process_end',
time_since_run_start=True,
comment='Postprocessing: ended')
if 'run_solution_returned' in timings.keys():
results.attrs['solution_time'] = (
timings['run_solution_returned'] -
timings['run_start']).total_seconds()
results.attrs['time_finished'] = (
timings['run_solution_returned'].strftime('%Y-%m-%d %H:%M:%S'))
return results
def test_negative_cost_unassigned_cap(self):
"""
Any negative cost associated with a capacity (e.g. cost_energy_cap) must
be applied to a capacity iff the upper bound of that capacity has been defined
"""
# should fail: resource_cap cost is negtive, resource_cap_max is infinite
override = AttrDict.from_yaml_string(
"techs.test_supply_plus.costs.monetary.resource_cap: -10")
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override,
override_groups='simple_supply_plus,one_day')
# should fail: storage_cap cost is negative, storage_cap_max is infinite
override = AttrDict.from_yaml_string("""
techs.test_storage:
constraints.storage_cap_max: .inf
costs.monetary.storage_cap: -10
""")
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override,
override_groups='simple_storage,one_day')
def test_negative_cost_unassigned_cap(self):
"""
Any negative cost associated with a capacity (e.g. cost_energy_cap) must
be applied to a capacity iff the upper bound of that capacity has been defined
"""
# should fail: resource_cap cost is negtive, resource_cap_max is infinite
override = AttrDict.from_yaml_string(
"techs.test_supply_plus.costs.monetary.resource_cap: -10"
)
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override, scenario='simple_supply_plus,one_day')
# should fail: storage_cap cost is negative, storage_cap_max is infinite
override = AttrDict.from_yaml_string(
"""
techs.test_storage:
constraints.storage_cap_max: .inf
costs.monetary.storage_cap: -10
"""
)
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override, scenario='simple_storage,one_day')
def test_defining_non_allowed_costs(self):
"""
A technology within an abstract base technology can only define a subset
of hardcoded costs, anything else will not be implemented, so are
not allowed for that technology. This includes misspellings
"""
# should fail: storage_cap_max not allowed for supply tech
override = AttrDict.from_yaml_string(
"""
techs.test_supply_elec.costs.monetary.storage_cap: 10
"""
)
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override, scenario='simple_supply,one_day')
# should fail: om_prod not allowed for demand tech
override = AttrDict.from_yaml_string(
"""
techs.test_demand_elec.costs.monetary.om_prod: 10
"""
)
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override, scenario='simple_supply,one_day')
def test_invalid_scenarios_str(self):
"""
Test that invalid scenario definition raises appropriate error
"""
override = AttrDict.from_yaml_string(
"""
scenarios:
scenario_1: 'foo1,foo2'
"""
)
with pytest.raises(exceptions.ModelError) as error:
build_model(override_dict=override, scenario='scenario_1')
assert check_error_or_warning(error, 'Scenario definition must be a list of override names.')
def test_unknown_carrier_tier(self):
"""
User can only use 'carrier_' + ['in', 'out', 'in_2', 'out_2', 'in_3',
'out_3', 'ratios']
"""
override1 = AttrDict.from_yaml_string(
"""
techs.test_supply_elec.essentials.carrier_1: power
"""
)
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override1, scenario='simple_supply,one_day')
override2 = AttrDict.from_yaml_string(
"""
techs.test_conversion_plus.essentials.carrier_out_4: power
"""
)
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override2, scenario='simple_conversion_plus,one_day')
def test_scenario_name_overlaps_overrides(self):
"""
Test that a scenario name cannot be a combination of override names
"""
override = AttrDict.from_yaml_string(
"""
scenarios:
'simple_supply,group_share_energy_cap_min': 'foobar'
"""
)
with pytest.raises(exceptions.ModelError) as error:
build_model(override_dict=override, scenario='simple_supply,group_share_energy_cap_min')
assert check_error_or_warning(error, 'Manually defined scenario cannot be a combination of override names.')
def test_missing_required_constraints(self):
"""
A technology within an abstract base technology must define a subset of
hardcoded constraints in order to function
"""
# should fail: missing one of ['energy_cap_max', 'energy_cap_equals', 'energy_cap_per_unit']
override_supply1 = AttrDict.from_yaml_string(
"""
techs:
supply_missing_constraint:
essentials:
parent: supply
carrier: electricity
name: supply missing constraint
constraints:
resource_area_max: 10
locations.1.techs.supply_missing_constraint:
"""
)
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override_supply1, scenario='simple_supply,one_day')
# should pass: giving one of ['energy_cap_max', 'energy_cap_equals', 'energy_cap_per_unit']
override_supply2 = AttrDict.from_yaml_string(
"""
techs:
supply_missing_constraint:
essentials:
parent: supply
carrier: electricity
name: supply missing constraint
constraints.energy_cap_max: 10
locations.1.techs.supply_missing_constraint:
"""
)
build_model(override_dict=override_supply2, scenario='simple_supply,one_day')
def test_abstract_base_tech_group_override(self):
"""
Abstract base technology groups can be overridden
"""
override = AttrDict.from_yaml_string(
"""
tech_groups:
supply:
constraints:
lifetime: 25
locations:
1.techs.test_supply_elec:
1.techs.test_demand_elec:
"""
)
build_model(override_dict=override, scenario='one_day')
def test_missing_constraints(self):
"""
A technology must define at least one constraint.
"""
override = AttrDict.from_yaml_string(
"""
techs:
supply_missing_constraint:
essentials:
parent: supply
carrier: electricity
name: supply missing constraint
locations.1.techs.supply_missing_constraint:
"""
)
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override, scenario='simple_supply,one_day')
def test_undefined_carriers(self):
"""
Test that user has input either carrier or carrier_in/_out for each tech
"""
override = AttrDict.from_yaml_string(
"""
techs:
test_undefined_carrier:
essentials:
parent: supply
name: test
constraints:
resource: .inf
energy_cap_max: .inf
locations.1.techs.test_undefined_carrier:
"""
)
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override, scenario='simple_supply,one_day')
def combine_overrides(config_model, overrides):
override_dict = AttrDict()
for override in overrides:
try:
yaml_string = config_model.overrides[override].to_yaml()
override_with_imports = AttrDict.from_yaml_string(yaml_string)
except KeyError:
raise exceptions.ModelError(
'Override `{}` is not defined.'.format(override)
)
try:
override_dict.union(override_with_imports, allow_override=False)
except KeyError as e:
raise exceptions.ModelError(
str(e)[1:-1] + '. Already specified but defined again in '
'override `{}`.'.format(override)
)
return override_dict
def test_unspecified_parent(self):
"""
All technologies and technology groups must specify a parent
"""
override = AttrDict.from_yaml_string(
"""
techs.test_supply_no_parent:
essentials:
name: Supply tech
carrier: gas
constraints:
energy_cap_max: 10
resource: .inf
locations.1.test_supply_no_parent:
"""
)
with pytest.raises(KeyError):
build_model(override_dict=override, scenario='simple_supply,one_day')
def test_name_overlap(self):
"""
No tech may have the same identifier as a tech group
"""
override = AttrDict.from_yaml_string(
"""
techs:
supply:
essentials:
name: Supply tech
carrier: gas
parent: supply
constraints:
energy_cap_max: 10
resource: .inf
locations:
1.techs.supply:
0.techs.supply:
"""
)
with pytest.raises(exceptions.ModelError):
build_model(override_dict=override, scenario='one_day')
def apply_overrides(config, scenario=None, override_dict=None):
"""
Generate processed Model configuration, applying any scenarios overrides.
Parameters
----------
config : AttrDict
a model configuration AttrDict
scenario : str, optional
override_dict : str or dict or AttrDict, optional
If a YAML string, converted to AttrDict
"""
debug_comments = AttrDict()
base_model_config_file = os.path.join(
os.path.dirname(calliope.__file__),
'config', 'model.yaml'
)
config_model = AttrDict.from_yaml(base_model_config_file)
# Interpret timeseries_data_path as relative
config.model.timeseries_data_path = relative_path(
config.config_path, config.model.timeseries_data_path
)
# The input files are allowed to override other model defaults
config_model.union(config, allow_override=True)
# First pass of applying override dict before applying scenarios,
# so that can override scenario definitions by override_dict
if override_dict:
if isinstance(override_dict, str):
override_dict = AttrDict.from_yaml_string(override_dict)
elif not isinstance(override_dict, AttrDict):
override_dict = AttrDict(override_dict)
warnings = checks.check_overrides(config_model, override_dict)
exceptions.print_warnings_and_raise_errors(warnings=warnings)
config_model.union(
override_dict, allow_override=True, allow_replacement=True
)
if scenario:
scenarios = config_model.get('scenarios', {})
if scenario in scenarios.keys():
# Manually defined scenario names cannot be the same as single
# overrides or any combination of semicolon-delimited overrides
if all([i in config_model.get('overrides', {})
for i in scenario.split(',')]):
raise exceptions.ModelError(
'Manually defined scenario cannot be a combination of override names.'
)
if not isinstance(scenarios[scenario], list):
raise exceptions.ModelError(
'Scenario definition must be a list of override names.'
)
overrides = [str(i) for i in scenarios[scenario]]
logger.info(
'Using scenario `{}` leading to the application of '
'overrides `{}`.'.format(scenario, overrides)
)
else:
overrides = str(scenario).split(',')
logger.info(
'Applying the following overrides without a '
'specific scenario name: {}'.format(overrides)
)
overrides_from_scenario = combine_overrides(config_model, overrides)
warnings = checks.check_overrides(config_model, overrides_from_scenario)
exceptions.print_warnings_and_raise_errors(warnings=warnings)
config_model.union(
overrides_from_scenario, allow_override=True, allow_replacement=True
)
for k, v in overrides_from_scenario.as_dict_flat().items():
debug_comments.set_key(
'{}'.format(k),
'Applied from override')
else:
overrides = []
# Second pass of applying override dict after applying scenarios,
# so that scenario-based overrides are overridden by override_dict!
if override_dict:
config_model.union(
override_dict, allow_override=True, allow_replacement=True
)
for k, v in override_dict.as_dict_flat().items():
debug_comments.set_key(
'{}'.format(k),
'Overridden via override dictionary.')
return config_model, debug_comments, overrides, scenario
