def test_import_must_be_list(self): yaml_string = """ import: 'somefile.yaml' """ with pytest.raises(ValueError) as excinfo: AttrDict.from_yaml_string(yaml_string, resolve_imports=True) assert check_error_or_warning(excinfo, "`import` must be a list.")
def test_exporting_unspecified_carrier(self): """ User can only define an export carrier if it is defined in ['carrier_out', 'carrier_out_2', 'carrier_out_3'] """ override_supply = lambda param: AttrDict.from_yaml_string( "techs.test_supply_elec.constraints.export_carrier: {}".format(param) ) override_converison_plus = lambda param: AttrDict.from_yaml_string( "techs.test_conversion_plus.constraints.export_carrier: {}".format(param) ) # should fail: exporting `heat` not allowed for electricity supply tech with pytest.raises(exceptions.ModelError): build_model(override_dict=override_supply('heat'), scenario='simple_supply,one_day') # should fail: exporting `random` not allowed for conversion_plus tech with pytest.raises(exceptions.ModelError): build_model(override_dict=override_converison_plus('random'), scenario='simple_conversion_plus,one_day') # should pass: exporting electricity for supply tech build_model(override_dict=override_supply('electricity'), scenario='simple_supply,one_day') # should pass: exporting heat for conversion tech build_model(override_dict=override_converison_plus('heat'), scenario='simple_conversion_plus,one_day')
def test_exporting_unspecified_carrier(self): """ User can only define an export carrier if it is defined in ['carrier_out', 'carrier_out_2', 'carrier_out_3'] """ override_supply = lambda param: AttrDict.from_yaml_string( "techs.test_supply_elec.constraints.export_carrier: {}".format( param)) override_converison_plus = lambda param: AttrDict.from_yaml_string( "techs.test_conversion_plus.constraints.export_carrier: {}".format( param)) # should fail: exporting `heat` not allowed for electricity supply tech with pytest.raises(exceptions.ModelError): build_model(override_dict=override_supply('heat'), scenario='simple_supply,one_day') # should fail: exporting `random` not allowed for conversion_plus tech with pytest.raises(exceptions.ModelError): build_model(override_dict=override_converison_plus('random'), scenario='simple_conversion_plus,one_day') # should pass: exporting electricity for supply tech build_model(override_dict=override_supply('electricity'), scenario='simple_supply,one_day') # should pass: exporting heat for conversion tech build_model(override_dict=override_converison_plus('heat'), scenario='simple_conversion_plus,one_day')
def test_missing_required_constraints(self): """ A technology within an abstract base technology must define a subset of hardcoded constraints in order to function """ # should fail: missing one of ['energy_cap_max', 'energy_cap_equals', 'energy_cap_per_unit'] override_supply1 = AttrDict.from_yaml_string(""" techs: supply_missing_constraint: essentials: parent: supply carrier: electricity name: supply missing constraint constraints: resource_area_max: 10 locations.1.techs.supply_missing_constraint: """) with pytest.raises(exceptions.ModelError): build_model(override_dict=override_supply1, override_groups='simple_supply,one_day') # should pass: giving one of ['energy_cap_max', 'energy_cap_equals', 'energy_cap_per_unit'] override_supply2 = AttrDict.from_yaml_string(""" techs: supply_missing_constraint: essentials: parent: supply carrier: electricity name: supply missing constraint constraints.energy_cap_max: 10 locations.1.techs.supply_missing_constraint: """) build_model(override_dict=override_supply2, override_groups='simple_supply,one_day')
def _init_from_model_data(self, model_data): if "_model_run" in model_data.attrs: self._model_run = AttrDict.from_yaml_string( model_data.attrs["_model_run"]) del model_data.attrs["_model_run"] if "_debug_data" in model_data.attrs: self._debug_data = AttrDict.from_yaml_string( model_data.attrs["_debug_data"]) del model_data.attrs["_debug_data"] self._model_data = model_data self.inputs = self._model_data.filter_by_attrs(is_result=0) self.model_config = UpdateObserverDict( initial_yaml_string=model_data.attrs.get("model_config", "{}"), name="model_config", observer=self._model_data, ) self.run_config = UpdateObserverDict( initial_yaml_string=model_data.attrs.get("run_config", "{}"), name="run_config", observer=self._model_data, ) results = self._model_data.filter_by_attrs(is_result=1) if len(results.data_vars) > 0: self.results = results log_time( logger, self._timings, "model_data_loaded", comment="Model: loaded model_data", )
def test_resource_as_carrier(self): """ No carrier in technology or technology group can be called `resource` """ override1 = AttrDict.from_yaml_string(""" techs: test_supply_elec: essentials: name: Supply tech carrier: resource parent: supply """) with pytest.raises(exceptions.ModelError): build_model(override_dict=override1, override_groups='simple_supply,one_day') override2 = AttrDict.from_yaml_string(""" tech_groups: test_supply_group: essentials: name: Supply tech carrier: resource parent: supply techs.test_supply_elec.essentials.parent: test_supply_group """) with pytest.raises(exceptions.ModelError): build_model(override_dict=override2, override_groups='simple_supply,one_day')
def _init_from_model_data(self, model_data): if '_model_run' in model_data.attrs: self._model_run = AttrDict.from_yaml_string( model_data.attrs['_model_run']) del model_data.attrs['_model_run'] if '_debug_data' in model_data.attrs: self._debug_data = AttrDict.from_yaml_string( model_data.attrs['_debug_data']) del model_data.attrs['_debug_data'] self._model_data = model_data self.inputs = self._model_data.filter_by_attrs(is_result=0) self.model_config = UpdateObserverDict( initial_yaml_string=model_data.attrs.get('model_config', '{}'), name='model_config', observer=self._model_data) self.run_config = UpdateObserverDict( initial_yaml_string=model_data.attrs.get('run_config', '{}'), name='run_config', observer=self._model_data) results = self._model_data.filter_by_attrs(is_result=1) if len(results.data_vars) > 0: self.results = results log_time(logger, self._timings, 'model_data_loaded', comment='Model: loaded model_data')
def test_resource_as_carrier(self): """ No carrier in technology or technology group can be called `resource` """ override1 = AttrDict.from_yaml_string( """ techs: test_supply_elec: essentials: name: Supply tech carrier: resource parent: supply """ ) with pytest.raises(exceptions.ModelError): build_model(override_dict=override1, scenario='simple_supply,one_day') override2 = AttrDict.from_yaml_string( """ tech_groups: test_supply_group: essentials: name: Supply tech carrier: resource parent: supply techs.test_supply_elec.essentials.parent: test_supply_group """ ) with pytest.raises(exceptions.ModelError): build_model(override_dict=override2, scenario='simple_supply,one_day')
def test_parser_error(self): with pytest.raises(ruamel_yaml.YAMLError): AttrDict.from_yaml_string(""" foo: bar baz: 1 - foobar bar: baz """)
def build_params(model_data, backend_model): # "Parameters" backend_model.__calliope_defaults = AttrDict.from_yaml_string( model_data.attrs["defaults"]) backend_model.__calliope_run_config = AttrDict.from_yaml_string( model_data.attrs["run_config"]) for k, v in model_data.data_vars.items(): if v.attrs["is_result"] == 0 or ( v.attrs.get("operate_param", 0) == 1 and backend_model.__calliope_run_config["mode"] == "operate"): with pd.option_context("mode.use_inf_as_na", True): _kwargs = { "initialize": v.to_series().dropna().to_dict(), "mutable": True, "within": getattr(po, get_domain(v)), } if not pd.isnull(backend_model.__calliope_defaults.get(k, None)): _kwargs["default"] = backend_model.__calliope_defaults[k] dims = [getattr(backend_model, i) for i in v.dims] if hasattr(backend_model, k): logger.debug( f"The parameter {k} is already an attribute of the Pyomo model." "It will be prepended with `calliope_` for differentiatation." ) k = f"calliope_{k}" setattr(backend_model, k, po.Param(*dims, **_kwargs)) for option_name, option_val in backend_model.__calliope_run_config[ "objective_options"].items(): if option_name == "cost_class": # TODO: shouldn't require filtering out unused costs (this should be caught by typedconfig?) objective_cost_class = { k: v for k, v in option_val.items() if k in backend_model.costs } backend_model.objective_cost_class = po.Param( backend_model.costs, initialize=objective_cost_class, mutable=True, within=po.Reals, ) else: setattr(backend_model, "objective_" + option_name, option_val) backend_model.bigM = po.Param( initialize=backend_model.__calliope_run_config.get("bigM", 1e10), mutable=True, within=po.NonNegativeReals, )
def test_import_preserves_comments(self, yaml_file): with tempfile.TemporaryDirectory() as tempdir: imported_file = os.path.join(tempdir, 'test_import.yaml') imported_yaml = """ somekey: 1 anotherkey: 2 # anotherkey's comment """ with open(imported_file, 'w') as f: f.write(imported_yaml) yaml_string = """ import: - {} foo: bar: 1 # Comment on bar baz: 2 3: 4: 5 """.format(imported_file) d = AttrDict.from_yaml_string(yaml_string, resolve_imports=True) assert 'anotherkey' in d.__dict_comments__ assert d.get_comments( 'anotherkey')['inline'] == "# anotherkey's comment\n"
def test_incorrect_location_coordinates(self): """ Either all or no locations must have `coordinates` defined and, if all defined, they must be in the same coordinate system (lat/lon or x/y) """ override = lambda param0, param1: AttrDict.from_yaml_string(""" locations: 0.coordinates: {} 1.coordinates: {} """.format(param0, param1)) cartesian0 = {'x': 0, 'y': 1} cartesian1 = {'x': 1, 'y': 1} geographic0 = {'lat': 0, 'lon': 1} geographic1 = {'lat': 1, 'lon': 1} # should fail: cannot have locations in one place and not in another with pytest.raises(exceptions.ModelError): build_model(override_dict=override(cartesian0, 'null'), override_groups='simple_storage,one_day') # should fail: cannot have cartesian coordinates in one place and geographic in another with pytest.raises(exceptions.ModelError): build_model(override_dict=override(cartesian0, geographic1), override_groups='simple_storage,one_day') # should pass: cartesian coordinates in both places build_model(override_dict=override(cartesian0, cartesian1), override_groups='simple_storage,one_day') # should pass: geographic coordinates in both places build_model(override_dict=override(geographic0, geographic1), override_groups='simple_storage,one_day')
def test_allowed_time_varying_constraints_storage(self): """ `file=` is only allowed on a hardcoded list of constraints, unless `_time_varying` is appended to the constraint (i.e. user input) """ allowed_constraints_no_file = list( set(defaults_model.tech_groups.storage.allowed_constraints). difference(defaults.file_allowed)) allowed_constraints_file = list( set(defaults_model.tech_groups.storage.allowed_constraints). intersection(defaults.file_allowed)) override = lambda param: AttrDict.from_yaml_string( "techs.test_storage.constraints.{}: file=binary_one_day.csv". format(param)) # should fail: Cannot have `file=` on the following constraints for param in allowed_constraints_no_file: with pytest.raises(exceptions.ModelError): build_model(override_dict=override(param), override_groups='simple_storage,one_day') # should pass: can have `file=` on the following constraints for param in allowed_constraints_file: build_model(override_dict=override(param), override_groups='simple_storage,one_day')
def test_allowed_time_varying_constraints(self): """ `file=` is only allowed on a hardcoded list of constraints, unless `_time_varying` is appended to the constraint (i.e. user input) """ allowed_constraints_no_file = list( set(defaults_model.tech_groups.storage.allowed_constraints) .difference(defaults.file_allowed) ) allowed_constraints_file = list( set(defaults_model.tech_groups.storage.allowed_constraints) .intersection(defaults.file_allowed) ) override = lambda param: AttrDict.from_yaml_string( "techs.test_storage.constraints.{}: file=binary_one_day.csv".format(param) ) # should fail: Cannot have `file=` on the following constraints for param in allowed_constraints_no_file: with pytest.raises(exceptions.ModelError) as errors: build_model(override_dict=override(param), scenario='simple_storage,one_day') assert check_error_or_warning( errors, 'Cannot load `{}` from file for configuration'.format(param) ) # should pass: can have `file=` on the following constraints for param in allowed_constraints_file: build_model(override_dict=override(param), scenario='simple_storage,one_day')
def test_incorrect_subset_time(self): """ If subset_time is a list, it must have two entries (start_time, end_time) If subset_time is not a list, it should successfully subset on the given string/integer """ override = lambda param: AttrDict.from_yaml_string( "model.subset_time: {}".format(param) ) # should fail: one string in list with pytest.raises(exceptions.ModelError): build_model(override_dict=override(['2005-01']), scenario='simple_supply') # should fail: three strings in list with pytest.raises(exceptions.ModelError): build_model(override_dict=override(['2005-01-01', '2005-01-02', '2005-01-03']), scenario='simple_supply') # should pass: two string in list as slice model = build_model(override_dict=override(['2005-01-01', '2005-01-07']), scenario='simple_supply') assert all(model.inputs.timesteps.to_index() == pd.date_range('2005-01', '2005-01-07 23:00:00', freq='H')) # should pass: one integer/string model = build_model(override_dict=override('2005-01'), scenario='simple_supply') assert all(model.inputs.timesteps.to_index() == pd.date_range('2005-01', '2005-01-31 23:00:00', freq='H')) # should fail: time subset out of range of input data with pytest.raises(KeyError): build_model(override_dict=override('2005-03'), scenario='simple_supply') # should fail: time subset out of range of input data with pytest.raises(exceptions.ModelError): build_model(override_dict=override(['2005-02-01', '2005-02-05']), scenario='simple_supply')
def run(model_data, timings, build_only=False): """ Parameters ---------- model_data : xarray.Dataset Pre-processed dataset of Calliope model data. timings : dict Stores timings of various stages of model processing. build_only : bool, optional If True, the backend only constructs its in-memory representation of the problem rather than solving it. Used for debugging and testing. """ BACKEND = {'pyomo': run_pyomo} INTERFACE = {'pyomo': pyomo_interface} run_config = AttrDict.from_yaml_string(model_data.attrs['run_config']) if run_config['mode'] == 'plan': results, backend = run_plan(model_data, timings, backend=BACKEND[run_config.backend], build_only=build_only) elif run_config['mode'] == 'operate': results, backend = run_operate(model_data, timings, backend=BACKEND[run_config.backend], build_only=build_only) return results, backend, INTERFACE[ run_config.backend].BackendInterfaceMethods
def test_valid_scenarios(self): """ Test that valid scenario definition raises no error and results in applied scenario. """ override = AttrDict.from_yaml_string(""" scenarios: scenario_1: ['one', 'two'] overrides: one: techs.test_supply_gas.constraints.energy_cap_max: 20 two: techs.test_supply_elec.constraints.energy_cap_max: 20 locations: 0: techs: test_supply_gas: test_supply_elec: test_demand_elec: """) model = build_model(override_dict=override, scenario='scenario_1') assert model._model_run.locations[ '0'].techs.test_supply_gas.constraints.energy_cap_max == 20 assert model._model_run.locations[ '0'].techs.test_supply_elec.constraints.energy_cap_max == 20
def test_union_replacement(self, attr_dict): d = attr_dict d_new = AttrDict.from_yaml_string(""" c: {_REPLACE_: foo} """) d.union(d_new, allow_override=True, allow_replacement=True) assert d.c == "foo"
def test_incorrect_subset_time(self): """ If subset_time is a list, it must have two entries (start_time, end_time) If subset_time is not a list, it should successfully subset on the given string/integer """ override = lambda param: AttrDict.from_yaml_string( "model.subset_time: {}".format(param) ) # should fail: one string in list with pytest.raises(exceptions.ModelError): build_model(override_dict=override(['2005-01']), override_groups='simple_supply') # should fail: three strings in list with pytest.raises(exceptions.ModelError): build_model(override_dict=override(['2005-01-01', '2005-01-02', '2005-01-03']), override_groups='simple_supply') # should pass: two string in list as slice model = build_model(override_dict=override(['2005-01-01', '2005-01-07']), override_groups='simple_supply') assert all(model.inputs.timesteps.to_index() == pd.date_range('2005-01', '2005-01-07 23:00:00', freq='H')) # should pass: one integer/string model = build_model(override_dict=override('2005-01'), override_groups='simple_supply') assert all(model.inputs.timesteps.to_index() == pd.date_range('2005-01', '2005-01-31 23:00:00', freq='H')) # should fail: time subset out of range of input data with pytest.raises(KeyError): build_model(override_dict=override('2005-03'), override_groups='simple_supply') # should fail: time subset out of range of input data with pytest.raises(exceptions.ModelError): build_model(override_dict=override(['2005-02-01', '2005-02-05']), override_groups='simple_supply')
def test_valid_scenarios(self): """ Test that valid scenario definition raises no error and results in applied scenario. """ override = AttrDict.from_yaml_string( """ scenarios: scenario_1: ['one', 'two'] overrides: one: techs.test_supply_gas.constraints.energy_cap_max: 20 two: techs.test_supply_elec.constraints.energy_cap_max: 20 locations: 0: techs: test_supply_gas: test_supply_elec: test_demand_elec: """ ) model = build_model(override_dict=override, scenario='scenario_1') assert model._model_run.locations['0'].techs.test_supply_gas.constraints.energy_cap_max == 20 assert model._model_run.locations['0'].techs.test_supply_elec.constraints.energy_cap_max == 20
def test_nested_import(self, yaml_file): with tempfile.TemporaryDirectory() as tempdir: imported_file = os.path.join(tempdir, "test_import.yaml") imported_yaml = """ somekey: 1 anotherkey: 2 """ with open(imported_file, "w") as f: f.write(imported_yaml) yaml_string = """ foobar: import: - {} foo: bar: 1 baz: 2 3: 4: 5 """.format(imported_file) d = AttrDict.from_yaml_string(yaml_string, resolve_imports="foobar") assert "foobar.somekey" in d.keys_nested() assert d.get_key("foobar.anotherkey") == 2
def postprocess_model_results(results, model_data, timings): """ Adds additional post-processed result variables to the given model results in-place. Model must have solved successfully. Parameters ---------- results : xarray Dataset Output from the solver backend model_data : xarray Dataset Calliope model data, stored as calliope.Model()._model_data timings : dict Calliope timing dictionary, stored as calliope.Model()._timings Returns ------- results : xarray Dataset Input results Dataset, with additional DataArray variables and removed all instances of unreasonably low numbers (set by zero_threshold) """ log_time(logger, timings, "post_process_start", comment="Postprocessing: started") if model_data.attrs['scale']: scale(model_data, lambda x: 1 / x) results['scale'] = model_data['scale'] scale(results, lambda x: 1 / x) run_config = AttrDict.from_yaml_string(model_data.attrs["run_config"]) results["capacity_factor"] = capacity_factor(results, model_data) results["systemwide_capacity_factor"] = systemwide_capacity_factor( results, model_data) results["systemwide_levelised_cost"] = systemwide_levelised_cost( results, model_data) results["total_levelised_cost"] = systemwide_levelised_cost(results, model_data, total=True) results = clean_results(results, run_config.get("zero_threshold", 0), timings) log_time( logger, timings, "post_process_end", time_since_run_start=True, comment="Postprocessing: ended", ) if "run_solution_returned" in timings.keys(): results.attrs["solution_time"] = ( timings["run_solution_returned"] - timings["run_start"]).total_seconds() results.attrs["time_finished"] = timings[ "run_solution_returned"].strftime("%Y-%m-%d %H:%M:%S") return results
def test_order_of_subdicts(self): d = AttrDict.from_yaml_string(""" A.B.C: 10 A.B: E: 20 """) assert d.A.B.C == 10 assert d.A.B.E == 20
def test_do_not_resolve_imports(self): yaml_string = """ import: ['somefile.yaml'] """ d = AttrDict.from_yaml_string(yaml_string, resolve_imports=False) # Should not raise an error about a missing file, as we ask for # imports not to be resolved assert d["import"] == ["somefile.yaml"]
def test_milp_constraints(self): """ If `units` is defined, but not `energy_cap_per_unit`, throw an error """ # should fail: no energy_cap_per_unit override1 = AttrDict.from_yaml_string("techs.test_supply_elec.constraints.units_max: 4") with pytest.raises(exceptions.ModelError): build_model(override_dict=override1, scenario='simple_supply,one_day') # should pass: energy_cap_per_unit given override2 = AttrDict.from_yaml_string(""" techs.test_supply_elec.constraints: units_max: 4 energy_cap_per_unit: 5 """) build_model(override_dict=override2, scenario='simple_supply,one_day')
def test_incorrect_date_format(self): """ Test the date parser catches a different date format from file than user input/default (inc. if it is just one line of a file that is incorrect) """ # should pass: changing datetime format from default override1 = AttrDict.from_yaml_string(""" model.timeseries_dateformat: "%d/%m/%Y %H:%M:%S" techs.test_demand_heat.constraints.resource: file=demand_heat_diff_dateformat.csv techs.test_demand_elec.constraints.resource: file=demand_heat_diff_dateformat.csv """) model = build_model(override_dict=override1, override_groups='simple_conversion') assert all(model.inputs.timesteps.to_index() == pd.date_range( '2005-01', '2005-02-01 23:00:00', freq='H')) # should fail: wrong dateformat input for one file override2 = AttrDict.from_yaml_string(""" techs.test_demand_heat.constraints.resource: file=demand_heat_diff_dateformat.csv """) with pytest.raises(exceptions.ModelError): build_model(override_dict=override2, override_groups='simple_conversion') # should fail: wrong dateformat input for all files override3 = AttrDict.from_yaml_string(""" model.timeseries_dateformat: "%d/%m/%Y %H:%M:%S" """) with pytest.raises(exceptions.ModelError): build_model(override_dict=override3, override_groups='simple_supply') # should fail: one value wrong in file override4 = AttrDict.from_yaml_string(""" techs.test_demand_heat.constraints.resource: file=demand_heat_wrong_dateformat.csv """) # check in output error that it points to: 07/01/2005 10:00:00 with pytest.raises(exceptions.ModelError): build_model(override_dict=override4, override_groups='simple_conversion')
def test_milp_constraints(self): """ If `units` is defined, but not `energy_cap_per_unit`, throw an error """ # should fail: no energy_cap_per_unit override1 = AttrDict.from_yaml_string("techs.test_supply_elec.constraints.units_max: 4") with pytest.raises(exceptions.ModelError): build_model(override_dict=override1, override_groups='simple_supply,one_day') # should pass: energy_cap_per_unit given override2 = AttrDict.from_yaml_string(""" techs.test_supply_elec.constraints: units_max: 4 energy_cap_per_unit: 5 """) build_model(override_dict=override2, override_groups='simple_supply,one_day')
def test_tech_as_parent(self): """ All technologies and technology groups must specify a parent """ override1 = AttrDict.from_yaml_string(""" techs.test_supply_tech_parent: essentials: name: Supply tech carrier: gas parent: test_supply_elec constraints: energy_cap_max: 10 resource: .inf locations.1.test_supply_tech_parent: """) with pytest.raises(exceptions.ModelError) as error: build_model(override_dict=override1, scenario='simple_supply,one_day') check_error_or_warning( error, 'tech `test_supply_tech_parent` has another tech as a parent') override2 = AttrDict.from_yaml_string(""" tech_groups.test_supply_group: essentials: carrier: gas parent: test_supply_elec constraints: energy_cap_max: 10 resource: .inf techs.test_supply_tech_parent.essentials: name: Supply tech parent: test_supply_group locations.1.test_supply_tech_parent: """) with pytest.raises(exceptions.ModelError) as error: build_model(override_dict=override2, scenario='simple_supply,one_day') check_error_or_warning( error, 'tech_group `test_supply_group` has a tech as a parent')
def test_tech_as_parent(self): """ All technologies and technology groups must specify a parent """ override1 = AttrDict.from_yaml_string( """ techs.test_supply_tech_parent: essentials: name: Supply tech carrier: gas parent: test_supply_elec constraints: energy_cap_max: 10 resource: .inf locations.1.test_supply_tech_parent: """ ) with pytest.raises(exceptions.ModelError) as error: build_model(override_dict=override1, scenario='simple_supply,one_day') check_error_or_warning(error, 'tech `test_supply_tech_parent` has another tech as a parent') override2 = AttrDict.from_yaml_string( """ tech_groups.test_supply_group: essentials: carrier: gas parent: test_supply_elec constraints: energy_cap_max: 10 resource: .inf techs.test_supply_tech_parent.essentials: name: Supply tech parent: test_supply_group locations.1.test_supply_tech_parent: """ ) with pytest.raises(exceptions.ModelError) as error: build_model(override_dict=override2, scenario='simple_supply,one_day') check_error_or_warning(error, 'tech_group `test_supply_group` has a tech as a parent')
def _init_from_model_data(self, model_data): if "_model_run" in model_data.attrs: self._model_run = AttrDict.from_yaml_string(model_data.attrs["_model_run"]) del model_data.attrs["_model_run"] if "_debug_data" in model_data.attrs: self._debug_data = AttrDict.from_yaml_string( model_data.attrs["_debug_data"] ) del model_data.attrs["_debug_data"] self._model_data = model_data self._add_model_data_methods() log_time( logger, self._timings, "model_data_loaded", comment="Model: loaded model_data", )
def test_union_preserves_comments(self, yaml_file): d = AttrDict.from_yaml(yaml_file) d_new = AttrDict.from_yaml_string(""" test: 1 # And a comment somekey: bar: baz: 2 # Another comment """) d.union(d_new) assert d.get_comments( 'somekey.bar.baz')['inline'] == '# Another comment\n'
def test_unknown_carrier_tier(self): """ User can only use 'carrier_' + ['in', 'out', 'in_2', 'out_2', 'in_3', 'out_3', 'ratios'] """ override1 = AttrDict.from_yaml_string(""" techs.test_supply_elec.essentials.carrier_1: power """) with pytest.raises(exceptions.ModelError): build_model(override_dict=override1, override_groups='simple_supply,one_day') override2 = AttrDict.from_yaml_string(""" techs.test_conversion_plus.essentials.carrier_out_4: power """) with pytest.raises(exceptions.ModelError): build_model(override_dict=override2, override_groups='simple_conversion_plus,one_day')
def test_defining_non_allowed_costs(self): """ A technology within an abstract base technology can only define a subset of hardcoded costs, anything else will not be implemented, so are not allowed for that technology. This includes misspellings """ # should fail: storage_cap_max not allowed for supply tech override = AttrDict.from_yaml_string(""" techs.test_supply_elec.costs.monetary.storage_cap: 10 """) with pytest.raises(exceptions.ModelError): build_model(override_dict=override, override_groups='simple_supply,one_day') # should fail: om_prod not allowed for demand tech override = AttrDict.from_yaml_string(""" techs.test_demand_elec.costs.monetary.om_prod: 10 """) with pytest.raises(exceptions.ModelError): build_model(override_dict=override, override_groups='simple_supply,one_day')
def __init__(self, initial_dict, initial_yaml_string, on_changed=None): if initial_yaml_string is not None: initial_dict = AttrDict.from_yaml_string(initial_yaml_string).as_dict() super().__init__(initial_dict) self.on_changed = on_changed for k, v in initial_dict.items(): if isinstance(v, dict): super().__setitem__(k, ObservedDict(v, None, on_changed=self.notify)) self.notify()
def get_result_array(backend_model, model_data): """ From a Pyomo model object, extract decision variable data and return it as an xarray Dataset. Any rogue input parameters that are constructed inside the backend (instead of being passed by calliope.Model().inputs) are also added to calliope.Model()._model_data in-place. """ subsets_config = AttrDict.from_yaml_string(model_data.attrs["subsets"]) def _get_dim_order(foreach): return tuple([i for i in model_data.dims.keys() if i in foreach]) all_variables = { i.name: get_var( backend_model, i.name, dims=_get_dim_order(subsets_config.variables[i.name].foreach), ) for i in backend_model.component_objects(ctype=po.Var) } # Add in expressions, which are combinations of variables (e.g. costs) all_variables.update({ i.name: get_var( backend_model, i.name, dims=_get_dim_order(subsets_config.expressions[i.name].foreach), expr=True, ) for i in backend_model.component_objects(ctype=po.Expression) }) # Get any parameters that did not appear in the user's model.inputs Dataset all_params = { i.name: get_var(backend_model, i.name, expr=True) for i in backend_model.component_objects( ctype=po.base.param.IndexedParam) if i.name not in model_data.data_vars.keys() and "objective_" not in i.name } results = string_to_datetime( backend_model, reorganise_xarray_dimensions(xr.Dataset(all_variables))) if all_params: additional_inputs = reorganise_xarray_dimensions( xr.Dataset(all_params)) for var in additional_inputs.data_vars: additional_inputs[var].attrs["is_result"] = 0 model_data.update(additional_inputs) model_data = string_to_datetime(backend_model, model_data) results = string_to_datetime(backend_model, results) return results
def postprocess_model_results(results, model_data, timings): """ Adds additional post-processed result variables to the given model results in-place. Model must have solved successfully. Parameters ---------- results : xarray Dataset Output from the solver backend model_data : xarray Dataset Calliope model data, stored as calliope.Model()._model_data timings : dict Calliope timing dictionary, stored as calliope.Model()._timings Returns ------- results : xarray Dataset Input results Dataset, with additional DataArray variables and removed all instances of unreasonably low numbers (set by zero_threshold) """ log_time(logger, timings, 'post_process_start', comment='Postprocessing: started') run_config = AttrDict.from_yaml_string(model_data.attrs['run_config']) results['capacity_factor'] = capacity_factor(results, model_data) results['systemwide_capacity_factor'] = systemwide_capacity_factor( results, model_data) results['systemwide_levelised_cost'] = systemwide_levelised_cost( results, model_data) results['total_levelised_cost'] = systemwide_levelised_cost(results, model_data, total=True) results = clean_results(results, run_config.get('zero_threshold', 0), timings) log_time(logger, timings, 'post_process_end', time_since_run_start=True, comment='Postprocessing: ended') if 'run_solution_returned' in timings.keys(): results.attrs['solution_time'] = ( timings['run_solution_returned'] - timings['run_start']).total_seconds() results.attrs['time_finished'] = ( timings['run_solution_returned'].strftime('%Y-%m-%d %H:%M:%S')) return results
def test_negative_cost_unassigned_cap(self): """ Any negative cost associated with a capacity (e.g. cost_energy_cap) must be applied to a capacity iff the upper bound of that capacity has been defined """ # should fail: resource_cap cost is negtive, resource_cap_max is infinite override = AttrDict.from_yaml_string( "techs.test_supply_plus.costs.monetary.resource_cap: -10") with pytest.raises(exceptions.ModelError): build_model(override_dict=override, override_groups='simple_supply_plus,one_day') # should fail: storage_cap cost is negative, storage_cap_max is infinite override = AttrDict.from_yaml_string(""" techs.test_storage: constraints.storage_cap_max: .inf costs.monetary.storage_cap: -10 """) with pytest.raises(exceptions.ModelError): build_model(override_dict=override, override_groups='simple_storage,one_day')
def test_negative_cost_unassigned_cap(self): """ Any negative cost associated with a capacity (e.g. cost_energy_cap) must be applied to a capacity iff the upper bound of that capacity has been defined """ # should fail: resource_cap cost is negtive, resource_cap_max is infinite override = AttrDict.from_yaml_string( "techs.test_supply_plus.costs.monetary.resource_cap: -10" ) with pytest.raises(exceptions.ModelError): build_model(override_dict=override, scenario='simple_supply_plus,one_day') # should fail: storage_cap cost is negative, storage_cap_max is infinite override = AttrDict.from_yaml_string( """ techs.test_storage: constraints.storage_cap_max: .inf costs.monetary.storage_cap: -10 """ ) with pytest.raises(exceptions.ModelError): build_model(override_dict=override, scenario='simple_storage,one_day')
def test_defining_non_allowed_costs(self): """ A technology within an abstract base technology can only define a subset of hardcoded costs, anything else will not be implemented, so are not allowed for that technology. This includes misspellings """ # should fail: storage_cap_max not allowed for supply tech override = AttrDict.from_yaml_string( """ techs.test_supply_elec.costs.monetary.storage_cap: 10 """ ) with pytest.raises(exceptions.ModelError): build_model(override_dict=override, scenario='simple_supply,one_day') # should fail: om_prod not allowed for demand tech override = AttrDict.from_yaml_string( """ techs.test_demand_elec.costs.monetary.om_prod: 10 """ ) with pytest.raises(exceptions.ModelError): build_model(override_dict=override, scenario='simple_supply,one_day')
def test_invalid_scenarios_str(self): """ Test that invalid scenario definition raises appropriate error """ override = AttrDict.from_yaml_string( """ scenarios: scenario_1: 'foo1,foo2' """ ) with pytest.raises(exceptions.ModelError) as error: build_model(override_dict=override, scenario='scenario_1') assert check_error_or_warning(error, 'Scenario definition must be a list of override names.')
def test_unknown_carrier_tier(self): """ User can only use 'carrier_' + ['in', 'out', 'in_2', 'out_2', 'in_3', 'out_3', 'ratios'] """ override1 = AttrDict.from_yaml_string( """ techs.test_supply_elec.essentials.carrier_1: power """ ) with pytest.raises(exceptions.ModelError): build_model(override_dict=override1, scenario='simple_supply,one_day') override2 = AttrDict.from_yaml_string( """ techs.test_conversion_plus.essentials.carrier_out_4: power """ ) with pytest.raises(exceptions.ModelError): build_model(override_dict=override2, scenario='simple_conversion_plus,one_day')
def test_scenario_name_overlaps_overrides(self): """ Test that a scenario name cannot be a combination of override names """ override = AttrDict.from_yaml_string( """ scenarios: 'simple_supply,group_share_energy_cap_min': 'foobar' """ ) with pytest.raises(exceptions.ModelError) as error: build_model(override_dict=override, scenario='simple_supply,group_share_energy_cap_min') assert check_error_or_warning(error, 'Manually defined scenario cannot be a combination of override names.')
def test_missing_required_constraints(self): """ A technology within an abstract base technology must define a subset of hardcoded constraints in order to function """ # should fail: missing one of ['energy_cap_max', 'energy_cap_equals', 'energy_cap_per_unit'] override_supply1 = AttrDict.from_yaml_string( """ techs: supply_missing_constraint: essentials: parent: supply carrier: electricity name: supply missing constraint constraints: resource_area_max: 10 locations.1.techs.supply_missing_constraint: """ ) with pytest.raises(exceptions.ModelError): build_model(override_dict=override_supply1, scenario='simple_supply,one_day') # should pass: giving one of ['energy_cap_max', 'energy_cap_equals', 'energy_cap_per_unit'] override_supply2 = AttrDict.from_yaml_string( """ techs: supply_missing_constraint: essentials: parent: supply carrier: electricity name: supply missing constraint constraints.energy_cap_max: 10 locations.1.techs.supply_missing_constraint: """ ) build_model(override_dict=override_supply2, scenario='simple_supply,one_day')
def test_abstract_base_tech_group_override(self): """ Abstract base technology groups can be overridden """ override = AttrDict.from_yaml_string( """ tech_groups: supply: constraints: lifetime: 25 locations: 1.techs.test_supply_elec: 1.techs.test_demand_elec: """ ) build_model(override_dict=override, scenario='one_day')
def test_missing_constraints(self): """ A technology must define at least one constraint. """ override = AttrDict.from_yaml_string( """ techs: supply_missing_constraint: essentials: parent: supply carrier: electricity name: supply missing constraint locations.1.techs.supply_missing_constraint: """ ) with pytest.raises(exceptions.ModelError): build_model(override_dict=override, scenario='simple_supply,one_day')
def test_undefined_carriers(self): """ Test that user has input either carrier or carrier_in/_out for each tech """ override = AttrDict.from_yaml_string( """ techs: test_undefined_carrier: essentials: parent: supply name: test constraints: resource: .inf energy_cap_max: .inf locations.1.techs.test_undefined_carrier: """ ) with pytest.raises(exceptions.ModelError): build_model(override_dict=override, scenario='simple_supply,one_day')
def combine_overrides(config_model, overrides): override_dict = AttrDict() for override in overrides: try: yaml_string = config_model.overrides[override].to_yaml() override_with_imports = AttrDict.from_yaml_string(yaml_string) except KeyError: raise exceptions.ModelError( 'Override `{}` is not defined.'.format(override) ) try: override_dict.union(override_with_imports, allow_override=False) except KeyError as e: raise exceptions.ModelError( str(e)[1:-1] + '. Already specified but defined again in ' 'override `{}`.'.format(override) ) return override_dict
def test_unspecified_parent(self): """ All technologies and technology groups must specify a parent """ override = AttrDict.from_yaml_string( """ techs.test_supply_no_parent: essentials: name: Supply tech carrier: gas constraints: energy_cap_max: 10 resource: .inf locations.1.test_supply_no_parent: """ ) with pytest.raises(KeyError): build_model(override_dict=override, scenario='simple_supply,one_day')
def test_name_overlap(self): """ No tech may have the same identifier as a tech group """ override = AttrDict.from_yaml_string( """ techs: supply: essentials: name: Supply tech carrier: gas parent: supply constraints: energy_cap_max: 10 resource: .inf locations: 1.techs.supply: 0.techs.supply: """ ) with pytest.raises(exceptions.ModelError): build_model(override_dict=override, scenario='one_day')
def apply_overrides(config, scenario=None, override_dict=None): """ Generate processed Model configuration, applying any scenarios overrides. Parameters ---------- config : AttrDict a model configuration AttrDict scenario : str, optional override_dict : str or dict or AttrDict, optional If a YAML string, converted to AttrDict """ debug_comments = AttrDict() base_model_config_file = os.path.join( os.path.dirname(calliope.__file__), 'config', 'model.yaml' ) config_model = AttrDict.from_yaml(base_model_config_file) # Interpret timeseries_data_path as relative config.model.timeseries_data_path = relative_path( config.config_path, config.model.timeseries_data_path ) # The input files are allowed to override other model defaults config_model.union(config, allow_override=True) # First pass of applying override dict before applying scenarios, # so that can override scenario definitions by override_dict if override_dict: if isinstance(override_dict, str): override_dict = AttrDict.from_yaml_string(override_dict) elif not isinstance(override_dict, AttrDict): override_dict = AttrDict(override_dict) warnings = checks.check_overrides(config_model, override_dict) exceptions.print_warnings_and_raise_errors(warnings=warnings) config_model.union( override_dict, allow_override=True, allow_replacement=True ) if scenario: scenarios = config_model.get('scenarios', {}) if scenario in scenarios.keys(): # Manually defined scenario names cannot be the same as single # overrides or any combination of semicolon-delimited overrides if all([i in config_model.get('overrides', {}) for i in scenario.split(',')]): raise exceptions.ModelError( 'Manually defined scenario cannot be a combination of override names.' ) if not isinstance(scenarios[scenario], list): raise exceptions.ModelError( 'Scenario definition must be a list of override names.' ) overrides = [str(i) for i in scenarios[scenario]] logger.info( 'Using scenario `{}` leading to the application of ' 'overrides `{}`.'.format(scenario, overrides) ) else: overrides = str(scenario).split(',') logger.info( 'Applying the following overrides without a ' 'specific scenario name: {}'.format(overrides) ) overrides_from_scenario = combine_overrides(config_model, overrides) warnings = checks.check_overrides(config_model, overrides_from_scenario) exceptions.print_warnings_and_raise_errors(warnings=warnings) config_model.union( overrides_from_scenario, allow_override=True, allow_replacement=True ) for k, v in overrides_from_scenario.as_dict_flat().items(): debug_comments.set_key( '{}'.format(k), 'Applied from override') else: overrides = [] # Second pass of applying override dict after applying scenarios, # so that scenario-based overrides are overridden by override_dict! if override_dict: config_model.union( override_dict, allow_override=True, allow_replacement=True ) for k, v in override_dict.as_dict_flat().items(): debug_comments.set_key( '{}'.format(k), 'Overridden via override dictionary.') return config_model, debug_comments, overrides, scenario