def _add_model_data_methods(self): self.inputs = self._model_data.filter_by_attrs(is_result=0) self.results = self._model_data.filter_by_attrs(is_result=1) self.model_config = UpdateObserverDict( initial_yaml_string=self._model_data.attrs.get("model_config", "{}"), name="model_config", observer=self._model_data, ) self.run_config = UpdateObserverDict( initial_yaml_string=self._model_data.attrs.get("run_config", "{}"), name="run_config", observer=self._model_data, ) self.subsets = UpdateObserverDict( initial_yaml_string=self._model_data.attrs.get("subsets", "{}"), name="subsets", observer=self._model_data, flat=True, ) results = self._model_data.filter_by_attrs(is_result=1) if len(results.data_vars) > 0: self.results = results log_time( logger, self._timings, "model_data_loaded", comment="Model: loaded model_data", )
def _init_from_model_data(self, model_data): if '_model_run' in model_data.attrs: self._model_run = AttrDict.from_yaml_string( model_data.attrs['_model_run']) del model_data.attrs['_model_run'] if '_debug_data' in model_data.attrs: self._debug_data = AttrDict.from_yaml_string( model_data.attrs['_debug_data']) del model_data.attrs['_debug_data'] self._model_data = model_data self.inputs = self._model_data.filter_by_attrs(is_result=0) self.model_config = UpdateObserverDict( initial_yaml_string=model_data.attrs.get('model_config', '{}'), name='model_config', observer=self._model_data) self.run_config = UpdateObserverDict( initial_yaml_string=model_data.attrs.get('run_config', '{}'), name='run_config', observer=self._model_data) results = self._model_data.filter_by_attrs(is_result=1) if len(results.data_vars) > 0: self.results = results log_time(logger, self._timings, 'model_data_loaded', comment='Model: loaded model_data')
def _init_from_model_data(self, model_data): if "_model_run" in model_data.attrs: self._model_run = AttrDict.from_yaml_string( model_data.attrs["_model_run"]) del model_data.attrs["_model_run"] if "_debug_data" in model_data.attrs: self._debug_data = AttrDict.from_yaml_string( model_data.attrs["_debug_data"]) del model_data.attrs["_debug_data"] self._model_data = model_data self.inputs = self._model_data.filter_by_attrs(is_result=0) self.model_config = UpdateObserverDict( initial_yaml_string=model_data.attrs.get("model_config", "{}"), name="model_config", observer=self._model_data, ) self.run_config = UpdateObserverDict( initial_yaml_string=model_data.attrs.get("run_config", "{}"), name="run_config", observer=self._model_data, ) results = self._model_data.filter_by_attrs(is_result=1) if len(results.data_vars) > 0: self.results = results log_time( logger, self._timings, "model_data_loaded", comment="Model: loaded model_data", )
def _init_from_model_run(self, model_run, debug_data, debug): self._model_run = model_run log_time( logger, self._timings, "model_run_creation", comment="Model: preprocessing stage 1 (model_run)", ) model_data_factory = ModelDataFactory(model_run) ( model_data_pre_clustering, model_data, data_pre_time, stripped_keys, ) = model_data_factory() self._model_data_pre_clustering = model_data_pre_clustering self._model_data = model_data if debug: self._debug_data = debug_data self._model_data_pre_time = data_pre_time self._model_data_stripped_keys = stripped_keys self.inputs = self._model_data.filter_by_attrs(is_result=0) log_time( logger, self._timings, "model_data_original_creation", comment="Model: preprocessing stage 2 (model_data)", ) # Ensure model and run attributes of _model_data update themselves model_config = { k: v for k, v in model_run.get("model", {}).items() if k != "file_allowed" } self.model_config = UpdateObserverDict(initial_dict=model_config, name="model_config", observer=self._model_data) self.run_config = UpdateObserverDict( initial_dict=model_run.get("run", {}), name="run_config", observer=self._model_data, ) self.subsets = UpdateObserverDict( initial_dict=model_run.get("subsets").as_dict_flat(), name="subsets", observer=self._model_data, ) log_time( logger, self._timings, "model_data_creation", comment="Model: preprocessing complete", )
def _init_from_model_run(self, model_run, debug_data): self._model_run = model_run self._debug_data = debug_data log_time( logger, self._timings, "model_run_creation", comment="Model: preprocessing stage 1 (model_run)", ) self._model_data_original = build_model_data(model_run) log_time( logger, self._timings, "model_data_original_creation", comment="Model: preprocessing stage 2 (model_data)", ) random_seed = self._model_run.get_key("model.random_seed", None) if random_seed: np.random.seed(seed=random_seed) # After setting the random seed, time clustering can take place time_config = model_run.model.get("time", None) if not time_config: _model_data = self._model_data_original else: _model_data = apply_time_clustering(self._model_data_original, model_run) self._model_data = final_timedimension_processing(_model_data) log_time( logger, self._timings, "model_data_creation", comment="Model: preprocessing complete", ) # Ensure model and run attributes of _model_data update themselves for var in self._model_data.data_vars: self._model_data[var].attrs["is_result"] = 0 self.inputs = self._model_data.filter_by_attrs(is_result=0) model_config = { k: v for k, v in model_run.get("model", {}).items() if k != "file_allowed" } self.model_config = UpdateObserverDict(initial_dict=model_config, name="model_config", observer=self._model_data) self.run_config = UpdateObserverDict( initial_dict=model_run.get("run", {}), name="run_config", observer=self._model_data, )
def model_data(self): model_data = xr.Dataset( coords={ dim: ["foo", "bar"] if dim != "techs" else ["foo", "bar", "foobar", "foobaz"] for dim in BASE_DIMS }, data_vars={ "node_tech": ( ["nodes", "techs"], np.random.choice(a=[np.nan, True], size=(2, 4)), ), "carrier": ( ["carrier_tiers", "carriers", "techs"], np.random.choice(a=[np.nan, True], size=(2, 2, 4)), ), "with_inf": ( ["nodes", "techs"], [[1.0, np.nan, 1.0, 3], [np.inf, 2.0, True, np.nan]], ), "all_inf": (["nodes", "techs"], np.ones((2, 4)) * np.inf), "all_nan": (["nodes", "techs"], np.ones((2, 4)) * np.nan), "inheritance": ( ["nodes", "techs"], [ ["foo.bar", "boo", "baz", "boo"], ["bar", "ar", "baz.boo", "foo.boo"], ], ), }, ) UpdateObserverDict( initial_dict=AttrDict({"foo": True, "baz": {"bar": "foobar"}}), name="run_config", observer=model_data, ) UpdateObserverDict( initial_dict={"foz": 0}, name="model_config", observer=model_data ) return model_data
def run_operate(model_data, timings, backend, build_only): """ For use when mode is 'operate', to allow the model to be built, edited, and iteratively run within Pyomo. """ log_time( logger, timings, "run_start", comment="Backend: starting model run in operational mode", ) defaults = UpdateObserverDict( initial_yaml_string=model_data.attrs["defaults"], name="defaults", observer=model_data, ) run_config = UpdateObserverDict( initial_yaml_string=model_data.attrs["run_config"], name="run_config", observer=model_data, ) # New param defaults = old maximum param defaults (e.g. energy_cap gets default from energy_cap_max) operate_params = { k.replace("_max", ""): v for k, v in defaults.items() if k.endswith("_max") } operate_params[ "purchased"] = 0 # no _max to work from here, so we hardcode a default defaults.update(operate_params) # Capacity results (from plan mode) can be used as the input to operate mode if any(model_data.filter_by_attrs( is_result=1).data_vars) and run_config.get( "operation.use_cap_results", False): # Anything with is_result = 1 will be ignored in the Pyomo model for varname, varvals in model_data.data_vars.items(): if varname in operate_params.keys(): varvals.attrs["is_result"] = 1 varvals.attrs["operate_param"] = 1 else: cap_max = xr.merge([ v.rename(k.replace("_max", "")) for k, v in model_data.data_vars.items() if "_max" in k ]) cap_equals = xr.merge([ v.rename(k.replace("_equals", "")) for k, v in model_data.data_vars.items() if "_equals" in k ]) caps = cap_max.update(cap_equals) for cap in caps.data_vars.values(): cap.attrs["is_result"] = 1 cap.attrs["operate_param"] = 1 model_data.update(caps) comments, warnings, errors = checks.check_operate_params(model_data) exceptions.print_warnings_and_raise_errors(warnings=warnings, errors=errors) # Initialize our variables solver = run_config["solver"] solver_io = run_config.get("solver_io", None) solver_options = run_config.get("solver_options", None) save_logs = run_config.get("save_logs", None) window = run_config["operation"]["window"] horizon = run_config["operation"]["horizon"] window_to_horizon = horizon - window # get the cumulative sum of timestep resolution, to find where we hit our window and horizon timestep_cumsum = model_data.timestep_resolution.cumsum( "timesteps").to_pandas() # get the timesteps at which we start and end our windows window_ends = timestep_cumsum.where((timestep_cumsum % window == 0) | ( timestep_cumsum == timestep_cumsum[-1])) window_starts = timestep_cumsum.where((~np.isnan(window_ends.shift(1))) | ( timestep_cumsum == timestep_cumsum[0])).dropna() window_ends = window_ends.dropna() horizon_ends = timestep_cumsum[timestep_cumsum.isin(window_ends.values + window_to_horizon)] if not any(window_starts): raise exceptions.ModelError( "Not enough timesteps or incorrect timestep resolution to run in " "operational mode with an optimisation window of {}".format( window)) # We will only update timseries parameters timeseries_data_vars = [ k for k, v in model_data.data_vars.items() if "timesteps" in v.dims and v.attrs["is_result"] == 0 ] # Loop through each window, solve over the horizon length, and add result to # result_array we only go as far as the end of the last horizon, which may # clip the last bit of data result_array = [] # track whether each iteration finds an optimal solution or not terminations = [] if build_only: iterations = [0] else: iterations = range(len(window_starts)) for i in iterations: start_timestep = window_starts.index[i] # Build full model in first instance if i == 0: warmstart = False end_timestep = horizon_ends.index[i] timesteps = slice(start_timestep, end_timestep) window_model_data = model_data.loc[dict(timesteps=timesteps)] log_time( logger, timings, "model_gen_1", comment="Backend: generating initial model", ) backend_model = backend.generate_model(window_model_data) # Build the full model in the last instance(s), # where number of timesteps is less than the horizon length elif i > len(horizon_ends) - 1: warmstart = False end_timestep = window_ends.index[i] timesteps = slice(start_timestep, end_timestep) window_model_data = model_data.loc[dict(timesteps=timesteps)] log_time( logger, timings, "model_gen_{}".format(i + 1), comment=( "Backend: iteration {}: generating new model for " "end of timeseries, with horizon = {} timesteps".format( i + 1, window_ends[i] - window_starts[i])), ) backend_model = backend.generate_model(window_model_data) # Update relevent Pyomo Params in intermediate instances else: warmstart = True end_timestep = horizon_ends.index[i] timesteps = slice(start_timestep, end_timestep) window_model_data = model_data.loc[dict(timesteps=timesteps)] log_time( logger, timings, "model_gen_{}".format(i + 1), comment="Backend: iteration {}: updating model parameters". format(i + 1), ) # Pyomo model sees the same timestamps each time, we just change the # values associated with those timestamps for var in timeseries_data_vars: # New values var_series = ( window_model_data[var].to_series().dropna().replace( "inf", np.inf)) # Same timestamps var_series.index = backend_model.__calliope_model_data["data"][ var].keys() var_dict = var_series.to_dict() # Update pyomo Param with new dictionary getattr(backend_model, var).store_values(var_dict) if not build_only: log_time( logger, timings, "model_run_{}".format(i + 1), time_since_run_start=True, comment="Backend: iteration {}: sending model to solver". format(i + 1), ) # After iteration 1, warmstart = True, which should speed up the process # Note: Warmstart isn't possible with GLPK (dealt with later on) _results = backend.solve_model( backend_model, solver=solver, solver_io=solver_io, solver_options=solver_options, save_logs=save_logs, warmstart=warmstart, ) log_time( logger, timings, "run_solver_exit_{}".format(i + 1), time_since_run_start=True, comment="Backend: iteration {}: solver finished running". format(i + 1), ) # xarray dataset is built for each iteration _termination = backend.load_results(backend_model, _results) terminations.append(_termination) _results = backend.get_result_array(backend_model, model_data) # We give back the actual timesteps for this iteration and take a slice # equal to the window length _results["timesteps"] = window_model_data.timesteps.copy() # We always save the window data. Until the last window(s) this will crop # the window_to_horizon timesteps. In the last window(s), optimistion will # only be occurring over a window length anyway _results = _results.loc[dict( timesteps=slice(None, window_ends.index[i]))] result_array.append(_results) # Set up initial storage for the next iteration if "loc_techs_store" in model_data.dims.keys(): storage_initial = _results.storage.loc[{ "timesteps": window_ends.index[i] }].drop("timesteps") model_data["storage_initial"].loc[ storage_initial.coords] = storage_initial.values backend_model.storage_initial.store_values( storage_initial.to_series().dropna().to_dict()) # Set up total operated units for the next iteration if "loc_techs_milp" in model_data.dims.keys(): operated_units = _results.operating_units.sum( "timesteps").astype(np.int) model_data["operated_units"].loc[{}] += operated_units.values backend_model.operated_units.store_values( operated_units.to_series().dropna().to_dict()) log_time( logger, timings, "run_solver_exit_{}".format(i + 1), time_since_run_start=True, comment="Backend: iteration {}: generated solution array". format(i + 1), ) if build_only: results = xr.Dataset() else: # Concatenate results over the timestep dimension to get a single # xarray Dataset of interest results = xr.concat(result_array, dim="timesteps") if all(i == "optimal" for i in terminations): results.attrs["termination_condition"] = "optimal" elif all(i in ["optimal", "feasible"] for i in terminations): results.attrs["termination_condition"] = "feasible" else: results.attrs["termination_condition"] = ",".join(terminations) log_time( logger, timings, "run_solution_returned", time_since_run_start=True, comment="Backend: generated full solution array", ) return results, backend_model
def run_spores(model_data, timings, interface, backend, build_only): """ For use when mode is 'spores', to allow the model to be built, edited, and iteratively run within Pyomo, modifying, at each iteration, the score of each loc::tech in such a way to generate Spatially explicit Practically Optimal RESults (SPORES). """ log_time( logger, timings, "run_start", comment="Backend: starting model run in SPORES mode", ) run_config = UpdateObserverDict( initial_yaml_string=model_data.attrs["run_config"], name="run_config", observer=model_data, ) backend_model = backend.generate_model(model_data) log_time( logger, timings, "run_backend_model_generated", time_since_run_start=True, comment="Backend: model generated", ) n_spores = run_config["spores_options"]["spores_number"] slack = run_config["spores_options"]["slack"] spores_score = run_config["spores_options"]["score_cost_class"] slack_group = run_config["spores_options"]["slack_cost_group"] # Define default scoring function, based on integer scoring method # TODO: make the function to run optional def _cap_loc_score_default(results, subset=None): if subset is None: subset = {} cap_loc_score = split_loc_techs(results["energy_cap"]).loc[subset] cap_loc_score = cap_loc_score.where(cap_loc_score > 1e-3, other=0) cap_loc_score = cap_loc_score.where(cap_loc_score == 0, other=100) return cap_loc_score.to_pandas() # Define function to update "spores_score" after each iteration of the method def _update_spores_score(backend_model, cap_loc_score): loc_tech_score_dict = { (spores_score, "{}::{}".format(i, j)): k for (i, j), k in cap_loc_score.stack().items() if "{}::{}".format(i, j) in model_data.loc_techs_investment_cost } interface.update_pyomo_param(backend_model, "cost_energy_cap", loc_tech_score_dict) def _warn_on_infeasibility(): return exceptions.warn( "Infeasible SPORE detected. Please check your model configuration. " "No more SPORES will be generated.") # Run once for the 'cost-optimal' solution results, backend_model = run_plan(model_data, timings, backend, build_only) if build_only: return results, backend_model # We have what we need, so break out of the loop if results.attrs["termination_condition"] in ["optimal", "feasible"]: results.attrs["objective_function_value"] = backend_model.obj() # Storing results and scores in the specific dictionaries spores_list = [results] cum_scores = _cap_loc_score_default(results) # Set group constraint "cost_max" equal to slacked cost slack_costs = model_data.group_cost_max.loc[{ "group_names_cost_max": slack_group }].dropna("costs") interface.update_pyomo_param( backend_model, "group_cost_max", {(_cost_class, slack_group): results.cost.loc[{ "costs": _cost_class }].sum().item() * (1 + slack) for _cost_class in slack_costs.costs.values}, ) # Modify objective function weights: spores_score -> 1, all others -> 0 interface.update_pyomo_param( backend_model, "objective_cost_class", { spores_score: 1, **{i: 0 for i in slack_costs.costs.values} }, ) # Update "spores_score" based on previous iteration _update_spores_score(backend_model, cum_scores) else: _warn_on_infeasibility() return results, backend_model log_time( logger, timings, "run_solution_returned", time_since_run_start=True, comment="Backend: generated solution array for the cost-optimal case", ) # Iterate over the number of SPORES requested by the user for _spore in range(0, n_spores): results, backend_model = run_plan(model_data, timings, backend, build_only, backend_rerun=backend_model) if results.attrs["termination_condition"] in ["optimal", "feasible"]: results.attrs["objective_function_value"] = backend_model.obj() # Storing results and scores in the specific dictionaries spores_list.append(results) cum_scores += _cap_loc_score_default(results) # Update "spores_score" based on previous iteration _update_spores_score(backend_model, cum_scores) else: _warn_on_infeasibility() break log_time( logger, timings, "run_solution_returned", time_since_run_start=True, comment= "Backend: generated solution array for the cost-optimal case", ) # TODO: make this function work with the spores dimension, # so that postprocessing can take place in core/model.py, as with run_plan and run_operate results = xr.concat(spores_list, dim="spores") return results, backend_model
def check_operate_params(model_data): """ if model mode = `operate`, check for clashes in capacity constraints. In this mode, all capacity constraints are set to parameters in the backend, so can easily lead to model infeasibility if not checked. Returns ------- comments : AttrDict debug output warnings : list possible problems that do not prevent the model run from continuing errors : list serious issues that should raise a ModelError """ defaults = UpdateObserverDict( initial_yaml_string=model_data.attrs["defaults"], name="defaults", observer=model_data, ) run_config = UpdateObserverDict( initial_yaml_string=model_data.attrs["run_config"], name="run_config", observer=model_data, ) warnings, errors = [], [] comments = AttrDict() def _get_param(loc_tech, var): if (_is_in(loc_tech, var) and not model_data[var].loc[loc_tech].isnull().any()): param = model_data[var].loc[loc_tech].values else: param = defaults[var] return param def _is_in(loc_tech, set_or_var): try: model_data[set_or_var].loc[loc_tech] return True except (KeyError, AttributeError): return False def _set_inf_and_warn(loc_tech, var, warnings, warning_text): if np.isinf(model_data[var].loc[loc_tech].item()): return (np.inf, warnings) elif model_data[var].loc[loc_tech].isnull().item(): var_name = model_data[var].loc[loc_tech] = np.inf return (var_name, warnings) else: var_name = model_data[var].loc[loc_tech] = np.inf warnings.append(warning_text) return var_name, warnings # Storage initial is carried over between iterations, so must be defined along with storage if ("loc_techs_store" in model_data.dims.keys() and "storage_initial" not in model_data.data_vars.keys()): model_data["storage_initial"] = xr.DataArray( [0.0 for loc_tech in model_data.loc_techs_store.values], dims="loc_techs_store", ) model_data["storage_initial"].attrs["is_result"] = 0.0 warnings.append( "Initial stored energy not defined, set to zero for all " "loc::techs in loc_techs_store, for use in iterative optimisation") # Operated units is carried over between iterations, so must be defined in a milp model if ("loc_techs_milp" in model_data.dims.keys() and "operated_units" not in model_data.data_vars.keys()): model_data["operated_units"] = xr.DataArray( [0 for loc_tech in model_data.loc_techs_milp.values], dims="loc_techs_milp", ) model_data["operated_units"].attrs["is_result"] = 1 model_data["operated_units"].attrs["operate_param"] = 1 warnings.append( "daily operated units not defined, set to zero for all " "loc::techs in loc_techs_milp, for use in iterative optimisation") for loc_tech in model_data.loc_techs.values: energy_cap = model_data.energy_cap.loc[loc_tech].item() # Must have energy_cap defined for all relevant techs in the model if (np.isinf(energy_cap) or np.isnan(energy_cap)) and ( _get_param(loc_tech, "energy_cap_min_use") or (_get_param(loc_tech, "force_resource") and _get_param(loc_tech, "resource_unit") == "energy_per_cap")): errors.append( "Operate mode: User must define a finite energy_cap (via " "energy_cap_equals or energy_cap_max) for {}".format(loc_tech)) elif _is_in(loc_tech, "loc_techs_finite_resource"): # Cannot have infinite resource area if linking resource and area (resource_unit = energy_per_area) if (_is_in(loc_tech, "loc_techs_area") and model_data.resource_unit.loc[loc_tech].item() == "energy_per_area"): if _is_in(loc_tech, "resource_area"): area = model_data.resource_area.loc[loc_tech].item() else: area = None if pd.isnull(area) or np.isinf(area): errors.append( "Operate mode: User must define a finite resource_area " "(via resource_area_equals or resource_area_max) for {}, " "as available resource is linked to resource_area " "(resource_unit = `energy_per_area`)".format(loc_tech)) # force resource overrides capacity constraints, so set capacity constraints to infinity if _get_param(loc_tech, "force_resource"): if not _is_in(loc_tech, "loc_techs_store"): # set resource_area to inf if the resource is linked to energy_cap using energy_per_cap if (model_data.resource_unit.loc[loc_tech].item() == "energy_per_cap"): if _is_in(loc_tech, "resource_area"): resource_area, warnings = _set_inf_and_warn( loc_tech, "resource_area", warnings, "Resource area constraint removed from {} as " "force_resource is applied and resource is linked " "to energy flow using `energy_per_cap`".format( loc_tech), ) # set energy_cap to inf if the resource is linked to resource_area using energy_per_area elif (model_data.resource_unit.loc[loc_tech].item() == "energy_per_area"): energy_cap, warnings = _set_inf_and_warn( loc_tech, "energy_cap", warnings, "Energy capacity constraint removed from {} as " "force_resource is applied and resource is linked " "to energy flow using `energy_per_area`".format( loc_tech), ) # set both energy_cap and resource_area to inf if the resource is not linked to anything elif (model_data.resource_unit.loc[loc_tech].item() == "energy"): if _is_in(loc_tech, "resource_area"): resource_area, warnings = _set_inf_and_warn( loc_tech, "resource_area", warnings, "Resource area constraint removed from {} as " "force_resource is applied and resource is not linked " "to energy flow (resource_unit = `energy`)". format(loc_tech), ) energy_cap, warnings = _set_inf_and_warn( loc_tech, "energy_cap", warnings, "Energy capacity constraint removed from {} as " "force_resource is applied and resource is not linked " "to energy flow (resource_unit = `energy`)".format( loc_tech), ) if _is_in(loc_tech, "resource_cap"): resource_cap, warnings = _set_inf_and_warn( loc_tech, "resource_cap", warnings, "Resource capacity constraint removed from {} as " "force_resource is applied".format(loc_tech), ) if _is_in(loc_tech, "loc_techs_store"): storage_cap = model_data.storage_cap.loc[loc_tech].item() if not pd.isnull(storage_cap) and not pd.isnull(energy_cap): if _get_param(loc_tech, "charge_rate") is not False: charge_rate = ( model_data["charge_rate"].loc[loc_tech].item()) if storage_cap * charge_rate < energy_cap: errors.append( "fixed storage capacity * charge_rate is not larger " "than fixed energy capacity for loc::tech {}". format(loc_tech)) if (_get_param(loc_tech, "energy_cap_per_storage_cap_max") is not False): energy_cap_per_storage_cap_max = ( model_data["energy_cap_per_storage_cap_max"]. loc[loc_tech].item()) if (storage_cap * energy_cap_per_storage_cap_max < energy_cap): errors.append( "fixed storage capacity * energy_cap_per_storage_cap_max is not larger " "than fixed energy capacity for loc::tech {}". format(loc_tech)) if (_get_param(loc_tech, "energy_cap_per_storage_cap_min") is not False): energy_cap_per_storage_cap_min = ( model_data["energy_cap_per_storage_cap_min"]. loc[loc_tech].item()) if (storage_cap * energy_cap_per_storage_cap_min > energy_cap): errors.append( "fixed storage capacity * energy_cap_per_storage_cap_min is not smaller " "than fixed energy capacity for loc::tech {}". format(loc_tech)) # Must define a resource capacity to ensure the Pyomo param is created # for it. But we just create an array of infs, so the capacity has no effect if ("resource_cap" not in model_data.data_vars.keys() and "loc_techs_supply_plus" in model_data.dims.keys()): model_data["resource_cap"] = xr.DataArray( [np.inf for i in model_data.loc_techs_supply_plus.values], dims="loc_techs_supply_plus", ) model_data["resource_cap"].attrs["is_result"] = 1 model_data["resource_cap"].attrs["operate_param"] = 1 warnings.append( "Resource capacity constraint defined and set to infinity " "for all supply_plus techs") window = run_config.get("operation", {}).get("window", None) horizon = run_config.get("operation", {}).get("horizon", None) if not window or not horizon: errors.append( "Operational mode requires a timestep window and horizon to be " "defined under run.operation") elif horizon < window: errors.append( "Iteration horizon must be larger than iteration window, " "for operational mode") # Cyclic storage isn't really valid in operate mode, so we ignore it, using # initial_storage instead (allowing us to pass storage between operation windows) if run_config.get("cyclic_storage", True): warnings.append( "Storage cannot be cyclic in operate run mode, setting " "`run.cyclic_storage` to False for this run") run_config["cyclic_storage"] = False if "group_demand_share_per_timestep_decision" in model_data.data_vars: warnings.append( "`demand_share_per_timestep_decision` group constraints cannot be " "used in operate mode, so will not be built.") del model_data["group_demand_share_per_timestep_decision"] ## TODO: this comes from preprocess checks so might not work out of the box here # Check if we're allowed to use operate mode if "allow_operate_mode" not in model_data.attrs.keys(): daily_timesteps = [ model_data.timestep_resolution.loc[i].values for i in np.unique( model_data.timesteps.to_index().strftime("%Y-%m-%d")) ] if not np.all(daily_timesteps == daily_timesteps[0]): model_data.attrs["allow_operate_mode"] = 0 warnings.append( "Operational mode requires the same timestep resolution profile " "to be emulated on each date") else: model_data.attrs["allow_operate_mode"] = 1 return comments, warnings, errors
def check_operate_params(model_data): """ if model mode = `operate`, check for clashes in capacity constraints. In this mode, all capacity constraints are set to parameters in the backend, so can easily lead to model infeasibility if not checked. Returns ------- comments : AttrDict debug output warnings : list possible problems that do not prevent the model run from continuing errors : list serious issues that should raise a ModelError """ defaults = UpdateObserverDict( initial_yaml_string=model_data.attrs['defaults'], name='defaults', observer=model_data) run_config = UpdateObserverDict( initial_yaml_string=model_data.attrs['run_config'], name='run_config', observer=model_data) warnings, errors = [], [] comments = AttrDict() def _get_param(loc_tech, var): if _is_in(loc_tech, var) and not any((pd.isnull( (model_data[var].loc[loc_tech].values, )), )): param = model_data[var].loc[loc_tech].values else: param = defaults[var] return param def _is_in(loc_tech, set_or_var): try: model_data[set_or_var].loc[loc_tech] return True except (KeyError, AttributeError): return False for loc_tech in model_data.loc_techs.values: energy_cap = model_data.energy_cap.loc[loc_tech].item() # Must have energy_cap defined for all relevant techs in the model if (pd.isnull(energy_cap) or np.isinf(energy_cap)) and not _is_in( loc_tech, 'force_resource'): errors.append( 'Operate mode: User must define a finite energy_cap (via ' 'energy_cap_equals or energy_cap_max) for {}'.format(loc_tech)) elif _is_in(loc_tech, 'loc_techs_finite_resource'): # force resource overrides capacity constraints, so set capacity constraints to infinity if _is_in(loc_tech, 'force_resource'): if not _is_in(loc_tech, 'loc_techs_store'): energy_cap = model_data.energy_cap.loc[loc_tech] = np.inf warnings.append( 'Energy capacity constraint removed from {} as ' 'force_resource is applied'.format(loc_tech)) if _is_in(loc_tech, 'resource_cap'): print(loc_tech, model_data.resource_cap.loc_techs_supply_plus) model_data.resource_cap.loc[loc_tech] = np.inf warnings.append( 'Resource capacity constraint removed from {} as ' 'force_resource is applied'.format(loc_tech)) # Cannot have infinite resource area (physically impossible) if _is_in(loc_tech, 'loc_techs_area'): area = model_data.resource_area.loc[loc_tech].item() if pd.isnull(area) or np.isinf(area): errors.append( 'Operate mode: User must define a finite resource_area ' '(via resource_area_equals or resource_area_max) for {}, ' 'as a finite available resource is considered'.format( loc_tech)) # Cannot have consumed resource being higher than energy_cap, as # constraints will clash. Doesn't affect supply_plus techs with a # storage buffer prior to carrier production. elif not _is_in(loc_tech, 'loc_techs_store'): resource_scale = _get_param(loc_tech, 'resource_scale') energy_cap_scale = _get_param(loc_tech, 'energy_cap_scale') resource_eff = _get_param(loc_tech, 'resource_eff') energy_eff = _get_param(loc_tech, 'energy_eff') resource = model_data.resource.loc[loc_tech].values if (energy_cap is not None and any( resource * resource_scale * resource_eff > energy_cap * energy_cap_scale * energy_eff)): errors.append( 'Operate mode: resource is forced to be higher than ' 'fixed energy cap for `{}`'.format(loc_tech)) if _is_in(loc_tech, 'loc_techs_store'): if _is_in(loc_tech, 'charge_rate'): storage_cap = model_data.storage_cap.loc[loc_tech].item() if storage_cap and energy_cap: charge_rate = model_data['charge_rate'].loc[loc_tech].item( ) if storage_cap * charge_rate < energy_cap: errors.append( 'fixed storage capacity * charge rate is not larger ' 'than fixed energy capacity for loc::tech {}'. format(loc_tech)) if _is_in(loc_tech, 'loc_techs_store'): if _is_in(loc_tech, 'energy_cap_per_storage_cap_max'): storage_cap = model_data.storage_cap.loc[loc_tech].item() if storage_cap and energy_cap: energy_cap_per_storage_cap_max = model_data[ 'energy_cap_per_storage_cap_max'].loc[loc_tech].item() if storage_cap * energy_cap_per_storage_cap_max < energy_cap: errors.append( 'fixed storage capacity * energy_cap_per_storage_cap_max is not larger ' 'than fixed energy capacity for loc::tech {}'. format(loc_tech)) elif _is_in(loc_tech, 'energy_cap_per_storage_cap_min'): storage_cap = model_data.storage_cap.loc[loc_tech].item() if storage_cap and energy_cap: energy_cap_per_storage_cap_min = model_data[ 'energy_cap_per_storage_cap_min'].loc[loc_tech].item() if storage_cap * energy_cap_per_storage_cap_min > energy_cap: errors.append( 'fixed storage capacity * energy_cap_per_storage_cap_min is not smaller ' 'than fixed energy capacity for loc::tech {}'. format(loc_tech)) # Must define a resource capacity to ensure the Pyomo param is created # for it. But we just create an array of infs, so the capacity has no effect if ('resource_cap' not in model_data.data_vars.keys() and 'loc_techs_supply_plus' in model_data.dims.keys()): model_data['resource_cap'] = xr.DataArray( [np.inf for i in model_data.loc_techs_supply_plus.values], dims='loc_techs_supply_plus') model_data['resource_cap'].attrs['is_result'] = 1 model_data['resource_cap'].attrs['operate_param'] = 1 warnings.append( 'Resource capacity constraint defined and set to infinity ' 'for all supply_plus techs') window = run_config.get('operation', {}).get('window', None) horizon = run_config.get('operation', {}).get('horizon', None) if not window or not horizon: errors.append( 'Operational mode requires a timestep window and horizon to be ' 'defined under run.operation') elif horizon < window: errors.append( 'Iteration horizon must be larger than iteration window, ' 'for operational mode') # Cyclic storage isn't really valid in operate mode, so we ignore it, using # initial_storage instead (allowing us to pass storage between operation windows) if run_config.get('cyclic_storage', True): warnings.append( 'Storage cannot be cyclic in operate run mode, setting ' '`run.cyclic_storage` to False for this run') run_config['cyclic_storage'] = False if 'group_demand_share_per_timestep_decision' in model_data.data_vars: warnings.append( '`demand_share_per_timestep_decision` group constraints cannot be ' 'used in operate mode, so will not be built.') del model_data['group_demand_share_per_timestep_decision'] return comments, warnings, errors
def check_operate_params(model_data): """ if model mode = `operate`, check for clashes in capacity constraints. In this mode, all capacity constraints are set to parameters in the backend, so can easily lead to model infeasibility if not checked. Returns ------- comments : AttrDict debug output warnings : list possible problems that do not prevent the model run from continuing errors : list serious issues that should raise a ModelError """ defaults = UpdateObserverDict( initial_yaml_string=model_data.attrs["defaults"], name="defaults", observer=model_data, flat=True, ) run_config = UpdateObserverDict( initial_yaml_string=model_data.attrs["run_config"], name="run_config", observer=model_data, ) warnings, errors = [], [] comments = AttrDict() def _is_missing(var): if var not in model_data.data_vars.keys(): return True else: return model_data[var].isnull() # Storage initial is carried over between iterations, so must be defined along with storage if (model_data.include_storage == 1).any(): if "storage_initial" not in model_data.data_vars.keys(): model_data["storage_initial"] = model_data.include_storage.astype( float) elif ((model_data.include_storage == 1) & _is_missing("storage_initial")).any(): model_data.storage_initial = model_data.storage_initial.fillna( (model_data.include_storage == 1).astype(float)) model_data["storage_initial"].attrs["is_result"] = 0.0 warnings.append( "Initial stored energy not defined, set to zero for all " "storage technologies, for use in iterative optimisation") if ((~_is_missing("storage_cap")) & (~_is_missing("energy_cap"))).any(): if ((model_data.storage_cap * model_data.get("energy_cap_per_storage_cap_max", np.inf)) < model_data.energy_cap).any(): errors.append( "fixed storage capacity * energy_cap_per_storage_cap_max is not larger " "than fixed energy capacity for some storage technologies") if ((model_data.storage_cap * model_data.get("energy_cap_per_storage_cap_min", 0)) > model_data.energy_cap).any(): errors.append( "fixed storage capacity * energy_cap_per_storage_cap_min is not smaller " "than fixed energy capacity for some technologies") # Operated units is carried over between iterations, so must be defined in a milp model if (model_data.cap_method == "integer").any(): if "operated_units" not in model_data.data_vars.keys(): model_data["operated_units"] = ( model_data.cap_method == "integer").astype(float) elif ((model_data.cap_method == "integer") & _is_missing("operated_units")).any(): model_data.operated_units = model_data.operated_units.fillna( (model_data.cap_method == "integer").astype(float)) model_data["operated_units"].attrs["is_result"] = 1 model_data["operated_units"].attrs["operate_param"] = 1 warnings.append( "daily operated units not defined, set to zero for all technologies defining an integer capacity method, for use in iterative optimisation" ) if (_is_missing("energy_cap") & (~_is_missing("energy_cap_min_use") | (~_is_missing("force_resource") & (model_data.resource_unit == "energy_per_cap")))).any(): errors.append( "Operate mode: User must define a finite energy_cap (via energy_cap_equals " "or energy_cap_max) if using force_resource or energy_cap_min_use") if (~_is_missing("resource") & (_is_missing("resource_area") & (model_data.resource_unit == "energy_per_area"))).any(): errors.append("Operate mode: User must define a finite resource_area " "(via resource_area_equals or resource_area_max) " "if available resource is linked to resource_area " "(resource_unit = `energy_per_area`)") if ("resource_area" in model_data.data_vars and (model_data.resource_unit == "energy_per_cap").any()): model_data["resource_area"] = model_data.resource_area.where(~( (model_data.force_resource == 1) & (model_data.resource_unit == "energy_per_cap"))) warnings.append( "Resource area constraint removed from technologies with " "force_resource applied and resource linked " "to energy flow using `energy_per_cap`") if ("energy_cap" in model_data.data_vars and (model_data.resource_unit == "energy_per_area").any()): model_data["energy_cap"] = model_data.energy_cap.where(~( (model_data.force_resource == 1) & (model_data.resource_unit == "energy_per_area"))) warnings.append( "Energy capacity constraint removed from technologies with " "force_resource applied and resource linked " "to energy flow using `energy_per_area`") if (model_data.resource_unit == "energy").any(): if "energy_cap" in model_data.data_vars: model_data["energy_cap"] = model_data.energy_cap.where(~( (model_data.force_resource == 1) & (model_data.resource_unit == "energy"))) warnings.append( "Energy capacity constraint removed from technologies with " "force_resource applied and resource not linked " "to energy flow (resource_unit = `energy`)") if "resource_area" in model_data.data_vars: model_data["resource_area"] = model_data.resource_area.where(~( (model_data.force_resource == 1) & (model_data.resource_unit == "energy"))) warnings.append( "Energy capacity constraint removed from technologies with " "force_resource applied and resource not linked " "to energy flow (resource_unit = `energy`)") if ("resource_cap" in model_data.data_vars and (model_data.force_resource == 1).any()): model_data["resource_cap"] = model_data.resource_cap.where( model_data.force_resource == 0) warnings.append( "Resource capacity constraint removed from technologies with " "force_resource applied.") # Must define a resource capacity to ensure the Pyomo param is created # for it. But we just create an array of infs, so the capacity has no effect # TODO: implement this in the masks window = run_config.get("operation", {}).get("window", None) horizon = run_config.get("operation", {}).get("horizon", None) if not window or not horizon: errors.append( "Operational mode requires a timestep window and horizon to be " "defined under run.operation") elif horizon < window: errors.append( "Iteration horizon must be larger than iteration window, " "for operational mode") # Cyclic storage isn't really valid in operate mode, so we ignore it, using # initial_storage instead (allowing us to pass storage between operation windows) if run_config.get("cyclic_storage", True): warnings.append( "Storage cannot be cyclic in operate run mode, setting " "`run.cyclic_storage` to False for this run") run_config["cyclic_storage"] = False return comments, warnings, errors
def run_plan( model_data, run_config, timings, backend, build_only, backend_rerun=False, allow_warmstart=False, persistent=True, opt=None, ): log_time(logger, timings, "run_start", comment="Backend: starting model run") warmstart = False if not backend_rerun: backend_model = backend.generate_model(model_data) log_time( logger, timings, "run_backend_model_generated", time_since_run_start=True, comment="Backend: model generated", ) else: backend_model = backend_rerun if allow_warmstart: warmstart = True run_config = UpdateObserverDict( initial_yaml_string=model_data.attrs["run_config"], name="run_config", observer=model_data, ) solver = run_config["solver"] solver_io = run_config.get("solver_io", None) solver_options = run_config.get("solver_options", None) save_logs = run_config.get("save_logs", None) if build_only: results = xr.Dataset() else: if "persistent" in solver and persistent is False: exceptions.warn( f"The chosen solver, `{solver}` will not be used in this run. " f"`{solver.replace('_persistent', '')}` will be used instead.") solver = solver.replace("_persistent", "") log_time( logger, timings, "run_solver_start", comment="Backend: sending model to solver", ) backend_results, opt = backend.solve_model( backend_model, solver=solver, solver_io=solver_io, solver_options=solver_options, save_logs=save_logs, warmstart=warmstart, opt=opt, ) log_time( logger, timings, "run_solver_exit", time_since_run_start=True, comment="Backend: solver finished running", ) termination = backend.load_results(backend_model, backend_results, opt) log_time(logger, timings, "run_results_loaded", comment="Backend: loaded results") if termination in ["optimal", "feasible"]: results = backend.get_result_array(backend_model, model_data) results.attrs["termination_condition"] = termination if "persistent" in opt.name and persistent is True: results.attrs["objective_function_value"] = opt.get_model_attr( "ObjVal") else: results.attrs["objective_function_value"] = backend_model.obj() else: results = xr.Dataset(attrs={"termination_condition": termination}) log_time( logger, timings, "run_solution_returned", time_since_run_start=True, comment="Backend: generated solution array", ) return results, backend_model, opt