class Model(object): """ A Calliope Model. """ def __init__(self, config, model_data=None, debug=False, *args, **kwargs): """ Returns a new Model from either the path to a YAML model configuration file or a dict fully specifying the model. Parameters ---------- config : str or dict or AttrDict If str, must be the path to a model configuration file. If dict or AttrDict, must fully specify the model. model_data : Dataset, optional Create a Model instance from a fully built model_data Dataset. This is only used if `config` is explicitly set to None and is primarily used to re-create a Model instance from a model previously saved to a NetCDF file. """ self._timings = {} # try to set logging output format assuming python interactive. Will # use CLI logging format if model called from CLI log_time(logger, self._timings, "model_creation", comment="Model: initialising") if isinstance(config, str): model_run, debug_data = model_run_from_yaml( config, *args, **kwargs) self._init_from_model_run(model_run, debug_data, debug) elif isinstance(config, dict): model_run, debug_data = model_run_from_dict( config, *args, **kwargs) self._init_from_model_run(model_run, debug_data, debug) elif model_data is not None and config is None: self._init_from_model_data(model_data) else: # expected input is a string pointing to a YAML file of the run # configuration or a dict/AttrDict in which the run and model # configurations are defined raise ValueError( "Input configuration must either be a string or a dictionary.") self._check_future_deprecation_warnings() self.plot = plotting.ModelPlotMethods(self) def _init_from_model_run(self, model_run, debug_data, debug): self._model_run = model_run log_time( logger, self._timings, "model_run_creation", comment="Model: preprocessing stage 1 (model_run)", ) model_data_factory = ModelDataFactory(model_run) ( model_data_pre_clustering, model_data, data_pre_time, stripped_keys, ) = model_data_factory() self._model_data_pre_clustering = model_data_pre_clustering self._model_data = model_data if debug: self._debug_data = debug_data self._model_data_pre_time = data_pre_time self._model_data_stripped_keys = stripped_keys self.inputs = self._model_data.filter_by_attrs(is_result=0) log_time( logger, self._timings, "model_data_original_creation", comment="Model: preprocessing stage 2 (model_data)", ) # Ensure model and run attributes of _model_data update themselves model_config = { k: v for k, v in model_run.get("model", {}).items() if k != "file_allowed" } self.model_config = UpdateObserverDict(initial_dict=model_config, name="model_config", observer=self._model_data) self.run_config = UpdateObserverDict( initial_dict=model_run.get("run", {}), name="run_config", observer=self._model_data, ) self.subsets = UpdateObserverDict( initial_dict=model_run.get("subsets").as_dict_flat(), name="subsets", observer=self._model_data, ) log_time( logger, self._timings, "model_data_creation", comment="Model: preprocessing complete", ) def _init_from_model_data(self, model_data): if "_model_run" in model_data.attrs: self._model_run = AttrDict.from_yaml_string( model_data.attrs["_model_run"]) del model_data.attrs["_model_run"] if "_debug_data" in model_data.attrs: self._debug_data = AttrDict.from_yaml_string( model_data.attrs["_debug_data"]) del model_data.attrs["_debug_data"] self._model_data = model_data self.inputs = self._model_data.filter_by_attrs(is_result=0) self.model_config = UpdateObserverDict( initial_yaml_string=model_data.attrs.get("model_config", "{}"), name="model_config", observer=self._model_data, ) self.run_config = UpdateObserverDict( initial_yaml_string=model_data.attrs.get("run_config", "{}"), name="run_config", observer=self._model_data, ) self.subsets = UpdateObserverDict( initial_yaml_string=model_data.attrs.get("subsets", "{}"), name="subsets", observer=self._model_data, flat=True, ) results = self._model_data.filter_by_attrs(is_result=1) if len(results.data_vars) > 0: self.results = results log_time( logger, self._timings, "model_data_loaded", comment="Model: loaded model_data", ) def run(self, force_rerun=False, **kwargs): """ Run the model. If ``force_rerun`` is True, any existing results will be overwritten. Additional kwargs are passed to the backend. """ # Check that results exist and are non-empty if hasattr(self, "results") and self.results.data_vars and not force_rerun: raise exceptions.ModelError( "This model object already has results. " "Use model.run(force_rerun=True) to force" "the results to be overwritten with a new run.") if (self.run_config["mode"] == "operate" and not self._model_data.attrs["allow_operate_mode"]): raise exceptions.ModelError( "Unable to run this model in operational mode, probably because " "there exist non-uniform timesteps (e.g. from time masking)") results, self._backend_model, interface = run_backend( self._model_data, self._timings, **kwargs) # Add additional post-processed result variables to results if results.attrs.get("termination_condition", None) in ["optimal", "feasible"]: results = postprocess_results.postprocess_model_results( results, self._model_data, self._timings) for var in results.data_vars: results[var].attrs["is_result"] = 1 self._model_data.update(results) self._model_data.attrs.update(results.attrs) self.results = self._model_data.filter_by_attrs(is_result=1) self.backend = interface(self) def get_formatted_array(self, var): """ Return an xr.DataArray with nodes, techs, and carriers as separate dimensions. Parameters ---------- var : str Decision variable for which to return a DataArray. """ warnings.warn( "get_formatted_array() is deprecated and will be removed in a " "future version. Use `model.results.variable` instead.", DeprecationWarning, ) if var not in self._model_data.data_vars: raise KeyError("Variable {} not in Model data".format(var)) return self._model_data[var] def to_netcdf(self, path): """ Save complete model data (inputs and, if available, results) to a NetCDF file at the given ``path``. """ io.save_netcdf(self._model_data, path, model=self) def to_csv(self, path, dropna=True): """ Save complete model data (inputs and, if available, results) as a set of CSV files to the given ``path``. Parameters ---------- dropna : bool, optional If True (default), NaN values are dropped when saving, resulting in significantly smaller CSV files. """ io.save_csv(self._model_data, path, dropna) def to_lp(self, path): """ Save built model to LP format at the given ``path``. If the backend model has not been built yet, it is built prior to saving. """ io.save_lp(self, path) def info(self): info_strings = [] model_name = self.model_config.get("name", "None") info_strings.append("Model name: {}".format(model_name)) msize = "{nodes} nodes, {techs} technologies, {times} timesteps".format( nodes=len(self._model_data.coords.get("nodes", [])), techs=( len(self._model_data.coords.get("techs_non_transmission", [])) + len(self._model_data.coords.get("techs_transmission_names", []))), times=len(self._model_data.coords.get("timesteps", [])), ) info_strings.append("Model size: {}".format(msize)) return "\n".join(info_strings) def _check_future_deprecation_warnings(self): """
def run_operate(model_data, timings, backend, build_only): """ For use when mode is 'operate', to allow the model to be built, edited, and iteratively run within Pyomo. """ log_time( logger, timings, "run_start", comment="Backend: starting model run in operational mode", ) defaults = UpdateObserverDict( initial_yaml_string=model_data.attrs["defaults"], name="defaults", observer=model_data, ) run_config = UpdateObserverDict( initial_yaml_string=model_data.attrs["run_config"], name="run_config", observer=model_data, ) # New param defaults = old maximum param defaults (e.g. energy_cap gets default from energy_cap_max) operate_params = { k.replace("_max", ""): v for k, v in defaults.items() if k.endswith("_max") } operate_params[ "purchased"] = 0 # no _max to work from here, so we hardcode a default defaults.update(operate_params) # Capacity results (from plan mode) can be used as the input to operate mode if any(model_data.filter_by_attrs( is_result=1).data_vars) and run_config.get( "operation.use_cap_results", False): # Anything with is_result = 1 will be ignored in the Pyomo model for varname, varvals in model_data.data_vars.items(): if varname in operate_params.keys(): varvals.attrs["is_result"] = 1 varvals.attrs["operate_param"] = 1 else: cap_max = xr.merge([ v.rename(k.replace("_max", "")) for k, v in model_data.data_vars.items() if "_max" in k ]) cap_equals = xr.merge([ v.rename(k.replace("_equals", "")) for k, v in model_data.data_vars.items() if "_equals" in k ]) caps = cap_max.update(cap_equals) for cap in caps.data_vars.values(): cap.attrs["is_result"] = 1 cap.attrs["operate_param"] = 1 model_data.update(caps) comments, warnings, errors = checks.check_operate_params(model_data) exceptions.print_warnings_and_raise_errors(warnings=warnings, errors=errors) # Initialize our variables solver = run_config["solver"] solver_io = run_config.get("solver_io", None) solver_options = run_config.get("solver_options", None) save_logs = run_config.get("save_logs", None) window = run_config["operation"]["window"] horizon = run_config["operation"]["horizon"] window_to_horizon = horizon - window # get the cumulative sum of timestep resolution, to find where we hit our window and horizon timestep_cumsum = model_data.timestep_resolution.cumsum( "timesteps").to_pandas() # get the timesteps at which we start and end our windows window_ends = timestep_cumsum.where((timestep_cumsum % window == 0) | ( timestep_cumsum == timestep_cumsum[-1])) window_starts = timestep_cumsum.where((~np.isnan(window_ends.shift(1))) | ( timestep_cumsum == timestep_cumsum[0])).dropna() window_ends = window_ends.dropna() horizon_ends = timestep_cumsum[timestep_cumsum.isin(window_ends.values + window_to_horizon)] if not any(window_starts): raise exceptions.ModelError( "Not enough timesteps or incorrect timestep resolution to run in " "operational mode with an optimisation window of {}".format( window)) # We will only update timseries parameters timeseries_data_vars = [ k for k, v in model_data.data_vars.items() if "timesteps" in v.dims and v.attrs["is_result"] == 0 ] # Loop through each window, solve over the horizon length, and add result to # result_array we only go as far as the end of the last horizon, which may # clip the last bit of data result_array = [] # track whether each iteration finds an optimal solution or not terminations = [] if build_only: iterations = [0] else: iterations = range(len(window_starts)) for i in iterations: start_timestep = window_starts.index[i] # Build full model in first instance if i == 0: warmstart = False end_timestep = horizon_ends.index[i] timesteps = slice(start_timestep, end_timestep) window_model_data = model_data.loc[dict(timesteps=timesteps)] log_time( logger, timings, "model_gen_1", comment="Backend: generating initial model", ) backend_model = backend.generate_model(window_model_data) # Build the full model in the last instance(s), # where number of timesteps is less than the horizon length elif i > len(horizon_ends) - 1: warmstart = False end_timestep = window_ends.index[i] timesteps = slice(start_timestep, end_timestep) window_model_data = model_data.loc[dict(timesteps=timesteps)] log_time( logger, timings, "model_gen_{}".format(i + 1), comment=( "Backend: iteration {}: generating new model for " "end of timeseries, with horizon = {} timesteps".format( i + 1, window_ends[i] - window_starts[i])), ) backend_model = backend.generate_model(window_model_data) # Update relevent Pyomo Params in intermediate instances else: warmstart = True end_timestep = horizon_ends.index[i] timesteps = slice(start_timestep, end_timestep) window_model_data = model_data.loc[dict(timesteps=timesteps)] log_time( logger, timings, "model_gen_{}".format(i + 1), comment="Backend: iteration {}: updating model parameters". format(i + 1), ) # Pyomo model sees the same timestamps each time, we just change the # values associated with those timestamps for var in timeseries_data_vars: # New values var_series = ( window_model_data[var].to_series().dropna().replace( "inf", np.inf)) # Same timestamps var_series.index = backend_model.__calliope_model_data["data"][ var].keys() var_dict = var_series.to_dict() # Update pyomo Param with new dictionary getattr(backend_model, var).store_values(var_dict) if not build_only: log_time( logger, timings, "model_run_{}".format(i + 1), time_since_run_start=True, comment="Backend: iteration {}: sending model to solver". format(i + 1), ) # After iteration 1, warmstart = True, which should speed up the process # Note: Warmstart isn't possible with GLPK (dealt with later on) _results = backend.solve_model( backend_model, solver=solver, solver_io=solver_io, solver_options=solver_options, save_logs=save_logs, warmstart=warmstart, ) log_time( logger, timings, "run_solver_exit_{}".format(i + 1), time_since_run_start=True, comment="Backend: iteration {}: solver finished running". format(i + 1), ) # xarray dataset is built for each iteration _termination = backend.load_results(backend_model, _results) terminations.append(_termination) _results = backend.get_result_array(backend_model, model_data) # We give back the actual timesteps for this iteration and take a slice # equal to the window length _results["timesteps"] = window_model_data.timesteps.copy() # We always save the window data. Until the last window(s) this will crop # the window_to_horizon timesteps. In the last window(s), optimistion will # only be occurring over a window length anyway _results = _results.loc[dict( timesteps=slice(None, window_ends.index[i]))] result_array.append(_results) # Set up initial storage for the next iteration if "loc_techs_store" in model_data.dims.keys(): storage_initial = _results.storage.loc[{ "timesteps": window_ends.index[i] }].drop("timesteps") model_data["storage_initial"].loc[ storage_initial.coords] = storage_initial.values backend_model.storage_initial.store_values( storage_initial.to_series().dropna().to_dict()) # Set up total operated units for the next iteration if "loc_techs_milp" in model_data.dims.keys(): operated_units = _results.operating_units.sum( "timesteps").astype(np.int) model_data["operated_units"].loc[{}] += operated_units.values backend_model.operated_units.store_values( operated_units.to_series().dropna().to_dict()) log_time( logger, timings, "run_solver_exit_{}".format(i + 1), time_since_run_start=True, comment="Backend: iteration {}: generated solution array". format(i + 1), ) if build_only: results = xr.Dataset() else: # Concatenate results over the timestep dimension to get a single # xarray Dataset of interest results = xr.concat(result_array, dim="timesteps") if all(i == "optimal" for i in terminations): results.attrs["termination_condition"] = "optimal" elif all(i in ["optimal", "feasible"] for i in terminations): results.attrs["termination_condition"] = "feasible" else: results.attrs["termination_condition"] = ",".join(terminations) log_time( logger, timings, "run_solution_returned", time_since_run_start=True, comment="Backend: generated full solution array", ) return results, backend_model
def check_operate_params(model_data): """ if model mode = `operate`, check for clashes in capacity constraints. In this mode, all capacity constraints are set to parameters in the backend, so can easily lead to model infeasibility if not checked. Returns ------- comments : AttrDict debug output warnings : list possible problems that do not prevent the model run from continuing errors : list serious issues that should raise a ModelError """ defaults = UpdateObserverDict( initial_yaml_string=model_data.attrs["defaults"], name="defaults", observer=model_data, ) run_config = UpdateObserverDict( initial_yaml_string=model_data.attrs["run_config"], name="run_config", observer=model_data, ) warnings, errors = [], [] comments = AttrDict() def _get_param(loc_tech, var): if (_is_in(loc_tech, var) and not model_data[var].loc[loc_tech].isnull().any()): param = model_data[var].loc[loc_tech].values else: param = defaults[var] return param def _is_in(loc_tech, set_or_var): try: model_data[set_or_var].loc[loc_tech] return True except (KeyError, AttributeError): return False def _set_inf_and_warn(loc_tech, var, warnings, warning_text): if np.isinf(model_data[var].loc[loc_tech].item()): return (np.inf, warnings) elif model_data[var].loc[loc_tech].isnull().item(): var_name = model_data[var].loc[loc_tech] = np.inf return (var_name, warnings) else: var_name = model_data[var].loc[loc_tech] = np.inf warnings.append(warning_text) return var_name, warnings # Storage initial is carried over between iterations, so must be defined along with storage if ("loc_techs_store" in model_data.dims.keys() and "storage_initial" not in model_data.data_vars.keys()): model_data["storage_initial"] = xr.DataArray( [0.0 for loc_tech in model_data.loc_techs_store.values], dims="loc_techs_store", ) model_data["storage_initial"].attrs["is_result"] = 0.0 warnings.append( "Initial stored energy not defined, set to zero for all " "loc::techs in loc_techs_store, for use in iterative optimisation") # Operated units is carried over between iterations, so must be defined in a milp model if ("loc_techs_milp" in model_data.dims.keys() and "operated_units" not in model_data.data_vars.keys()): model_data["operated_units"] = xr.DataArray( [0 for loc_tech in model_data.loc_techs_milp.values], dims="loc_techs_milp", ) model_data["operated_units"].attrs["is_result"] = 1 model_data["operated_units"].attrs["operate_param"] = 1 warnings.append( "daily operated units not defined, set to zero for all " "loc::techs in loc_techs_milp, for use in iterative optimisation") for loc_tech in model_data.loc_techs.values: energy_cap = model_data.energy_cap.loc[loc_tech].item() # Must have energy_cap defined for all relevant techs in the model if (np.isinf(energy_cap) or np.isnan(energy_cap)) and ( _get_param(loc_tech, "energy_cap_min_use") or (_get_param(loc_tech, "force_resource") and _get_param(loc_tech, "resource_unit") == "energy_per_cap")): errors.append( "Operate mode: User must define a finite energy_cap (via " "energy_cap_equals or energy_cap_max) for {}".format(loc_tech)) elif _is_in(loc_tech, "loc_techs_finite_resource"): # Cannot have infinite resource area if linking resource and area (resource_unit = energy_per_area) if (_is_in(loc_tech, "loc_techs_area") and model_data.resource_unit.loc[loc_tech].item() == "energy_per_area"): if _is_in(loc_tech, "resource_area"): area = model_data.resource_area.loc[loc_tech].item() else: area = None if pd.isnull(area) or np.isinf(area): errors.append( "Operate mode: User must define a finite resource_area " "(via resource_area_equals or resource_area_max) for {}, " "as available resource is linked to resource_area " "(resource_unit = `energy_per_area`)".format(loc_tech)) # force resource overrides capacity constraints, so set capacity constraints to infinity if _get_param(loc_tech, "force_resource"): if not _is_in(loc_tech, "loc_techs_store"): # set resource_area to inf if the resource is linked to energy_cap using energy_per_cap if (model_data.resource_unit.loc[loc_tech].item() == "energy_per_cap"): if _is_in(loc_tech, "resource_area"): resource_area, warnings = _set_inf_and_warn( loc_tech, "resource_area", warnings, "Resource area constraint removed from {} as " "force_resource is applied and resource is linked " "to energy flow using `energy_per_cap`".format( loc_tech), ) # set energy_cap to inf if the resource is linked to resource_area using energy_per_area elif (model_data.resource_unit.loc[loc_tech].item() == "energy_per_area"): energy_cap, warnings = _set_inf_and_warn( loc_tech, "energy_cap", warnings, "Energy capacity constraint removed from {} as " "force_resource is applied and resource is linked " "to energy flow using `energy_per_area`".format( loc_tech), ) # set both energy_cap and resource_area to inf if the resource is not linked to anything elif (model_data.resource_unit.loc[loc_tech].item() == "energy"): if _is_in(loc_tech, "resource_area"): resource_area, warnings = _set_inf_and_warn( loc_tech, "resource_area", warnings, "Resource area constraint removed from {} as " "force_resource is applied and resource is not linked " "to energy flow (resource_unit = `energy`)". format(loc_tech), ) energy_cap, warnings = _set_inf_and_warn( loc_tech, "energy_cap", warnings, "Energy capacity constraint removed from {} as " "force_resource is applied and resource is not linked " "to energy flow (resource_unit = `energy`)".format( loc_tech), ) if _is_in(loc_tech, "resource_cap"): resource_cap, warnings = _set_inf_and_warn( loc_tech, "resource_cap", warnings, "Resource capacity constraint removed from {} as " "force_resource is applied".format(loc_tech), ) if _is_in(loc_tech, "loc_techs_store"): storage_cap = model_data.storage_cap.loc[loc_tech].item() if not pd.isnull(storage_cap) and not pd.isnull(energy_cap): if _get_param(loc_tech, "charge_rate") is not False: charge_rate = ( model_data["charge_rate"].loc[loc_tech].item()) if storage_cap * charge_rate < energy_cap: errors.append( "fixed storage capacity * charge_rate is not larger " "than fixed energy capacity for loc::tech {}". format(loc_tech)) if (_get_param(loc_tech, "energy_cap_per_storage_cap_max") is not False): energy_cap_per_storage_cap_max = ( model_data["energy_cap_per_storage_cap_max"]. loc[loc_tech].item()) if (storage_cap * energy_cap_per_storage_cap_max < energy_cap): errors.append( "fixed storage capacity * energy_cap_per_storage_cap_max is not larger " "than fixed energy capacity for loc::tech {}". format(loc_tech)) if (_get_param(loc_tech, "energy_cap_per_storage_cap_min") is not False): energy_cap_per_storage_cap_min = ( model_data["energy_cap_per_storage_cap_min"]. loc[loc_tech].item()) if (storage_cap * energy_cap_per_storage_cap_min > energy_cap): errors.append( "fixed storage capacity * energy_cap_per_storage_cap_min is not smaller " "than fixed energy capacity for loc::tech {}". format(loc_tech)) # Must define a resource capacity to ensure the Pyomo param is created # for it. But we just create an array of infs, so the capacity has no effect if ("resource_cap" not in model_data.data_vars.keys() and "loc_techs_supply_plus" in model_data.dims.keys()): model_data["resource_cap"] = xr.DataArray( [np.inf for i in model_data.loc_techs_supply_plus.values], dims="loc_techs_supply_plus", ) model_data["resource_cap"].attrs["is_result"] = 1 model_data["resource_cap"].attrs["operate_param"] = 1 warnings.append( "Resource capacity constraint defined and set to infinity " "for all supply_plus techs") window = run_config.get("operation", {}).get("window", None) horizon = run_config.get("operation", {}).get("horizon", None) if not window or not horizon: errors.append( "Operational mode requires a timestep window and horizon to be " "defined under run.operation") elif horizon < window: errors.append( "Iteration horizon must be larger than iteration window, " "for operational mode") # Cyclic storage isn't really valid in operate mode, so we ignore it, using # initial_storage instead (allowing us to pass storage between operation windows) if run_config.get("cyclic_storage", True): warnings.append( "Storage cannot be cyclic in operate run mode, setting " "`run.cyclic_storage` to False for this run") run_config["cyclic_storage"] = False if "group_demand_share_per_timestep_decision" in model_data.data_vars: warnings.append( "`demand_share_per_timestep_decision` group constraints cannot be " "used in operate mode, so will not be built.") del model_data["group_demand_share_per_timestep_decision"] ## TODO: this comes from preprocess checks so might not work out of the box here # Check if we're allowed to use operate mode if "allow_operate_mode" not in model_data.attrs.keys(): daily_timesteps = [ model_data.timestep_resolution.loc[i].values for i in np.unique( model_data.timesteps.to_index().strftime("%Y-%m-%d")) ] if not np.all(daily_timesteps == daily_timesteps[0]): model_data.attrs["allow_operate_mode"] = 0 warnings.append( "Operational mode requires the same timestep resolution profile " "to be emulated on each date") else: model_data.attrs["allow_operate_mode"] = 1 return comments, warnings, errors
def check_operate_params(model_data): """ if model mode = `operate`, check for clashes in capacity constraints. In this mode, all capacity constraints are set to parameters in the backend, so can easily lead to model infeasibility if not checked. Returns ------- comments : AttrDict debug output warnings : list possible problems that do not prevent the model run from continuing errors : list serious issues that should raise a ModelError """ defaults = UpdateObserverDict( initial_yaml_string=model_data.attrs['defaults'], name='defaults', observer=model_data) run_config = UpdateObserverDict( initial_yaml_string=model_data.attrs['run_config'], name='run_config', observer=model_data) warnings, errors = [], [] comments = AttrDict() def _get_param(loc_tech, var): if _is_in(loc_tech, var) and not any((pd.isnull( (model_data[var].loc[loc_tech].values, )), )): param = model_data[var].loc[loc_tech].values else: param = defaults[var] return param def _is_in(loc_tech, set_or_var): try: model_data[set_or_var].loc[loc_tech] return True except (KeyError, AttributeError): return False for loc_tech in model_data.loc_techs.values: energy_cap = model_data.energy_cap.loc[loc_tech].item() # Must have energy_cap defined for all relevant techs in the model if (pd.isnull(energy_cap) or np.isinf(energy_cap)) and not _is_in( loc_tech, 'force_resource'): errors.append( 'Operate mode: User must define a finite energy_cap (via ' 'energy_cap_equals or energy_cap_max) for {}'.format(loc_tech)) elif _is_in(loc_tech, 'loc_techs_finite_resource'): # force resource overrides capacity constraints, so set capacity constraints to infinity if _is_in(loc_tech, 'force_resource'): if not _is_in(loc_tech, 'loc_techs_store'): energy_cap = model_data.energy_cap.loc[loc_tech] = np.inf warnings.append( 'Energy capacity constraint removed from {} as ' 'force_resource is applied'.format(loc_tech)) if _is_in(loc_tech, 'resource_cap'): print(loc_tech, model_data.resource_cap.loc_techs_supply_plus) model_data.resource_cap.loc[loc_tech] = np.inf warnings.append( 'Resource capacity constraint removed from {} as ' 'force_resource is applied'.format(loc_tech)) # Cannot have infinite resource area (physically impossible) if _is_in(loc_tech, 'loc_techs_area'): area = model_data.resource_area.loc[loc_tech].item() if pd.isnull(area) or np.isinf(area): errors.append( 'Operate mode: User must define a finite resource_area ' '(via resource_area_equals or resource_area_max) for {}, ' 'as a finite available resource is considered'.format( loc_tech)) # Cannot have consumed resource being higher than energy_cap, as # constraints will clash. Doesn't affect supply_plus techs with a # storage buffer prior to carrier production. elif not _is_in(loc_tech, 'loc_techs_store'): resource_scale = _get_param(loc_tech, 'resource_scale') energy_cap_scale = _get_param(loc_tech, 'energy_cap_scale') resource_eff = _get_param(loc_tech, 'resource_eff') energy_eff = _get_param(loc_tech, 'energy_eff') resource = model_data.resource.loc[loc_tech].values if (energy_cap is not None and any( resource * resource_scale * resource_eff > energy_cap * energy_cap_scale * energy_eff)): errors.append( 'Operate mode: resource is forced to be higher than ' 'fixed energy cap for `{}`'.format(loc_tech)) if _is_in(loc_tech, 'loc_techs_store'): if _is_in(loc_tech, 'charge_rate'): storage_cap = model_data.storage_cap.loc[loc_tech].item() if storage_cap and energy_cap: charge_rate = model_data['charge_rate'].loc[loc_tech].item( ) if storage_cap * charge_rate < energy_cap: errors.append( 'fixed storage capacity * charge rate is not larger ' 'than fixed energy capacity for loc::tech {}'. format(loc_tech)) if _is_in(loc_tech, 'loc_techs_store'): if _is_in(loc_tech, 'energy_cap_per_storage_cap_max'): storage_cap = model_data.storage_cap.loc[loc_tech].item() if storage_cap and energy_cap: energy_cap_per_storage_cap_max = model_data[ 'energy_cap_per_storage_cap_max'].loc[loc_tech].item() if storage_cap * energy_cap_per_storage_cap_max < energy_cap: errors.append( 'fixed storage capacity * energy_cap_per_storage_cap_max is not larger ' 'than fixed energy capacity for loc::tech {}'. format(loc_tech)) elif _is_in(loc_tech, 'energy_cap_per_storage_cap_min'): storage_cap = model_data.storage_cap.loc[loc_tech].item() if storage_cap and energy_cap: energy_cap_per_storage_cap_min = model_data[ 'energy_cap_per_storage_cap_min'].loc[loc_tech].item() if storage_cap * energy_cap_per_storage_cap_min > energy_cap: errors.append( 'fixed storage capacity * energy_cap_per_storage_cap_min is not smaller ' 'than fixed energy capacity for loc::tech {}'. format(loc_tech)) # Must define a resource capacity to ensure the Pyomo param is created # for it. But we just create an array of infs, so the capacity has no effect if ('resource_cap' not in model_data.data_vars.keys() and 'loc_techs_supply_plus' in model_data.dims.keys()): model_data['resource_cap'] = xr.DataArray( [np.inf for i in model_data.loc_techs_supply_plus.values], dims='loc_techs_supply_plus') model_data['resource_cap'].attrs['is_result'] = 1 model_data['resource_cap'].attrs['operate_param'] = 1 warnings.append( 'Resource capacity constraint defined and set to infinity ' 'for all supply_plus techs') window = run_config.get('operation', {}).get('window', None) horizon = run_config.get('operation', {}).get('horizon', None) if not window or not horizon: errors.append( 'Operational mode requires a timestep window and horizon to be ' 'defined under run.operation') elif horizon < window: errors.append( 'Iteration horizon must be larger than iteration window, ' 'for operational mode') # Cyclic storage isn't really valid in operate mode, so we ignore it, using # initial_storage instead (allowing us to pass storage between operation windows) if run_config.get('cyclic_storage', True): warnings.append( 'Storage cannot be cyclic in operate run mode, setting ' '`run.cyclic_storage` to False for this run') run_config['cyclic_storage'] = False if 'group_demand_share_per_timestep_decision' in model_data.data_vars: warnings.append( '`demand_share_per_timestep_decision` group constraints cannot be ' 'used in operate mode, so will not be built.') del model_data['group_demand_share_per_timestep_decision'] return comments, warnings, errors
class Model(object): """ A Calliope Model. """ def __init__(self, config, model_data=None, *args, **kwargs): """ Returns a new Model from either the path to a YAML model configuration file or a dict fully specifying the model. Parameters ---------- config : str or dict or AttrDict If str, must be the path to a model configuration file. If dict or AttrDict, must fully specify the model. model_data : Dataset, optional Create a Model instance from a fully built model_data Dataset. This is only used if `config` is explicitly set to None and is primarily used to re-create a Model instance from a model previously saved to a NetCDF file. """ self._timings = {} # try to set logging output format assuming python interactive. Will # use CLI logging format if model called from CLI log_time(logger, self._timings, 'model_creation', comment='Model: initialising') if isinstance(config, str): model_run, debug_data = model_run_from_yaml( config, *args, **kwargs) self._init_from_model_run(model_run, debug_data) elif isinstance(config, dict): model_run, debug_data = model_run_from_dict( config, *args, **kwargs) self._init_from_model_run(model_run, debug_data) elif model_data is not None and config is None: self._init_from_model_data(model_data) else: # expected input is a string pointing to a YAML file of the run # configuration or a dict/AttrDict in which the run and model # configurations are defined raise ValueError( 'Input configuration must either be a string or a dictionary.') self._check_future_deprecation_warnings() self.plot = plotting.ModelPlotMethods(self) def _init_from_model_run(self, model_run, debug_data): self._model_run = model_run self._debug_data = debug_data log_time(logger, self._timings, 'model_run_creation', comment='Model: preprocessing stage 1 (model_run)') self._model_data_original = build_model_data(model_run) log_time(logger, self._timings, 'model_data_original_creation', comment='Model: preprocessing stage 2 (model_data)') random_seed = self._model_run.get_key('model.random_seed', None) if random_seed: np.random.seed(seed=random_seed) # After setting the random seed, time clustering can take place time_config = model_run.model.get('time', None) if not time_config: _model_data = self._model_data_original else: _model_data = apply_time_clustering(self._model_data_original, model_run) self._model_data = final_timedimension_processing(_model_data) log_time(logger, self._timings, 'model_data_creation', comment='Model: preprocessing complete') # Ensure model and run attributes of _model_data update themselves for var in self._model_data.data_vars: self._model_data[var].attrs['is_result'] = 0 self.inputs = self._model_data.filter_by_attrs(is_result=0) model_config = { k: v for k, v in model_run.get('model', {}).items() if k != 'file_allowed' } self.model_config = UpdateObserverDict(initial_dict=model_config, name='model_config', observer=self._model_data) self.run_config = UpdateObserverDict(initial_dict=model_run.get( 'run', {}), name='run_config', observer=self._model_data) def _init_from_model_data(self, model_data): if '_model_run' in model_data.attrs: self._model_run = AttrDict.from_yaml_string( model_data.attrs['_model_run']) del model_data.attrs['_model_run'] if '_debug_data' in model_data.attrs: self._debug_data = AttrDict.from_yaml_string( model_data.attrs['_debug_data']) del model_data.attrs['_debug_data'] self._model_data = model_data self.inputs = self._model_data.filter_by_attrs(is_result=0) self.model_config = UpdateObserverDict( initial_yaml_string=model_data.attrs.get('model_config', '{}'), name='model_config', observer=self._model_data) self.run_config = UpdateObserverDict( initial_yaml_string=model_data.attrs.get('run_config', '{}'), name='run_config', observer=self._model_data) results = self._model_data.filter_by_attrs(is_result=1) if len(results.data_vars) > 0: self.results = results log_time(logger, self._timings, 'model_data_loaded', comment='Model: loaded model_data') def save_commented_model_yaml(self, path): """ Save a fully built and commented version of the model to a YAML file at the given ``path``. Comments in the file indicate where values were overridden. This is Calliope's internal representation of a model directly before the model_data xarray.Dataset is built, and can be useful for debugging possible issues in the model formulation. """ if not self._model_run or not self._debug_data: raise KeyError( 'This model does not have the fully built model attached, ' 'so `save_commented_model_yaml` is not available. Likely ' 'reason is that the model was built with a verion of Calliope ' 'prior to 0.6.5.') yaml = ruamel_yaml.YAML() model_run_debug = self._model_run.copy() try: del model_run_debug['timeseries_data'] # Can't be serialised! except KeyError: # Possible that timeseries_data is already gone if the model # was read from a NetCDF file pass # Turn sets in model_run into lists for YAML serialization for k, v in model_run_debug.sets.items(): model_run_debug.sets[k] = list(v) debug_comments = self._debug_data['comments'] stream = StringIO() yaml.dump(model_run_debug.as_dict(), stream=stream) debug_yaml = yaml.load(stream.getvalue()) for k in debug_comments.model_run.keys_nested(): v = debug_comments.model_run.get_key(k) if v: keys = k.split('.') apply_to_dict(debug_yaml, keys[:-1], 'yaml_add_eol_comment', (v, keys[-1])) dumper = ruamel_yaml.dumper.RoundTripDumper dumper.ignore_aliases = lambda self, data: True with open(path, 'w') as f: ruamel_yaml.dump(debug_yaml, stream=f, Dumper=dumper, default_flow_style=False) def run(self, force_rerun=False, **kwargs): """ Run the model. If ``force_rerun`` is True, any existing results will be overwritten. Additional kwargs are passed to the backend. """ # Check that results exist and are non-empty if hasattr(self, 'results') and self.results.data_vars and not force_rerun: raise exceptions.ModelError( 'This model object already has results. ' 'Use model.run(force_rerun=True) to force' 'the results to be overwritten with a new run.') if (self.run_config['mode'] == 'operate' and not self._model_data.attrs['allow_operate_mode']): raise exceptions.ModelError( 'Unable to run this model in operational mode, probably because ' 'there exist non-uniform timesteps (e.g. from time masking)') results, self._backend_model, interface = run_backend( self._model_data, self._timings, **kwargs) # Add additional post-processed result variables to results if results.attrs.get('termination_condition', None) == 'optimal': results = postprocess.postprocess_model_results( results, self._model_data, self._timings) for var in results.data_vars: results[var].attrs['is_result'] = 1 self._model_data.update(results) self._model_data.attrs.update(results.attrs) if 'run_solution_returned' in self._timings.keys(): self._model_data.attrs['solution_time'] = ( self._timings['run_solution_returned'] - self._timings['run_start']).total_seconds() self._model_data.attrs['time_finished'] = ( self._timings['run_solution_returned'].strftime( '%Y-%m-%d %H:%M:%S')) self.results = self._model_data.filter_by_attrs(is_result=1) self.backend = interface(self) def get_formatted_array(self, var, index_format='index'): """ Return an xr.DataArray with locs, techs, and carriers as separate dimensions. Parameters ---------- var : str Decision variable for which to return a DataArray. index_format : str, default = 'index' 'index' to return the `loc_tech(_carrier)` dimensions as individual indexes, 'multiindex' to return them as a MultiIndex. The latter has the benefit of having a smaller memory footprint, but you cannot undertake dimension specific operations (e.g. formatted_array.sum('locs')) """ if var not in self._model_data.data_vars: raise KeyError("Variable {} not in Model data".format(var)) if index_format not in ['index', 'multiindex']: raise ValueError( "Argument 'index_format' must be one of 'index' or 'multiindex'" ) elif index_format == 'index': return_as = 'DataArray' elif index_format == 'multiindex': return_as = 'MultiIndex DataArray' return split_loc_techs(self._model_data[var], return_as=return_as) def to_netcdf(self, path): """ Save complete model data (inputs and, if available, results) to a NetCDF file at the given ``path``. """ io.save_netcdf(self._model_data, path, model=self) def to_csv(self, path, dropna=True): """ Save complete model data (inputs and, if available, results) as a set of CSV files to the given ``path``. Parameters ---------- dropna : bool, optional If True (default), NaN values are dropped when saving, resulting in significantly smaller CSV files. """ io.save_csv(self._model_data, path, dropna) def to_lp(self, path): """ Save built model to LP format at the given ``path``. If the backend model has not been built yet, it is built prior to saving. """ io.save_lp(self, path) def info(self): info_strings = [] model_name = self.model_config.get('name', 'None') info_strings.append('Model name: {}'.format(model_name)) msize = '{locs} locations, {techs} technologies, {times} timesteps'.format( locs=len(self._model_data.coords.get('locs', [])), techs=( len(self._model_data.coords.get('techs_non_transmission', [])) + len(self._model_data.coords.get('techs_transmission_names', []))), times=len(self._model_data.coords.get('timesteps', []))) info_strings.append('Model size: {}'.format(msize)) return '\n'.join(info_strings) def _check_future_deprecation_warnings(self): """ Method for all FutureWarnings and DeprecationWarnings. Comment above each warning should specify Calliope version in which it was added, and the version in which it should be updated/removed. """ # Warning that group_share constraints will removed in 0.7.0 # # Added in 0.6.4-dev, to be removed in v0.7.0-dev if any('group_share_' in i for i in self._model_data.data_vars.keys()): warnings.warn( '`group_share` constraints will be removed in v0.7.0 -- ' 'use the new model-wide constraints instead.', FutureWarning) # Warning that charge rate will be removed in 0.7.0 # Added in 0.6.4-dev, to be removed in 0.7.0-dev # Rename charge rate to energy_cap_per_storage_cap_max if self._model_data is not None and "charge_rate" in self._model_data: warnings.warn( '`charge_rate` is renamed to `energy_cap_per_storage_cap_max` ' 'and will be removed in v0.7.0.', FutureWarning)
def check_operate_params(model_data): """ if model mode = `operate`, check for clashes in capacity constraints. In this mode, all capacity constraints are set to parameters in the backend, so can easily lead to model infeasibility if not checked. Returns ------- comments : AttrDict debug output warnings : list possible problems that do not prevent the model run from continuing errors : list serious issues that should raise a ModelError """ defaults = UpdateObserverDict( initial_yaml_string=model_data.attrs["defaults"], name="defaults", observer=model_data, flat=True, ) run_config = UpdateObserverDict( initial_yaml_string=model_data.attrs["run_config"], name="run_config", observer=model_data, ) warnings, errors = [], [] comments = AttrDict() def _is_missing(var): if var not in model_data.data_vars.keys(): return True else: return model_data[var].isnull() # Storage initial is carried over between iterations, so must be defined along with storage if (model_data.include_storage == 1).any(): if "storage_initial" not in model_data.data_vars.keys(): model_data["storage_initial"] = model_data.include_storage.astype( float) elif ((model_data.include_storage == 1) & _is_missing("storage_initial")).any(): model_data.storage_initial = model_data.storage_initial.fillna( (model_data.include_storage == 1).astype(float)) model_data["storage_initial"].attrs["is_result"] = 0.0 warnings.append( "Initial stored energy not defined, set to zero for all " "storage technologies, for use in iterative optimisation") if ((~_is_missing("storage_cap")) & (~_is_missing("energy_cap"))).any(): if ((model_data.storage_cap * model_data.get("energy_cap_per_storage_cap_max", np.inf)) < model_data.energy_cap).any(): errors.append( "fixed storage capacity * energy_cap_per_storage_cap_max is not larger " "than fixed energy capacity for some storage technologies") if ((model_data.storage_cap * model_data.get("energy_cap_per_storage_cap_min", 0)) > model_data.energy_cap).any(): errors.append( "fixed storage capacity * energy_cap_per_storage_cap_min is not smaller " "than fixed energy capacity for some technologies") # Operated units is carried over between iterations, so must be defined in a milp model if (model_data.cap_method == "integer").any(): if "operated_units" not in model_data.data_vars.keys(): model_data["operated_units"] = ( model_data.cap_method == "integer").astype(float) elif ((model_data.cap_method == "integer") & _is_missing("operated_units")).any(): model_data.operated_units = model_data.operated_units.fillna( (model_data.cap_method == "integer").astype(float)) model_data["operated_units"].attrs["is_result"] = 1 model_data["operated_units"].attrs["operate_param"] = 1 warnings.append( "daily operated units not defined, set to zero for all technologies defining an integer capacity method, for use in iterative optimisation" ) if (_is_missing("energy_cap") & (~_is_missing("energy_cap_min_use") | (~_is_missing("force_resource") & (model_data.resource_unit == "energy_per_cap")))).any(): errors.append( "Operate mode: User must define a finite energy_cap (via energy_cap_equals " "or energy_cap_max) if using force_resource or energy_cap_min_use") if (~_is_missing("resource") & (_is_missing("resource_area") & (model_data.resource_unit == "energy_per_area"))).any(): errors.append("Operate mode: User must define a finite resource_area " "(via resource_area_equals or resource_area_max) " "if available resource is linked to resource_area " "(resource_unit = `energy_per_area`)") if ("resource_area" in model_data.data_vars and (model_data.resource_unit == "energy_per_cap").any()): model_data["resource_area"] = model_data.resource_area.where(~( (model_data.force_resource == 1) & (model_data.resource_unit == "energy_per_cap"))) warnings.append( "Resource area constraint removed from technologies with " "force_resource applied and resource linked " "to energy flow using `energy_per_cap`") if ("energy_cap" in model_data.data_vars and (model_data.resource_unit == "energy_per_area").any()): model_data["energy_cap"] = model_data.energy_cap.where(~( (model_data.force_resource == 1) & (model_data.resource_unit == "energy_per_area"))) warnings.append( "Energy capacity constraint removed from technologies with " "force_resource applied and resource linked " "to energy flow using `energy_per_area`") if (model_data.resource_unit == "energy").any(): if "energy_cap" in model_data.data_vars: model_data["energy_cap"] = model_data.energy_cap.where(~( (model_data.force_resource == 1) & (model_data.resource_unit == "energy"))) warnings.append( "Energy capacity constraint removed from technologies with " "force_resource applied and resource not linked " "to energy flow (resource_unit = `energy`)") if "resource_area" in model_data.data_vars: model_data["resource_area"] = model_data.resource_area.where(~( (model_data.force_resource == 1) & (model_data.resource_unit == "energy"))) warnings.append( "Energy capacity constraint removed from technologies with " "force_resource applied and resource not linked " "to energy flow (resource_unit = `energy`)") if ("resource_cap" in model_data.data_vars and (model_data.force_resource == 1).any()): model_data["resource_cap"] = model_data.resource_cap.where( model_data.force_resource == 0) warnings.append( "Resource capacity constraint removed from technologies with " "force_resource applied.") # Must define a resource capacity to ensure the Pyomo param is created # for it. But we just create an array of infs, so the capacity has no effect # TODO: implement this in the masks window = run_config.get("operation", {}).get("window", None) horizon = run_config.get("operation", {}).get("horizon", None) if not window or not horizon: errors.append( "Operational mode requires a timestep window and horizon to be " "defined under run.operation") elif horizon < window: errors.append( "Iteration horizon must be larger than iteration window, " "for operational mode") # Cyclic storage isn't really valid in operate mode, so we ignore it, using # initial_storage instead (allowing us to pass storage between operation windows) if run_config.get("cyclic_storage", True): warnings.append( "Storage cannot be cyclic in operate run mode, setting " "`run.cyclic_storage` to False for this run") run_config["cyclic_storage"] = False return comments, warnings, errors
def run_plan( model_data, run_config, timings, backend, build_only, backend_rerun=False, allow_warmstart=False, persistent=True, opt=None, ): log_time(logger, timings, "run_start", comment="Backend: starting model run") warmstart = False if not backend_rerun: backend_model = backend.generate_model(model_data) log_time( logger, timings, "run_backend_model_generated", time_since_run_start=True, comment="Backend: model generated", ) else: backend_model = backend_rerun if allow_warmstart: warmstart = True run_config = UpdateObserverDict( initial_yaml_string=model_data.attrs["run_config"], name="run_config", observer=model_data, ) solver = run_config["solver"] solver_io = run_config.get("solver_io", None) solver_options = run_config.get("solver_options", None) save_logs = run_config.get("save_logs", None) if build_only: results = xr.Dataset() else: if "persistent" in solver and persistent is False: exceptions.warn( f"The chosen solver, `{solver}` will not be used in this run. " f"`{solver.replace('_persistent', '')}` will be used instead.") solver = solver.replace("_persistent", "") log_time( logger, timings, "run_solver_start", comment="Backend: sending model to solver", ) backend_results, opt = backend.solve_model( backend_model, solver=solver, solver_io=solver_io, solver_options=solver_options, save_logs=save_logs, warmstart=warmstart, opt=opt, ) log_time( logger, timings, "run_solver_exit", time_since_run_start=True, comment="Backend: solver finished running", ) termination = backend.load_results(backend_model, backend_results, opt) log_time(logger, timings, "run_results_loaded", comment="Backend: loaded results") if termination in ["optimal", "feasible"]: results = backend.get_result_array(backend_model, model_data) results.attrs["termination_condition"] = termination if "persistent" in opt.name and persistent is True: results.attrs["objective_function_value"] = opt.get_model_attr( "ObjVal") else: results.attrs["objective_function_value"] = backend_model.obj() else: results = xr.Dataset(attrs={"termination_condition": termination}) log_time( logger, timings, "run_solution_returned", time_since_run_start=True, comment="Backend: generated solution array", ) return results, backend_model, opt