def init_set(self, name: str, idx_sets: Sequence[str] = None, idx_names: Sequence[str] = None) -> None: """Initialize a new set. Parameters ---------- name : str Name of the set. idx_sets : sequence of str or str, optional Names of other sets that index this set. idx_names : sequence of str or str, optional Names of the dimensions indexed by `idx_sets`. Raises ------ ValueError If the set (or another object with the same *name*) already exists. RuntimeError If the Scenario is not checked out (see :meth:`~TimeSeries.check_out`). """ idx_sets = as_str_list(idx_sets) or [] idx_names = as_str_list(idx_names) return self._backend("init_item", "set", name, idx_sets, idx_names)
def run(self, scenario): """Execute the model.""" backend = scenario.platform._backend self.scenario = scenario if self.use_temp_dir: # Create a temporary directory; automatically deleted at the end of # the context _temp_dir = TemporaryDirectory() self.temp_dir = _temp_dir.name def format(key): value = getattr(self, key) try: return value.format(**self.__dict__) except AttributeError: # Something like a Path; don't format it return value # Process args in order command = ['gams'] model_file = Path(format('model_file')) command.append('"{}"'.format(model_file)) self.case = format('case').replace(' ', '_') self.in_file = Path(format('in_file')) self.out_file = Path(format('out_file')) for arg in self.solve_args: command.append(arg.format(**self.__dict__)) command.extend(self.gams_args) if os.name == 'nt': command = ' '.join(command) s_arg = dict(filters=dict(scenario=scenario)) try: # Write model data to file backend.write_file(self.in_file, ItemType.SET | ItemType.PAR, **s_arg) except NotImplementedError: # pragma: no cover # Currently there is no such Backend raise NotImplementedError('GAMSModel requires a Backend that can ' 'write to GDX files, e.g. JDBCBackend') # Invoke GAMS cwd = self.temp_dir if self.use_temp_dir else model_file.parent check_call(command, shell=os.name == 'nt', cwd=cwd) # Read model solution backend.read_file(self.out_file, ItemType.MODEL, **s_arg, check_solution=self.check_solution, comment=self.comment or '', equ_list=as_str_list(self.equ_list) or [], var_list=as_str_list(self.var_list) or [], )
def _keys(self, name, key_or_keys): if isinstance(key_or_keys, (list, pd.Series)): return as_str_list(key_or_keys) elif isinstance(key_or_keys, (pd.DataFrame, dict)): if isinstance(key_or_keys, dict): key_or_keys = pd.DataFrame.from_dict(key_or_keys, orient="columns") idx_names = self.idx_names(name) return [as_str_list(row, idx_names) for _, row in key_or_keys.iterrows()] else: return [str(key_or_keys)]
def run(self, scenario): """Execute the model.""" if not isinstance(scenario.platform._backend, JDBCBackend): raise ValueError('GAMSModel can only solve Scenarios with ' 'JDBCBackend') self.scenario = scenario def format(key): value = getattr(self, key) try: return value.format(**self.__dict__) except AttributeError: # Something like a Path; don't format it return value # Process args in order command = ['gams'] model_file = Path(format('model_file')).resolve() command.append('"{}"'.format(model_file)) self.case = format('case').replace(' ', '_') self.in_file = Path(format('in_file')).resolve() self.out_file = Path(format('out_file')).resolve() for arg in self.solve_args: command.append(arg.format(**self.__dict__)) command.extend(self.gams_args) if os.name == 'nt': command = ' '.join(command) # Write model data to file scenario._backend('write_gdx', self.in_file) # Invoke GAMS check_call(command, shell=os.name == 'nt', cwd=model_file.parent) # Reset Python data cache scenario.clear_cache() # Read model solution scenario._backend('read_gdx', self.out_file, self.check_solution, self.comment or '', as_str_list(self.equ_list) or [], as_str_list(self.var_list) or [], )
def init_equ(self, name: str, idx_sets=None, idx_names=None) -> None: """Initialize a new equation. Parameters ---------- name : str Name of the equation. idx_sets : sequence of str or str, optional Name(s) of index sets for a 1+-dimensional variable. idx_names : sequence of str or str, optional Names of the dimensions indexed by `idx_sets`. """ idx_sets = as_str_list(idx_sets) or [] idx_names = as_str_list(idx_names) return self._backend("init_item", "equ", name, idx_sets, idx_names)
def check_access(self, user: str, models: Union[str, Sequence[str]], access: str = "view") -> Union[bool, Dict[str, bool]]: """Check access to specific models. Parameters ---------- user: str Registered user name models : str or list of str Model(s) name access : str, optional Access type - view or edit Returns ------- bool or dict of bool """ models_list = as_str_list(models) if not models_list: raise ValueError("must supply at least 1 model name") result = self._backend.get_auth(user, models_list, access) if isinstance(models, str): return result[models] else: return {model: result.get(model) == 1 for model in models_list}
def init_par( self, name: str, idx_sets: Sequence[str], idx_names: Sequence[str] = None ) -> None: """Initialize a new parameter. Parameters ---------- name : str Name of the parameter. idx_sets : sequence of str or str, optional Names of sets that index this parameter. idx_names : sequence of str or str, optional Names of the dimensions indexed by `idx_sets`. """ idx_sets = as_str_list(idx_sets) or [] idx_names = as_str_list(idx_names) return self._backend("init_item", "par", name, idx_sets, idx_names)
def remove_meta(self, name: Union[str, Sequence[str]]) -> None: """Remove :ref:`data-meta` for this object. Parameters ---------- name : str or list of str Either single metadata name/identifier, or list of names. """ self.platform._backend.remove_meta(as_str_list(name), self.model, self.scenario, self.version)
def add_cat(self, name, cat, keys, is_unique=False): """Map elements from *keys* to category *cat* within set *name*. Parameters ---------- name : str Name of the set. cat : str Name of the category. keys : str or list of str Element keys to be added to the category mapping. is_unique: bool, optional If `True`, then *cat* must have only one element. An exception is raised if *cat* already has an element, or if ``len(keys) > 1``. """ self._backend("cat_set_elements", name, str(cat), as_str_list(keys), is_unique)
def export_timeseries_data( self, path: PathLike, default: bool = True, model: str = None, scenario: str = None, variable=None, unit=None, region=None, export_all_runs: bool = False, ) -> None: """Export time series data to CSV file across multiple :class:`.TimeSeries`. Refer :meth:`.TimeSeries.add_timeseries` about adding time series data. Parameters ---------- path : os.PathLike File name to export data to; must have the suffix '.csv'. Result file will contain the following columns: - model - scenario - version - variable - unit - region - meta - subannual - year - value default : bool, optional :obj:`True` to include only TimeSeries versions marked as default. model: str, optional Only return data for this model name. scenario: str, optional Only return data for this scenario name. variable: list of str, optional Only return data for variable name(s) in this list. unit: list of str, optional Only return data for unit name(s) in this list. region: list of str, optional Only return data for region(s) in this list. export_all_runs: boolean, optional Export all existing model+scenario run combinations. """ if export_all_runs and (model or scenario): raise ValueError( "Invalid arguments: export_all_runs cannot be used when providing a " "model or scenario.") filters = { "model": as_str_list(model) or [], "scenario": as_str_list(scenario) or [], "variable": as_str_list(variable) or [], "unit": as_str_list(unit) or [], "region": as_str_list(region) or [], "default": default, "export_all_runs": export_all_runs, } self._backend.write_file(path, ItemType.TS, filters=filters)
def item_get_elements(self, s, type, name, filters=None): if filters: # Convert filter elements to strings filters = {dim: as_str_list(ele) for dim, ele in filters.items()} try: # Retrieve the cached value with this exact set of filters return self.cache_get(s, type, name, filters) except KeyError: pass # Cache miss try: # Retrieve a cached, unfiltered value of the same item unfiltered = self.cache_get(s, type, name, None) except KeyError: pass # Cache miss else: # Success; filter and return return filtered(unfiltered, filters) # Failed to load item from cache # Retrieve the item item = self._get_item(s, type, name, load=True) idx_names = list(item.getIdxNames()) idx_sets = list(item.getIdxSets()) # Get list of elements, using filters if provided if filters is not None: jFilter = java.HashMap() for idx_name, values in filters.items(): # Retrieve the elements of the index set as a list idx_set = idx_sets[idx_names.index(idx_name)] elements = self.item_get_elements(s, 'set', idx_set).tolist() # Filter for only included values and store filtered_elements = filter(lambda e: e in values, elements) jFilter.put(idx_name, to_jlist(filtered_elements)) jList = item.getElements(jFilter) else: jList = item.getElements() if item.getDim() > 0: # Mapping set or multi-dimensional equation, parameter, or variable columns = copy(idx_names) # Prepare dtypes for index columns dtypes = {} for idx_name, idx_set in zip(columns, idx_sets): # NB using categoricals could be more memory-efficient, but # requires adjustment of tests/documentation. See # https://github.com/iiasa/ixmp/issues/228 # dtypes[idx_name] = CategoricalDtype( # self.item_get_elements(s, 'set', idx_set)) dtypes[idx_name] = str # Prepare dtypes for additional columns if type == 'par': columns.extend(['value', 'unit']) dtypes['value'] = float # Same as above # dtypes['unit'] = CategoricalDtype(self.jobj.getUnitList()) dtypes['unit'] = str elif type in ('equ', 'var'): columns.extend(['lvl', 'mrg']) dtypes.update({'lvl': float, 'mrg': float}) # Prepare empty DataFrame result = pd.DataFrame(index=pd.RangeIndex(len(jList)), columns=columns) # Copy vectors from Java into DataFrame columns # NB [:] causes JPype to use a faster code path for i in range(len(idx_sets)): result.iloc[:, i] = item.getCol(i, jList)[:] if type == 'par': result.loc[:, 'value'] = item.getValues(jList)[:] result.loc[:, 'unit'] = item.getUnits(jList)[:] elif type in ('equ', 'var'): result.loc[:, 'lvl'] = item.getLevels(jList)[:] result.loc[:, 'mrg'] = item.getMarginals(jList)[:] # .loc assignment above modifies dtypes; set afterwards result = result.astype(dtypes) elif type == 'set': # Index sets # dtype=object is to silence a warning in pandas 1.0 result = pd.Series(item.getCol(0, jList), dtype=object) elif type == 'par': # Scalar parameters result = dict(value=item.getScalarValue().floatValue(), unit=str(item.getScalarUnit())) elif type in ('equ', 'var'): # Scalar equations and variables result = dict(lvl=item.getScalarLevel().floatValue(), mrg=item.getScalarMarginal().floatValue()) # Store cache self.cache(s, type, name, filters, result) return result
def add_set( self, name: str, key: Union[str, Sequence[str], Dict, pd.DataFrame], comment: str = None, ) -> None: """Add elements to an existing set. Parameters ---------- name : str Name of the set. key : str or iterable of str or dict or :class:`pandas.DataFrame` Element(s) to be added. If `name` exists, the elements are appended to existing elements. comment : str or iterable of str, optional Comment describing the element(s). If given, there must be the same number of comments as elements. Raises ------ KeyError If the set `name` does not exist. :meth:`init_set` must be called before :meth:`add_set`. ValueError For invalid forms or combinations of `key` and `comment`. """ # TODO expand docstring (here or in doc/source/api.rst) with examples, per # test_scenario.test_add_set. if isinstance(key, list) and len(key) == 0: return # No elements to add # Get index names for set *name*, may raise KeyError idx_names = self.idx_names(name) # Check arguments and convert to two lists: keys and comments if len(idx_names) == 0: # Basic set. Keys must be strings. if isinstance(key, (dict, pd.DataFrame)): raise TypeError( f"keys for basic set {repr(name)} must be str or list of str; got " f"{type(key)}") # Ensure keys is a list of str keys = as_str_list(key) else: # Set defined over 1+ other sets # Check for ambiguous arguments if comment and isinstance( key, (dict, pd.DataFrame)) and "comment" in key: raise ValueError( "ambiguous; both key['comment'] and comment= given") if isinstance(key, pd.DataFrame): # DataFrame of key values and perhaps comments try: # Pop a 'comment' column off the DataFrame, convert to list comment = key.pop("comment").to_list() except KeyError: pass # Convert key to list of list of key values keys = [] for row in key.to_dict(orient="records"): keys.append(as_str_list(row, idx_names=idx_names)) elif isinstance(key, dict): # Dict of lists of key values # Pop a 'comment' list from the dict comment = key.pop("comment", None) # Convert to list of list of key values keys = list(map(as_str_list, zip(*[key[i] for i in idx_names]))) elif isinstance(key[0], str): # List of key values; wrap keys = [as_str_list(key)] elif isinstance(key[0], list): # List of lists of key values; convert to list of list of str keys = list(map(as_str_list, key)) elif isinstance(key, str) and len(idx_names) == 1: # Bare key given for a 1D set; wrap for convenience keys = [[key]] else: # Other, invalid value raise ValueError(key) # Process comments to a list of str, or let them all be None comments = as_str_list(comment) if comment else repeat(None, len(keys)) # Combine iterators to tuples. If the lengths are mismatched, the sentinel # value 'False' is filled in to_add = list(zip_longest(keys, comments, fillvalue=False)) # Check processed arguments for e, c in to_add: # Check for sentinel values if e is False: raise ValueError(f"Comment {repr(c)} without matching key") elif c is False: raise ValueError(f"Key {repr(e)} without matching comment") elif len(idx_names) and len(idx_names) != len(e): raise ValueError( f"{len(e)}-D key {repr(e)} invalid for " f"{len(idx_names)}-D set {name}{repr(idx_names)}") # Send to backend elements = ((kc[0], None, None, kc[1]) for kc in to_add) self._backend("item_set_elements", "set", name, elements)
def run(self, scenario): """Execute the model.""" # Store the scenario so its attributes can be referenced by format() self.scenario = scenario # Format or retrieve the model file option model_file = Path(self.format_option("model_file")) # Determine working directory for the GAMS call, possibly a temporary directory self.cwd = Path( tempfile.mkdtemp()) if self.use_temp_dir else model_file.parent # The "case" name self.case = self.clean_path( self.format_option("case").replace(" ", "_")) # Input and output file names self.in_file = Path(self.format_option("in_file")) self.out_file = Path(self.format_option("out_file")) # Assemble the full command: executable, model file, model-specific arguments, # and general GAMS arguments command = (["gams", f'"{model_file}"'] + [self.format(arg) for arg in self.solve_args] + self.gams_args) if os.name == "nt": # Windows: join the commands to a single string command = " ".join(command) # Remove stored reference to the Scenario to allow it to be GC'd later delattr(self, "scenario") # Common argument for write_file and read_file s_arg = dict(filters=dict(scenario=scenario)) try: # Write model data to file scenario.platform._backend.write_file(self.in_file, ItemType.SET | ItemType.PAR, **s_arg) except NotImplementedError: # pragma: no cover # No coverage because there currently is no such Backend that doesn't # support GDX # Remove the temporary directory, which should be empty self.remove_temp_dir() raise NotImplementedError( "GAMSModel requires a Backend that can write to GDX files, e.g. " "JDBCBackend") try: # Invoke GAMS check_call(command, shell=os.name == "nt", cwd=self.cwd) except CalledProcessError as exc: # Do not remove self.temp_dir; the user may want to inspect the GDX file raise self.format_exception(exc, model_file) from None # Read model solution scenario.platform._backend.read_file( self.out_file, ItemType.MODEL, **s_arg, check_solution=self.check_solution, comment=self.comment or "", equ_list=as_str_list(self.equ_list) or [], var_list=as_str_list(self.var_list) or [], ) # Finished: remove the temporary directory, if any self.remove_temp_dir()
def timeseries( self, region: Union[str, Sequence[str]] = None, variable: Union[str, Sequence[str]] = None, unit: Union[str, Sequence[str]] = None, year: Union[int, Sequence[int]] = None, iamc: bool = False, subannual: Union[bool, str] = "auto", ) -> pd.DataFrame: """Retrieve time series data. Parameters ---------- iamc : bool, optional Return data in wide/'IAMC' format. If :obj:`False`, return data in long format; see :meth:`add_timeseries`. region : str or list of str, optional Regions to include in returned data. variable : str or list of str, optional Variables to include in returned data. unit : str or list of str, optional Units to include in returned data. year : str or int or list of (str or int), optional Years to include in returned data. subannual : bool or 'auto', optional Whether to include column for sub-annual specification (if :class:`bool`); if 'auto', include column if sub-annual data (other than 'Year') exists in returned data frame. Raises ------ ValueError If `subannual` is :obj:`False` but Scenario has (filtered) sub-annual data. Returns ------- pandas.DataFrame Specified data. """ # Retrieve data, convert to pandas.DataFrame df = pd.DataFrame( self._backend( "get_data", as_str_list(region) or [], as_str_list(variable) or [], as_str_list(unit) or [], as_str_list(year) or [], ), columns=FIELDS["ts_get"], ) df["model"] = self.model df["scenario"] = self.scenario # drop `subannual` column if not requested (False) or required ('auto') if subannual is not True: has_subannual = not all(df["subannual"] == "Year") if subannual is False and has_subannual: msg = ( "timeseries data has subannual values, ", "use `subannual=True or 'auto'`", ) raise ValueError(msg) if not has_subannual: df.drop("subannual", axis=1, inplace=True) if iamc: # Convert to wide format index = IAMC_IDX if "subannual" in df.columns: index = index + ["subannual"] df = df.pivot_table(index=index, columns="year")["value"].reset_index() df.columns.names = [None] return df