def init_set(self,
name: str,
idx_sets: Sequence[str] = None,
idx_names: Sequence[str] = None) -> None:
"""Initialize a new set.
Parameters
----------
name : str
Name of the set.
idx_sets : sequence of str or str, optional
Names of other sets that index this set.
idx_names : sequence of str or str, optional
Names of the dimensions indexed by `idx_sets`.
Raises
------
ValueError
If the set (or another object with the same *name*) already exists.
RuntimeError
If the Scenario is not checked out (see :meth:`~TimeSeries.check_out`).
"""
idx_sets = as_str_list(idx_sets) or []
idx_names = as_str_list(idx_names)
return self._backend("init_item", "set", name, idx_sets, idx_names)
def run(self, scenario):
"""Execute the model."""
backend = scenario.platform._backend
self.scenario = scenario
if self.use_temp_dir:
# Create a temporary directory; automatically deleted at the end of
# the context
_temp_dir = TemporaryDirectory()
self.temp_dir = _temp_dir.name
def format(key):
value = getattr(self, key)
try:
return value.format(**self.__dict__)
except AttributeError:
# Something like a Path; don't format it
return value
# Process args in order
command = ['gams']
model_file = Path(format('model_file'))
command.append('"{}"'.format(model_file))
self.case = format('case').replace(' ', '_')
self.in_file = Path(format('in_file'))
self.out_file = Path(format('out_file'))
for arg in self.solve_args:
command.append(arg.format(**self.__dict__))
command.extend(self.gams_args)
if os.name == 'nt':
command = ' '.join(command)
s_arg = dict(filters=dict(scenario=scenario))
try:
# Write model data to file
backend.write_file(self.in_file, ItemType.SET | ItemType.PAR,
**s_arg)
except NotImplementedError: # pragma: no cover
# Currently there is no such Backend
raise NotImplementedError('GAMSModel requires a Backend that can '
'write to GDX files, e.g. JDBCBackend')
# Invoke GAMS
cwd = self.temp_dir if self.use_temp_dir else model_file.parent
check_call(command, shell=os.name == 'nt', cwd=cwd)
# Read model solution
backend.read_file(self.out_file, ItemType.MODEL, **s_arg,
check_solution=self.check_solution,
comment=self.comment or '',
equ_list=as_str_list(self.equ_list) or [],
var_list=as_str_list(self.var_list) or [],
)
def _keys(self, name, key_or_keys):
if isinstance(key_or_keys, (list, pd.Series)):
return as_str_list(key_or_keys)
elif isinstance(key_or_keys, (pd.DataFrame, dict)):
if isinstance(key_or_keys, dict):
key_or_keys = pd.DataFrame.from_dict(key_or_keys, orient="columns")
idx_names = self.idx_names(name)
return [as_str_list(row, idx_names) for _, row in key_or_keys.iterrows()]
else:
return [str(key_or_keys)]
def run(self, scenario):
"""Execute the model."""
if not isinstance(scenario.platform._backend, JDBCBackend):
raise ValueError('GAMSModel can only solve Scenarios with '
'JDBCBackend')
self.scenario = scenario
def format(key):
value = getattr(self, key)
try:
return value.format(**self.__dict__)
except AttributeError:
# Something like a Path; don't format it
return value
# Process args in order
command = ['gams']
model_file = Path(format('model_file')).resolve()
command.append('"{}"'.format(model_file))
self.case = format('case').replace(' ', '_')
self.in_file = Path(format('in_file')).resolve()
self.out_file = Path(format('out_file')).resolve()
for arg in self.solve_args:
command.append(arg.format(**self.__dict__))
command.extend(self.gams_args)
if os.name == 'nt':
command = ' '.join(command)
# Write model data to file
scenario._backend('write_gdx', self.in_file)
# Invoke GAMS
check_call(command, shell=os.name == 'nt', cwd=model_file.parent)
# Reset Python data cache
scenario.clear_cache()
# Read model solution
scenario._backend('read_gdx', self.out_file,
self.check_solution,
self.comment or '',
as_str_list(self.equ_list) or [],
as_str_list(self.var_list) or [],
)
def init_equ(self, name: str, idx_sets=None, idx_names=None) -> None:
"""Initialize a new equation.
Parameters
----------
name : str
Name of the equation.
idx_sets : sequence of str or str, optional
Name(s) of index sets for a 1+-dimensional variable.
idx_names : sequence of str or str, optional
Names of the dimensions indexed by `idx_sets`.
"""
idx_sets = as_str_list(idx_sets) or []
idx_names = as_str_list(idx_names)
return self._backend("init_item", "equ", name, idx_sets, idx_names)
def check_access(self,
user: str,
models: Union[str, Sequence[str]],
access: str = "view") -> Union[bool, Dict[str, bool]]:
"""Check access to specific models.
Parameters
----------
user: str
Registered user name
models : str or list of str
Model(s) name
access : str, optional
Access type - view or edit
Returns
-------
bool or dict of bool
"""
models_list = as_str_list(models)
if not models_list:
raise ValueError("must supply at least 1 model name")
result = self._backend.get_auth(user, models_list, access)
if isinstance(models, str):
return result[models]
else:
return {model: result.get(model) == 1 for model in models_list}
def init_par(
self, name: str, idx_sets: Sequence[str], idx_names: Sequence[str] = None
) -> None:
"""Initialize a new parameter.
Parameters
----------
name : str
Name of the parameter.
idx_sets : sequence of str or str, optional
Names of sets that index this parameter.
idx_names : sequence of str or str, optional
Names of the dimensions indexed by `idx_sets`.
"""
idx_sets = as_str_list(idx_sets) or []
idx_names = as_str_list(idx_names)
return self._backend("init_item", "par", name, idx_sets, idx_names)
def remove_meta(self, name: Union[str, Sequence[str]]) -> None:
"""Remove :ref:`data-meta` for this object.
Parameters
----------
name : str or list of str
Either single metadata name/identifier, or list of names.
"""
self.platform._backend.remove_meta(as_str_list(name), self.model,
self.scenario, self.version)
def add_cat(self, name, cat, keys, is_unique=False):
"""Map elements from *keys* to category *cat* within set *name*.
Parameters
----------
name : str
Name of the set.
cat : str
Name of the category.
keys : str or list of str
Element keys to be added to the category mapping.
is_unique: bool, optional
If `True`, then *cat* must have only one element. An exception is
raised if *cat* already has an element, or if ``len(keys) > 1``.
"""
self._backend("cat_set_elements", name, str(cat), as_str_list(keys), is_unique)
def export_timeseries_data(
self,
path: PathLike,
default: bool = True,
model: str = None,
scenario: str = None,
variable=None,
unit=None,
region=None,
export_all_runs: bool = False,
) -> None:
"""Export time series data to CSV file across multiple :class:`.TimeSeries`.
Refer :meth:`.TimeSeries.add_timeseries` about adding time series data.
Parameters
----------
path : os.PathLike
File name to export data to; must have the suffix '.csv'.
Result file will contain the following columns:
- model
- scenario
- version
- variable
- unit
- region
- meta
- subannual
- year
- value
default : bool, optional
:obj:`True` to include only TimeSeries versions marked as default.
model: str, optional
Only return data for this model name.
scenario: str, optional
Only return data for this scenario name.
variable: list of str, optional
Only return data for variable name(s) in this list.
unit: list of str, optional
Only return data for unit name(s) in this list.
region: list of str, optional
Only return data for region(s) in this list.
export_all_runs: boolean, optional
Export all existing model+scenario run combinations.
"""
if export_all_runs and (model or scenario):
raise ValueError(
"Invalid arguments: export_all_runs cannot be used when providing a "
"model or scenario.")
filters = {
"model": as_str_list(model) or [],
"scenario": as_str_list(scenario) or [],
"variable": as_str_list(variable) or [],
"unit": as_str_list(unit) or [],
"region": as_str_list(region) or [],
"default": default,
"export_all_runs": export_all_runs,
}
self._backend.write_file(path, ItemType.TS, filters=filters)
def item_get_elements(self, s, type, name, filters=None):
if filters:
# Convert filter elements to strings
filters = {dim: as_str_list(ele) for dim, ele in filters.items()}
try:
# Retrieve the cached value with this exact set of filters
return self.cache_get(s, type, name, filters)
except KeyError:
pass # Cache miss
try:
# Retrieve a cached, unfiltered value of the same item
unfiltered = self.cache_get(s, type, name, None)
except KeyError:
pass # Cache miss
else:
# Success; filter and return
return filtered(unfiltered, filters)
# Failed to load item from cache
# Retrieve the item
item = self._get_item(s, type, name, load=True)
idx_names = list(item.getIdxNames())
idx_sets = list(item.getIdxSets())
# Get list of elements, using filters if provided
if filters is not None:
jFilter = java.HashMap()
for idx_name, values in filters.items():
# Retrieve the elements of the index set as a list
idx_set = idx_sets[idx_names.index(idx_name)]
elements = self.item_get_elements(s, 'set', idx_set).tolist()
# Filter for only included values and store
filtered_elements = filter(lambda e: e in values, elements)
jFilter.put(idx_name, to_jlist(filtered_elements))
jList = item.getElements(jFilter)
else:
jList = item.getElements()
if item.getDim() > 0:
# Mapping set or multi-dimensional equation, parameter, or variable
columns = copy(idx_names)
# Prepare dtypes for index columns
dtypes = {}
for idx_name, idx_set in zip(columns, idx_sets):
# NB using categoricals could be more memory-efficient, but
# requires adjustment of tests/documentation. See
# https://github.com/iiasa/ixmp/issues/228
# dtypes[idx_name] = CategoricalDtype(
# self.item_get_elements(s, 'set', idx_set))
dtypes[idx_name] = str
# Prepare dtypes for additional columns
if type == 'par':
columns.extend(['value', 'unit'])
dtypes['value'] = float
# Same as above
# dtypes['unit'] = CategoricalDtype(self.jobj.getUnitList())
dtypes['unit'] = str
elif type in ('equ', 'var'):
columns.extend(['lvl', 'mrg'])
dtypes.update({'lvl': float, 'mrg': float})
# Prepare empty DataFrame
result = pd.DataFrame(index=pd.RangeIndex(len(jList)),
columns=columns)
# Copy vectors from Java into DataFrame columns
# NB [:] causes JPype to use a faster code path
for i in range(len(idx_sets)):
result.iloc[:, i] = item.getCol(i, jList)[:]
if type == 'par':
result.loc[:, 'value'] = item.getValues(jList)[:]
result.loc[:, 'unit'] = item.getUnits(jList)[:]
elif type in ('equ', 'var'):
result.loc[:, 'lvl'] = item.getLevels(jList)[:]
result.loc[:, 'mrg'] = item.getMarginals(jList)[:]
# .loc assignment above modifies dtypes; set afterwards
result = result.astype(dtypes)
elif type == 'set':
# Index sets
# dtype=object is to silence a warning in pandas 1.0
result = pd.Series(item.getCol(0, jList), dtype=object)
elif type == 'par':
# Scalar parameters
result = dict(value=item.getScalarValue().floatValue(),
unit=str(item.getScalarUnit()))
elif type in ('equ', 'var'):
# Scalar equations and variables
result = dict(lvl=item.getScalarLevel().floatValue(),
mrg=item.getScalarMarginal().floatValue())
# Store cache
self.cache(s, type, name, filters, result)
return result
def add_set(
self,
name: str,
key: Union[str, Sequence[str], Dict, pd.DataFrame],
comment: str = None,
) -> None:
"""Add elements to an existing set.
Parameters
----------
name : str
Name of the set.
key : str or iterable of str or dict or :class:`pandas.DataFrame`
Element(s) to be added. If `name` exists, the elements are appended to
existing elements.
comment : str or iterable of str, optional
Comment describing the element(s). If given, there must be the same number
of comments as elements.
Raises
------
KeyError
If the set `name` does not exist. :meth:`init_set` must be called before
:meth:`add_set`.
ValueError
For invalid forms or combinations of `key` and `comment`.
"""
# TODO expand docstring (here or in doc/source/api.rst) with examples, per
# test_scenario.test_add_set.
if isinstance(key, list) and len(key) == 0:
return # No elements to add
# Get index names for set *name*, may raise KeyError
idx_names = self.idx_names(name)
# Check arguments and convert to two lists: keys and comments
if len(idx_names) == 0:
# Basic set. Keys must be strings.
if isinstance(key, (dict, pd.DataFrame)):
raise TypeError(
f"keys for basic set {repr(name)} must be str or list of str; got "
f"{type(key)}")
# Ensure keys is a list of str
keys = as_str_list(key)
else:
# Set defined over 1+ other sets
# Check for ambiguous arguments
if comment and isinstance(
key, (dict, pd.DataFrame)) and "comment" in key:
raise ValueError(
"ambiguous; both key['comment'] and comment= given")
if isinstance(key, pd.DataFrame):
# DataFrame of key values and perhaps comments
try:
# Pop a 'comment' column off the DataFrame, convert to list
comment = key.pop("comment").to_list()
except KeyError:
pass
# Convert key to list of list of key values
keys = []
for row in key.to_dict(orient="records"):
keys.append(as_str_list(row, idx_names=idx_names))
elif isinstance(key, dict):
# Dict of lists of key values
# Pop a 'comment' list from the dict
comment = key.pop("comment", None)
# Convert to list of list of key values
keys = list(map(as_str_list,
zip(*[key[i] for i in idx_names])))
elif isinstance(key[0], str):
# List of key values; wrap
keys = [as_str_list(key)]
elif isinstance(key[0], list):
# List of lists of key values; convert to list of list of str
keys = list(map(as_str_list, key))
elif isinstance(key, str) and len(idx_names) == 1:
# Bare key given for a 1D set; wrap for convenience
keys = [[key]]
else:
# Other, invalid value
raise ValueError(key)
# Process comments to a list of str, or let them all be None
comments = as_str_list(comment) if comment else repeat(None, len(keys))
# Combine iterators to tuples. If the lengths are mismatched, the sentinel
# value 'False' is filled in
to_add = list(zip_longest(keys, comments, fillvalue=False))
# Check processed arguments
for e, c in to_add:
# Check for sentinel values
if e is False:
raise ValueError(f"Comment {repr(c)} without matching key")
elif c is False:
raise ValueError(f"Key {repr(e)} without matching comment")
elif len(idx_names) and len(idx_names) != len(e):
raise ValueError(
f"{len(e)}-D key {repr(e)} invalid for "
f"{len(idx_names)}-D set {name}{repr(idx_names)}")
# Send to backend
elements = ((kc[0], None, None, kc[1]) for kc in to_add)
self._backend("item_set_elements", "set", name, elements)
def run(self, scenario):
"""Execute the model."""
# Store the scenario so its attributes can be referenced by format()
self.scenario = scenario
# Format or retrieve the model file option
model_file = Path(self.format_option("model_file"))
# Determine working directory for the GAMS call, possibly a temporary directory
self.cwd = Path(
tempfile.mkdtemp()) if self.use_temp_dir else model_file.parent
# The "case" name
self.case = self.clean_path(
self.format_option("case").replace(" ", "_"))
# Input and output file names
self.in_file = Path(self.format_option("in_file"))
self.out_file = Path(self.format_option("out_file"))
# Assemble the full command: executable, model file, model-specific arguments,
# and general GAMS arguments
command = (["gams", f'"{model_file}"'] +
[self.format(arg)
for arg in self.solve_args] + self.gams_args)
if os.name == "nt":
# Windows: join the commands to a single string
command = " ".join(command)
# Remove stored reference to the Scenario to allow it to be GC'd later
delattr(self, "scenario")
# Common argument for write_file and read_file
s_arg = dict(filters=dict(scenario=scenario))
try:
# Write model data to file
scenario.platform._backend.write_file(self.in_file,
ItemType.SET | ItemType.PAR,
**s_arg)
except NotImplementedError: # pragma: no cover
# No coverage because there currently is no such Backend that doesn't
# support GDX
# Remove the temporary directory, which should be empty
self.remove_temp_dir()
raise NotImplementedError(
"GAMSModel requires a Backend that can write to GDX files, e.g. "
"JDBCBackend")
try:
# Invoke GAMS
check_call(command, shell=os.name == "nt", cwd=self.cwd)
except CalledProcessError as exc:
# Do not remove self.temp_dir; the user may want to inspect the GDX file
raise self.format_exception(exc, model_file) from None
# Read model solution
scenario.platform._backend.read_file(
self.out_file,
ItemType.MODEL,
**s_arg,
check_solution=self.check_solution,
comment=self.comment or "",
equ_list=as_str_list(self.equ_list) or [],
var_list=as_str_list(self.var_list) or [],
)
# Finished: remove the temporary directory, if any
self.remove_temp_dir()
def timeseries(
self,
region: Union[str, Sequence[str]] = None,
variable: Union[str, Sequence[str]] = None,
unit: Union[str, Sequence[str]] = None,
year: Union[int, Sequence[int]] = None,
iamc: bool = False,
subannual: Union[bool, str] = "auto",
) -> pd.DataFrame:
"""Retrieve time series data.
Parameters
----------
iamc : bool, optional
Return data in wide/'IAMC' format. If :obj:`False`, return data in long
format; see :meth:`add_timeseries`.
region : str or list of str, optional
Regions to include in returned data.
variable : str or list of str, optional
Variables to include in returned data.
unit : str or list of str, optional
Units to include in returned data.
year : str or int or list of (str or int), optional
Years to include in returned data.
subannual : bool or 'auto', optional
Whether to include column for sub-annual specification (if :class:`bool`);
if 'auto', include column if sub-annual data (other than 'Year') exists in
returned data frame.
Raises
------
ValueError
If `subannual` is :obj:`False` but Scenario has (filtered) sub-annual data.
Returns
-------
pandas.DataFrame
Specified data.
"""
# Retrieve data, convert to pandas.DataFrame
df = pd.DataFrame(
self._backend(
"get_data",
as_str_list(region) or [],
as_str_list(variable) or [],
as_str_list(unit) or [],
as_str_list(year) or [],
),
columns=FIELDS["ts_get"],
)
df["model"] = self.model
df["scenario"] = self.scenario
# drop `subannual` column if not requested (False) or required ('auto')
if subannual is not True:
has_subannual = not all(df["subannual"] == "Year")
if subannual is False and has_subannual:
msg = (
"timeseries data has subannual values, ",
"use `subannual=True or 'auto'`",
)
raise ValueError(msg)
if not has_subannual:
df.drop("subannual", axis=1, inplace=True)
if iamc:
# Convert to wide format
index = IAMC_IDX
if "subannual" in df.columns:
index = index + ["subannual"]
df = df.pivot_table(index=index,
columns="year")["value"].reset_index()
df.columns.names = [None]
return df
