def load(self):
if self.loaded:
logger.info("Nothing to do. Symbol already loaded.")
return
if not self.file:
raise Error(
"Cannot load {} because there is no file pointer".format(
repr(self)))
if not self.index:
raise Error(
"Cannot load {} because there is no symbol index".format(
repr(self)))
if self.data_type == GamsDataType.Parameter and HAVE_GDX2PY:
self.dataframe = gdx2py.par2list(self.file.filename, self.name)
self._loaded = True
return
data = []
ret, records = gdxcc.gdxDataReadStrStart(self.file.H, self.index)
for i in range(records):
ret, elements, values, afdim = gdxcc.gdxDataReadStr(self.file.H)
# make sure we pick value columns up correctly
data.append(
elements +
[values[col_ind] for col_name, col_ind in self.value_cols])
if self.data_type == GamsDataType.Set:
data[-1][-1] = True
# gdxdict called gdxGetElemText here, but I do not currently see value in doing that
self.dataframe = data
if not self.data_type == GamsDataType.Set:
self.dataframe = convert_gdx_to_np_svs(self.dataframe,
self.num_dims, self.file)
self._loaded = True
return
def dims(self, value):
if (self._dims is not None) and (self.loaded and
((self.num_dims > 0) or
(self.num_records > 0))):
if not isinstance(value, list) or len(value) != self.num_dims:
raise Error(
f"Cannot set dims to {value}, because the number of "
"dimensions has already been set to {self.num_dims}.")
if isinstance(value, int):
self._dims = ['*'] * value
self._init_dataframe()
return
if not isinstance(value, list):
raise Error(
'dims must be an int or a list. Was passed {} of type {}.'.
format(value, type(value)))
for dim in value:
if not isinstance(dim, str):
raise Error(
'Individual dimensions must be denoted by strings. Was passed {} as element of {}.'
.format(dim, value))
assert (self._dims is None) or (
self.loaded and (self.num_dims == 0) and
(self.num_records == 0)) or (len(value) == self.num_dims)
self._dims = value
if self.loaded and self.num_records > 0:
self._dataframe.columns = self.dims + self.value_col_names
return
self._init_dataframe()
def dims(self, value):
if self.loaded and self.num_records > 0:
if not isinstance(value, list) or len(value) != self.num_dims:
logger.warn(
"Cannot set dims to {}, because dataframe with dims {} already contains data."
.format(value, self.dims))
if isinstance(value, int):
self._dims = ['*'] * value
self._init_dataframe()
return
if not isinstance(value, list):
raise Error(
'dims must be an int or a list. Was passed {} of type {}.'.
format(value, type(value)))
for dim in value:
if not isinstance(dim, string_types):
raise Error(
'Individual dimensions must be denoted by strings. Was passed {} as element of {}.'
.format(dim, value))
if self.num_dims > 0 and self.num_dims != len(value):
logger.warn(
"{}'s number of dimensions is changing from {} to {}.".format(
self.name, self.num_dims, len(value)))
self._dims = value
if self.loaded and self.num_records > 0:
self._dataframe.columns = self.dims + self.value_col_names
return
self._init_dataframe()
def _check_insert_setitem(self,key,value):
if not isinstance(value,GdxSymbol):
raise Error("GdxFiles only contain GdxSymbols. GdxFile was given a {}.".format(type(value)))
if not isinstance(key,int):
raise Error("When adding or replacing GdxSymbols in GdxFiles, only integer, not name indices, may be used.")
if key > len(self):
raise Error("Invalid key, {}".format(key))
return
def write(self,index=None):
if not self.loaded:
raise Error("Cannot write unloaded symbol {}.".format(repr(self.name)))
if self.data_type == GamsDataType.Set:
self._fixup_set_value()
if index is not None:
self._index = index
if self.index == 0:
# universal set
gdxcc.gdxUELRegisterRawStart(self.file.H)
gdxcc.gdxUELRegisterRaw(self.file.H,self.name)
gdxcc.gdxUELRegisterDone(self.file.H)
return
# write the data
userinfo = 0
if self.variable_type is not None:
userinfo = self.variable_type.value
elif self.equation_type is not None:
userinfo = self.equation_type.value
if not gdxcc.gdxDataWriteStrStart(self.file.H,
self.name,
self.description,
self.num_dims,
self.data_type.value,
userinfo):
raise GdxError(self.file.H,"Could not start writing data for symbol {}".format(repr(self.name)))
# set domain information
if self.num_dims > 0:
if self.index:
if not gdxcc.gdxSymbolSetDomainX(self.file.H,self.index,self.dims):
raise GdxError(self.file.H,"Could not set domain information for {}. Domains are {}".format(repr(self.name),repr(self.dims)))
else:
logger.info("Not writing domain information because symbol index is unknown.")
values = gdxcc.doubleArray(gdxcc.GMS_VAL_MAX)
# make sure index is clean -- needed for merging in convert_np_to_gdx_svs
self.dataframe = self.dataframe.reset_index(drop=True)
for row in convert_np_to_gdx_svs(self.dataframe,self.num_dims,self.file).itertuples(index=False,name=None):
dims = [str(x) for x in row[:self.num_dims]]
vals = row[self.num_dims:]
for col_name, col_ind in self.value_cols:
values[col_ind] = float(0.0)
try:
if isinstance(vals[col_ind],Number):
values[col_ind] = float(vals[col_ind])
except:
raise Error("Unable to set element {} from {}.".format(col_ind,vals))
gdxcc.gdxDataWriteStr(self.file.H,dims,values)
gdxcc.gdxDataWriteDone(self.file.H)
return
def write(self, filename):
# only write if all symbols loaded
for symbol in self:
if not symbol.loaded:
raise Error(
"All symbols must be loaded before this file can be written."
)
ret = gdxcc.gdxOpenWrite(self.H, filename, "gdxpds")
if not ret:
raise GdxError(
self.H,
"Could not open {} for writing. Consider cloning this file (.clone()) before trying to write"
.format(repr(filename)))
self._filename = filename
# set special values
ret = gdxcc.gdxSetSpecialValues(self.H, self.special_values)
if ret == 0:
raise GdxError(self.H, "Unable to set special values")
# write the universal set
self.universal_set.write()
for i, symbol in enumerate(self, start=1):
try:
symbol.write(index=i)
except:
logger.error("Unable to write {} to {}".format(
symbol, filename))
raise
gdxcc.gdxClose(self.H)
def data_type(self, value):
if not self.loaded or self.num_records > 0:
raise Error("Cannot change the data_type of a GdxSymbol that is yet to be read for file or contains records.")
self._data_type = GamsDataType(value)
self.variable_type = None
self.equation_type = None
self._init_dataframe()
return
def read(self, filename):
"""
Opens gdx file at filename and reads meta-data. If not self.lazy_load,
also loads all symbols.
Throws an Error if not self.empty.
Throws a GdxError if any calls to gdxcc fail.
"""
if not self.empty:
raise Error(
"GdxFile.read can only be used if the GdxFile is .empty")
# open the file
rc = gdxcc.gdxOpenRead(self.H, filename)
if not rc[0]:
raise GdxError(self.H, "Could not open '{}'".format(filename))
self._filename = filename
# read in meta-data ...
# ... for the file
ret, self._version, self._producer = gdxcc.gdxFileVersion(self.H)
if ret != 1:
raise GdxError(self.H, "Could not get file version")
ret, symbol_count, element_count = gdxcc.gdxSystemInfo(self.H)
logger.debug(
"Opening '{}' with {} symbols and {} elements with lazy_load = {}."
.format(filename, symbol_count, element_count, self.lazy_load))
# ... for the symbols
ret, name, dims, data_type = gdxcc.gdxSymbolInfo(self.H, 0)
if ret != 1:
raise GdxError(self.H,
"Could not get symbol info for the universal set")
self.universal_set = GdxSymbol(name,
data_type,
dims=dims,
file=self,
index=0)
for i in range(symbol_count):
index = i + 1
ret, name, dims, data_type = gdxcc.gdxSymbolInfo(self.H, index)
if ret != 1:
raise GdxError(
self.H,
"Could not get symbol info for symbol {}".format(index))
self.append(
GdxSymbol(name, data_type, dims=dims, file=self, index=index))
# read all symbols if not lazy_load
if not self.lazy_load:
for symbol in self:
symbol.load()
return
def clone(self):
if not self.loaded:
raise Error("Symbol {} cannot be cloned because it is not yet loaded.".format(repr(self.name)))
assert self.loaded
result = GdxSymbol(self.name,self.data_type,
dims=self.dims,
description=self.description,
variable_type=self.variable_type,
equation_type=self.equation_type)
result.dataframe = copy.deepcopy(self.dataframe)
assert result.loaded
return result
def load(self):
if self.loaded:
logger.info("Nothing to do. Symbol already loaded.")
return
if not self.file:
raise Error(
"Cannot load {} because there is no file pointer".format(
repr(self)))
if not self.index:
raise Error(
"Cannot load {} because there is no symbol index".format(
repr(self)))
if self.data_type == GamsDataType.Parameter and HAVE_GDX2PY:
self.dataframe = gdx2py.par2list(self.file.filename, self.name)
self._loaded = True
return
_ret, records = gdxcc.gdxDataReadStrStart(self.file.H, self.index)
def reader():
handle = self.file.H
for i in range(records):
yield gdxcc.gdxDataReadStr(handle)
vc = self.value_cols # do this for speed in the next line
data = [
elements + [values[col_ind] for col_name, col_ind in vc]
for ret, elements, values, afdim in reader()
]
# gdxdict called gdxGetElemText here, but I do not currently see value in doing that
self.dataframe = data
if not self.data_type == GamsDataType.Set:
self.dataframe = special.convert_gdx_to_np_svs(
self.dataframe, self.num_dims)
self._loaded = True
return
def get_value_col_default(self,value_col_name):
if not value_col_name in self.value_col_names:
raise Error("{} is not one of the value columns for this GdxSymbol, which is a {}".format(value_col_name,self.data_type))
value_col = GamsValueType(value_col_name)
if self.data_type == GamsDataType.Set:
assert value_col == GamsValueType.Level
return c_bool(True)
if (self.data_type == GamsDataType.Variable) and (
(value_col == GamsValueType.Lower) or
(value_col == GamsValueType.Upper)):
lb_default, ub_default = GAMS_VARIABLE_DEFAULT_LOWER_UPPER_BOUNDS[self.variable_type]
if value_col == GamsValueType.Lower:
return lb_default
else:
assert value_col == GamsValueType.Upper
return ub_default
return GAMS_VALUE_DEFAULTS[value_col]
def dataframe(self, data):
try:
# get data in common format and start dealing with dimensions
if isinstance(data, pds.DataFrame):
df = data.copy()
has_col_names = True
else:
df = pds.DataFrame(data)
has_col_names = False
if df.empty:
# clarify dimensionality, as needed for loading empty GdxSymbols
df = pds.DataFrame(data,
columns=self.dims + self.value_cols)
# finish handling dimensions
n = len(df.columns)
if (self.num_dims > 0) or (self.num_records > 0):
if not ((n == self.num_dims) or
(n == self.num_dims + len(self.value_cols))):
raise Error("Cannot set dataframe to {} because the number ".format(df.head()) + \
"of dimensions would change. This symbol has {} ".format(self.num_dims) + \
"dimensions, currently represented by {}.".format(self.dims))
num_dims = self.num_dims
else:
# num_dims not explicitly established yet. in this case we must
# assume value columns have been provided or dimensionality is 0
num_dims = max(n - len(self.value_cols), 0)
if (num_dims == 0) and (n < len(self.value_cols)):
raise Error("Cannot set dataframe to {} because the number ".format(df.head()) + \
"of dimensions cannot be established consistent with {}.".format(self))
if self.loaded and (num_dims > 0):
logger.warning(
'Inferring {} to have {} dimensions. '.format(
self.name, num_dims) +
'Recommended practice is to explicitly set gdxpds.gdx.GdxSymbol dims in the constructor.'
)
replace_dims = True
if has_col_names:
dim_cols = list(df.columns)[:num_dims]
elif self.num_dims == num_dims:
dim_cols = self.dims
replace_dims = False
else:
dim_cols = ['*'] * num_dims
for col in dim_cols:
if not isinstance(col, str):
replace_dims = False
logger.info(
"Not using dataframe column names to set dimensions because {} is not a string."
.format(col))
if num_dims != self.num_dims:
self.dims = num_dims
break
if replace_dims:
self.dims = dim_cols
# all done establishing dimensions
assert self.num_dims == num_dims
# finalize the dataframe
if n == self.num_dims:
self._append_default_values(df)
df.columns = self.dims + self.value_col_names
self._dataframe = df
except Exception:
logger.error(
"Unable to set dataframe for {} to\n{}\n\nIn process dataframe: {}"
.format(self, data, self._dataframe))
raise
if self.data_type == GamsDataType.Set:
self._fixup_set_value()
return
