def __init__(self, gams_dir=None, lazy_load=True):
"""
Initializes a GdxFile object by connecting to GAMS and creating a pointer.
Throws a GdxError if either of those operations fail.
"""
self.lazy_load = lazy_load
self._version = None
self._producer = None
self._filename = None
self._symbols = OrderedDict()
NeedsGamsDir.__init__(self, gams_dir=gams_dir)
self._H = self._create_gdx_object()
self.universal_set = GdxSymbol('*',
GamsDataType.Set,
dims=1,
file=None,
index=0)
self.universal_set._file = self
# get special values
self.special_values = gdxcc.doubleArray(gdxcc.GMS_SVIDX_MAX)
gdxcc.gdxGetSpecialValues(self.H, self.special_values)
self.gdx_to_np_svs = {}
self.np_to_gdx_svs = {}
for i in range(gdxcc.GMS_SVIDX_MAX):
if i >= len(NUMPY_SPECIAL_VALUES):
break
gdx_val = self.special_values[i]
self.gdx_to_np_svs[gdx_val] = NUMPY_SPECIAL_VALUES[i]
self.np_to_gdx_svs[i] = gdx_val
atexit.register(self.cleanup)
return
def load_specials(gams_dir_finder):
"""
Load special values
Needs to be called after gdxcc is loaded. Populates the module attributes
SPECIAL_VALUES, GDX_TO_NP_SVS, and NP_TO_GDX_SVS.
Parameters
----------
gams_dir_finder : :class:`gdxpds.tools.GamsDirFinder`
"""
global SPECIAL_VALUES
global GDX_TO_NP_SVS
global NP_TO_GDX_SVS
H = gdxcc.new_gdxHandle_tp()
rc = gdxcc.gdxCreateD(H, gams_dir_finder.gams_dir, gdxcc.GMS_SSSIZE)
if not rc:
raise Exception(rc[1])
# get special values
special_values = gdxcc.doubleArray(gdxcc.GMS_SVIDX_MAX)
gdxcc.gdxGetSpecialValues(H, special_values)
SPECIAL_VALUES = []
GDX_TO_NP_SVS = {}
NP_TO_GDX_SVS = {}
for i in range(gdxcc.GMS_SVIDX_MAX):
if i >= len(NUMPY_SPECIAL_VALUES):
break
SPECIAL_VALUES.append(special_values[i])
gdx_val = special_values[i]
GDX_TO_NP_SVS[gdx_val] = NUMPY_SPECIAL_VALUES[i]
NP_TO_GDX_SVS[NUMPY_SPECIAL_VALUES[i]] = gdx_val
gdxcc.gdxFree(H)
def write(self,index=None):
if not self.loaded:
raise Error("Cannot write unloaded symbol {}.".format(repr(self.name)))
if self.data_type == GamsDataType.Set:
self._fixup_set_value()
if index is not None:
self._index = index
if self.index == 0:
# universal set
gdxcc.gdxUELRegisterRawStart(self.file.H)
gdxcc.gdxUELRegisterRaw(self.file.H,self.name)
gdxcc.gdxUELRegisterDone(self.file.H)
return
# write the data
userinfo = 0
if self.variable_type is not None:
userinfo = self.variable_type.value
elif self.equation_type is not None:
userinfo = self.equation_type.value
if not gdxcc.gdxDataWriteStrStart(self.file.H,
self.name,
self.description,
self.num_dims,
self.data_type.value,
userinfo):
raise GdxError(self.file.H,"Could not start writing data for symbol {}".format(repr(self.name)))
# set domain information
if self.num_dims > 0:
if self.index:
if not gdxcc.gdxSymbolSetDomainX(self.file.H,self.index,self.dims):
raise GdxError(self.file.H,"Could not set domain information for {}. Domains are {}".format(repr(self.name),repr(self.dims)))
else:
logger.info("Not writing domain information because symbol index is unknown.")
values = gdxcc.doubleArray(gdxcc.GMS_VAL_MAX)
# make sure index is clean -- needed for merging in convert_np_to_gdx_svs
self.dataframe = self.dataframe.reset_index(drop=True)
for row in convert_np_to_gdx_svs(self.dataframe,self.num_dims,self.file).itertuples(index=False,name=None):
dims = [str(x) for x in row[:self.num_dims]]
vals = row[self.num_dims:]
for col_name, col_ind in self.value_cols:
values[col_ind] = float(0.0)
try:
if isinstance(vals[col_ind],Number):
values[col_ind] = float(vals[col_ind])
except:
raise Error("Unable to set element {} from {}.".format(col_ind,vals))
gdxcc.gdxDataWriteStr(self.file.H,dims,values)
gdxcc.gdxDataWriteDone(self.file.H)
return
def _insert_symbols(gdxHandle, sets, parameters):
"""
Function that writes all sets and parameters to the gdxHandle
:param sets: dictionary with all the sets
:param parameters: dictionary with all the parameters
"""
# It is essential to write the sets first, otherwise h might be written in the wrong order
for s in sets:
gdxSymbolType = gdxcc.GMS_DT_SET
dims = 1
gdxcc.gdxDataWriteStrStart(gdxHandle, s, "", dims, gdxSymbolType, 0)
gdxValues = gdxcc.doubleArray(5)
gdxValues[
gdxcc.
GMS_VAL_LEVEL] = 0.0 # 0.0 == Y (explanatory text of set in gdx)
Nrows = len(sets[s])
for row in range(Nrows):
gdxKeys = [str(ss) for ss in shrink_to_64([sets[s][row]])
] # Reduce the size if bigger than 64 characters
try:
success = gdxcc.gdxDataWriteStr(gdxHandle, gdxKeys, gdxValues)
except:
success = False
if not success:
logging.error('Key ' + gdxKeys[0] + ' of set ' + s +
' could not be written')
gdxcc.gdxDataWriteDone(gdxHandle)
# Check array sizes for parameters:
for p in parameters:
variable = parameters[p]
# Check that the required fields are present:
dims = len(variable['sets'])
shape = variable['val'].shape
Nrows = variable['val'].shape[0]
gdxSymbolType = gdxcc.GMS_DT_PAR
gdxcc.gdxDataWriteStrStart(gdxHandle, p, "", dims, gdxSymbolType, 0)
gdxValues = gdxcc.doubleArray(5)
gdxValues[
gdxcc.
GMS_VAL_LEVEL] = 0.0 # 0.0 == Y (explanatory text of set in gdx)
if len(shape) != dims:
logging.error('Variable ' + p + ': The \'val\' data matrix has ' +
str(len(shape)) + ' dimensions and should have ' +
str(dims))
sys.exit(1)
for i in range(dims):
if shape[i] != len(sets[variable['sets'][i]]):
logging.error('Variable ' + p +
': The \'val\' data matrix has ' +
str(shape[i]) + ' elements for dimention ' +
str(variable['sets'][i]) + ' while there are ' +
str(len(variable['sets'])) + ' set values')
sys.exit(1)
for index, value in np.ndenumerate(variable['val']):
# Write line by line if value is non null
if value != 0 and not pd.isnull(value):
gdxKeys = [] # All the set values for this line
for i in range(dims):
key = sets[variable['sets'][i]][index[
i]] # Get the string value of the set by using the indice in the val matrix
gdxKeys.append(str(key))
gdxKeys = shrink_to_64(
gdxKeys) # Reduce the size if bigger than 64 characters
gdxValues[gdxcc.GMS_VAL_LEVEL] = float(value)
try:
success = gdxcc.gdxDataWriteStr(gdxHandle, gdxKeys,
gdxValues)
except:
logging.error("Didn't work")
success = False
if not success:
logging.error('Key ' + gdxKeys[0] + ' of parameter ' + p +
' could not be written')
gdxcc.gdxDataWriteDone(gdxHandle)
logging.debug('Parameter ' + p + ' successfully written')
logging.debug('Set ' + s + ' successfully written')
def test_roundtrip_just_special_values(manage_rundir):
outdir = os.path.join(run_dir,'special_values')
if not os.path.exists(outdir):
os.mkdir(outdir)
# create gdx file containing all special values
with gdxpds.gdx.GdxFile() as f:
df = pds.DataFrame([['sv' + str(i+1), f.special_values[i]] for i in range(gdxcc.GMS_SVIDX_MAX-2)],
columns=['sv','Value'])
logger.info("Special values are:\n{}".format(df))
# save this directly as a GdxSymbol
filename = os.path.join(outdir,'direct_write_special_values.gdx')
ret = gdxcc.gdxOpenWrite(f.H,filename,"gdxpds")
if not ret:
raise gdxpds.gdx.GdxError(f.H,"Could not open {} for writing. Consider cloning this file (.clone()) before trying to write".format(repr(filename)))
# set special values
ret = gdxcc.gdxSetSpecialValues(f.H,f.special_values)
if ret == 0:
raise gdxpds.gdx.GdxError(f.H,"Unable to set special values")
# write the universal set
f.universal_set.write()
if not gdxcc.gdxDataWriteStrStart(f.H,
'special_values',
'',
1,
gdxpds.gdx.GamsDataType.Parameter.value,
0):
raise gdxpds.gdx.GdxError(f.H,"Could not start writing data for symbol special_values")
# set domain information
if not gdxcc.gdxSymbolSetDomainX(f.H,1,[df.columns[0]]):
raise gdxpds.gdx.GdxError(f.H,"Could not set domain information for special_values.")
values = gdxcc.doubleArray(gdxcc.GMS_VAL_MAX)
for row in df.itertuples(index=False,name=None):
dims = [str(x) for x in row[:1]]
vals = row[1:]
for col_name, col_ind in gdxpds.gdx.GAMS_VALUE_COLS_MAP[gdxpds.gdx.GamsDataType.Parameter]:
values[col_ind] = float(vals[col_ind])
gdxcc.gdxDataWriteStr(f.H,dims,values)
gdxcc.gdxDataWriteDone(f.H)
gdxcc.gdxClose(f.H)
# general test for expected values
def check_special_values(gdx_file):
df = gdx_file['special_values'].dataframe
for i, val in enumerate(df['Value'].values):
assert gdxpds.gdx.gdx_val_equal(gdx_file.np_to_gdx_svs[i],gdx_file.special_values[i],gdx_file)
# now roundtrip it gdx-only
with gdxpds.gdx.GdxFile(lazy_load=False) as f:
f.read(filename)
check_special_values(f)
with f.clone() as g:
rt_filename = os.path.join(outdir,'roundtripped.gdx')
g.write(rt_filename)
with gdxpds.gdx.GdxFile(lazy_load=False) as g:
g.read(filename)
check_special_values(g)
# now roundtrip it through csv
roundtripped_gdx = roundtrip_one_gdx(filename,'roundtrip_just_special_values')
with gdxpds.gdx.GdxFile(lazy_load=False) as h:
h.read(roundtripped_gdx)
check_special_values(h)
# typename is 'Variable' or 'Equation'
limits = OrderedDict()
for i, level_name in enumerate(level_names):
limits[level_name] = values[i]
d.setinfo(name)["limits"] = limits
elif typename[0:2] == "Se":
# typename is 'Set'
ret, description, node = gdxcc.gdxGetElemText(
H, int(values[gdxcc.GMS_VAL_LEVEL]))
if ret != 0:
d.setinfo(name)["description"] = description
#- Writing Tools ---------------------------------------------------------------
values = gdxcc.doubleArray(gdxcc.GMS_VAL_MAX)
def set_symbol(H, d, name, typename, userinfo, values, dims):
if typename == "Set":
text_index = 0
if "description" in d.getinfo(name):
ret, text_index = gdxcc.gdxAddSetText(
H,
d.getinfo(name)["description"])
values[gdxcc.GMS_VAL_LEVEL] = float(text_index)
else:
values[gdxcc.GMS_VAL_LEVEL] = d[name]
if (typename == "Variable"
or typename == "Equation") and "limits" in d.getinfo(name):
if success_gdxcc and success_path:
print('\n \nTRY TO GENERATE GDX FILE')
try:
gdxHandle = gdxcc.new_gdxHandle_tp()
gdxcc.gdxCreateD(gdxHandle, gamspath, gdxcc.GMS_SSSIZE)
gdxcc.gdxOpenWrite(gdxHandle, 'test.gdx', "")
# Write a set:
dims = 1
setname = 'set_test'
keys = ['aa']
gdxcc.gdxDataWriteStrStart(gdxHandle, setname, "", dims, gdxcc.GMS_DT_SET, 0)
gdxValues = gdxcc.doubleArray(5)
gdxValues[gdxcc.GMS_VAL_LEVEL] = 0.0 # 0.0 == Y (explanatory text of set in gdx)
try:
success = gdxcc.gdxDataWriteStr(gdxHandle, keys, gdxValues)
except Exception as e:
success = False
print('ERROR: the set could not be written to the gdx file. Error msg: ' + str(e))
gdxcc.gdxDataWriteDone(gdxHandle)
gdxcc.gdxClose(gdxHandle)
except Exception as ee:
print('ERROR: the gdxfile could not be created. Error msg: ' + str(ee))
success = False
if success and os.path.isfile('test.gdx'):
print('GDX successfully written. Cleaning up')
os.remove('test.gdx')
#- Reading tools ---------------------------------------------------------------
def read_symbol(H, d, name, typename, values):
d[name] = True if typename == "Set" else values[gdxcc.GMS_VAL_LEVEL]
if typename[0] == "V" or typename[0] == "E":
# typename is 'Variable' or 'Equation'
limits = OrderedDict()
for i, level_name in enumerate(level_names):
limits[level_name] = values[i]
d.setinfo(name)["limits"] = limits
elif typename[0:2] == "Se":
# typename is 'Set'
ret, description, node = gdxcc.gdxGetElemText(H, int(values[gdxcc.GMS_VAL_LEVEL]))
if ret != 0:
d.setinfo(name)["description"] = description
#- Writing Tools ---------------------------------------------------------------
values = gdxcc.doubleArray(gdxcc.GMS_VAL_MAX)
def set_symbol(H, d, name, typename, userinfo, values, dims):
if typename == "Set":
text_index = 0
if "description" in d.getinfo(name):
ret, text_index = gdxcc.gdxAddSetText(H, d.getinfo(name)["description"])
values[gdxcc.GMS_VAL_LEVEL] = float(text_index)
def _write_symbol(self, symname: str, symbol: _GAMSSymbol):
"""Write a Pandas series to a GAMS Set symbol
Args:
symbol: GAMS symbol object
Raises:
RuntimeError
"""
# Get number of dimensions
dim = symbol.dimension
try:
recs = len(symbol)
except TypeError:
recs = 1
set_has_text = False # TODO
# Begin writing to a symbol
ret = gdxcc.gdxDataWriteStrStart(
self._h,
symname,
symbol.expl_text,
dim,
GMS_DTYPES[symbol._type],
GMS_USERINFO_SET_PARAMETER,
)
if not ret:
raise RuntimeError(
gdxcc.gdxErrorStr(self._h, gdxcc.gdxGetLastError(self._h))[1])
# Define domain
if symbol.domain is not None:
domain = [(d if d is not None else '*') for d in symbol.domain]
else:
domain = dim * ['*']
ret = gdxcc.gdxSymbolSetDomainX(self._h, self._find_symbol(symname),
domain)
if not ret:
raise RuntimeError(
"Unable to set domain for symbol '{}'".format(symname))
# Init value array
value_arr = gdxcc.doubleArray(gdxcc.GMS_VAL_MAX)
if dim == 0: # It’s a scalar
value_arr[GMS_VAL_LEVEL] = float(symbol)
gdxcc.gdxDataWriteStr(self._h, list(), value_arr)
elif isinstance(symbol, GAMSSet):
for key in symbol:
# For sets, register the associated text in the string table and
# store index number as the value
if set_has_text:
pass # TODO
# assoc_text = copy(values[i])
# ret, value_arr[GMS_VAL_LEVEL] = gdxAddSetText(self._h,
# str(assoc_text))
# if not ret:
# warn("Unable to register string '{}' "
# "to string table".format(assoc_text))
else:
value_arr[GMS_VAL_LEVEL] = 0
gdxcc.gdxDataWriteStr(self._h, list(key), value_arr)
elif isinstance(symbol, GAMSParameter):
for key, value in symbol:
value_arr[GMS_VAL_LEVEL] = value
gdxcc.gdxDataWriteStr(self._h, list(key), value_arr)
gdxcc.gdxDataWriteDone(self._h)
