def get_df(file_name, param_name):
ws = gams.GamsWorkspace()
gdxFile = os.path.join(os.getcwd(), file_name)
gdxHandle = gdxcc.new_gdxHandle_tp()
rc = gdxcc.gdxCreate(gdxHandle, gdxcc.GMS_SSSIZE)
assert rc[0],rc[1]
assert gdxcc.gdxOpenRead(gdxHandle, gdxFile)[0]
nrUels = gdxcc.gdxUMUelInfo(gdxHandle)[1]
uelMap = []
for i in range(nrUels+1):
uelMap.append(gdxcc.gdxUMUelGet(gdxHandle, i)[1])
ret, symNr = gdxcc.gdxFindSymbol(gdxHandle, param_name)
assert ret, param_name + " parameter not found"
ret, nrRecs = gdxcc.gdxDataReadRawStart(gdxHandle, symNr)
assert ret, "Error in gdxDataReadRawStart: " + gdxcc.gdxErrorStr(gdxHandle, gdxcc.gdxGetLastError(gdxHandle))[1]
ls = []
for i in range(nrRecs):
ret = gdxcc.gdxDataReadRaw(gdxHandle)
sets = [uelMap[x] for x in ret[1]]
val = ret[2][gdxcc.GMS_VAL_LEVEL]
if val == 5e300:
val = 0
ls.append(sets+[val])
assert not gdxcc.gdxClose(gdxHandle)
assert gdxcc.gdxFree(gdxHandle)
df = pd.DataFrame(ls)
return df
def _find_symbol(self, name: str) -> int:
"""Find symbol number by name
Args:
name: Symbol name
"""
ret, symno = gdxcc.gdxFindSymbol(self._h, name)
if ret > 0:
return symno
else:
return None
def gdx_to_list(gams_dir, filename, varname='all', verbose=False):
"""
This function loads the gdx with the results of the simulation
All results are stored in an unordered list
:param gams_dir: Gams working directory
:param filename: Path to the gdx file to be read
:param varname: In case online one variable is needed, specify it name (otherwise specify 'all')
:returns: Dictionary with all the collected values (within lists)
"""
from gdxcc import gdxSymbolInfo, gdxCreateD, gdxOpenRead, GMS_SSSIZE, gdxDataReadDone, new_gdxHandle_tp, \
gdxDataReadStr, gdxFindSymbol, gdxErrorStr, gdxDataReadStrStart, gdxGetLastError
out = {}
tgdx = tm.time()
gdxHandle = new_gdxHandle_tp()
gdxCreateD(gdxHandle, gams_dir, GMS_SSSIZE)
# make sure the file path is properly formatted:
filename = filename.replace('/', os.path.sep).replace(
'\\\\', os.path.sep).replace('\\', os.path.sep)
filename = str(filename) # removing possible unicode formatting
if not os.path.isfile(filename):
logging.critical('Gdx file "' + filename + '" does not exist')
sys.exit(1)
gdxOpenRead(gdxHandle, filename)
if varname == 'all':
# go through all the symbols one by one and add their data to the dict
symNr = 0
SymbolInfo = gdxSymbolInfo(gdxHandle, 0)
while SymbolInfo[0] > 0:
ret, nrRecs = gdxDataReadStrStart(gdxHandle, symNr)
assert ret, "Error in gdx data string" + gdxErrorStr(
gdxHandle, gdxGetLastError(gdxHandle))[1]
res = []
for i in range(nrRecs):
ret, elements, values, afdim = gdxDataReadStr(gdxHandle)
res.append(elements + [values[0]])
out[SymbolInfo[1]] = res
symNr += 1
SymbolInfo = gdxSymbolInfo(gdxHandle, symNr)
else:
# find the number of the required symbol:
ret, symNr = gdxFindSymbol(gdxHandle, varname)
assert ret, "Symbol not found"
ret, nrRecs = gdxDataReadStrStart(gdxHandle, symNr)
assert ret, "Error in gdx data string" + gdxErrorStr(
gdxHandle, gdxGetLastError(gdxHandle))[1]
res = []
for i in range(nrRecs):
ret, elements, values, afdim = gdxDataReadStr(gdxHandle)
res.append(elements + [values[0]])
out[varname] = res
gdxDataReadDone(gdxHandle)
if verbose:
logging.info("Loading gdx file " + filename +
" took {}s".format(tm.time() - tgdx))
return out
def query(self, name, reshape=RESHAPE_DEFAULT, gamsdir=None, filt=None, idval=None):
if __gdxpy_mode__ == GDX_MODE_API:
gdxHandle = self.gdxHandle
ret, symNr = gdxcc.gdxFindSymbol(gdxHandle, name)
assert ret, "Symbol '%s' not found in GDX '%s'" % (name,self.internal_filename)
sinfo = self.get_sid_info(symNr)
dim = sinfo['dim']
# assert dim>0, "Symbol '%s' is a scalar, not supported" % (name)
symType = sinfo['stype']
ret, nrRecs = gdxcc.gdxDataReadStrStart(gdxHandle, symNr)
assert ret, "Error in gdxDataReadStrStart: "+gdxcc.gdxErrorStr(gdxHandle,gdxGetLastError(gdxHandle))[1]
if idval is None:
if symType == gdxcc.GMS_DT_EQU:
idval = gdxcc.GMS_VAL_MARGINAL
else:
idval = gdxcc.GMS_VAL_LEVEL
ifilt = None
vtable = []
rcounter = 0
rowtype = None
if filt != None:
if isinstance(filt,list):
filt = '^({0})$'.format('|'.join([re.escape(x) for x in filt]))
filt_regex = re.compile(filt, re.IGNORECASE)
for i in range(nrRecs):
vrow = [None]*(dim+1)
ret, elements, values, afdim = gdxcc.gdxDataReadStr(gdxHandle)
assert ret, "Error in gdxDataReadStr: "+gdxcc.gdxErrorStr(gdxHandle,gdxGetLastError(gdxHandle))[1]
if (filt != None):
match_filt = False
for e in elements:
m = filt_regex.match(e)
if m != None:
match_filt = True
break
if not match_filt:
continue
d = -1
for d in range(dim):
try:
vrow[d] = int(elements[d])
except:
vrow[d] = elements[d].lower()
vrow[d+1] = values[idval]
vtable.append(vrow)
rcounter += 1
gdxcc.gdxDataReadDone(gdxHandle)
# ifilt = 1
cols = ['s%d' % x for x in range(dim)]+['val',]
#print vtable[:5]
#print cols
df = pd.DataFrame(vtable,columns=cols)
#print df
#if ifilt != None:
# df = df.drop(df.columns[ifilt], axis=1)
#print "%d / %d records read from <%s>" % (rcounter, nrRecs, self.internal_filename)
elif __gdxpy_mode__ == GDX_MODE_SHELL:
cmdline = r'gdxdump.exe {0} symb={1} Format=csv NoHeader'.format(self.internal_filename, name)
p = subprocess.Popen(cmdline, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False)
# TODO check for return code
# p = subprocess.Popen(cmdline +' | tr "[:upper:]" "[:lower:]"', stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
strdata = p.communicate()[0] #.replace("'.","',").replace(', \r\n','\r\n').replace(" ",",")
sepchar = ','
strdata = strdata.replace('eps','1e-16')
csvfile = StringIO.StringIO(strdata)
#print strdata[:500]
df = pd.read_csv(csvfile,sep=sepchar,quotechar='"',prefix='s',header=None,error_bad_lines=False).dropna()
else:
raise Exception('Function "get_symbols_list" not available outside GDX API/SHELL mode')
#raise e, None, sys.exc_info()[2]
#print_traceback(e)
#print df.columns.values
#print df.columns.values[-1], list(df.columns.values[:-2]), df.columns.values[-2]
#print df
if symType == gdxcc.GMS_DT_SET:
reshape = RESHAPE_SERIES
df = dfreshape(df, reshape)
if symType == gdxcc.GMS_DT_SET:
df = df.index
self.data = df
return df
def has_symbol(self,name):
if __gdxpy_mode__ != GDX_MODE_API:
raise Exception('Function "get_sid_info" not available outside GDX API mode')
ret, symNr = gdxcc.gdxFindSymbol(self.gdxHandle, name)
return ret
def query(self, name, reshape=RESHAPE_DEFAULT, filt=None, idval=None, idxlower=True):
'''
Query attribute `idval` from symbol `name`, and return a data structure shaped according to `reshape`.
'''
gdx_handle = self.gdx_handle
ret, symNr = gdxcc.gdxFindSymbol(gdx_handle, name)
assert ret, "Symbol '%s' not found in GDX '%s'" % (name, self.internal_filename)
sinfo = self.get_sid_info(symNr)
dim = sinfo['dim']
symType = sinfo['stype']
ret, nrRecs = gdxcc.gdxDataReadStrStart(gdx_handle, symNr)
assert ret, get_last_error('gdxDataReadStrStart', gdx_handle)
if idval is None:
if symType == gdxcc.GMS_DT_EQU:
idval = gdxcc.GMS_VAL_MARGINAL
else:
idval = gdxcc.GMS_VAL_LEVEL
ifilt = None
vtable = [None]*(nrRecs)
rcounter = 0
rowtype = None
if filt != None:
if isinstance(filt,list):
filt = '^({0})$'.format('|'.join([re.escape(x) for x in filt]))
if isinstance(filt, str):
filt_func = re.compile(filt, re.IGNORECASE).match
else:
filt_func = filt
for i in range(nrRecs):
vrow = [None]*(dim+1)
ret, elements, values, afdim = gdxcc.gdxDataReadStr(gdx_handle)
assert ret, get_last_error('gdxDataReadStr', gdx_handle)
if (filt != None):
match_filt = False
for e in elements:
m = filt_func(e)
if m != None:
match_filt = True
break
if not match_filt:
continue
d = -1
for d in range(dim):
try:
vrow[d] = int(elements[d])
except:
vrow[d] = elements[d].lower() if idxlower else elements[d]
vrow[d+1] = values[idval]
vtable[rcounter] = vrow
rcounter += 1
gdxcc.gdxDataReadDone(gdx_handle)
cols = ['s%d' % x for x in range(dim)]+['val',]
df = pd.DataFrame(vtable[:rcounter],columns=cols)
logger.debug("%d / %d records read from <%s>" % (rcounter, nrRecs, self.internal_filename))
if symType == gdxcc.GMS_DT_SET:
reshape = RESHAPE_SERIES
df = dfreshape(df, reshape)
if symType == gdxcc.GMS_DT_SET:
df = df.index
self.data = df
return df
def has_symbol(self,name):
ret, symNr = gdxcc.gdxFindSymbol(self.gdx_handle, name)
return ret
