def get_df(file_name, param_name): ws = gams.GamsWorkspace() gdxFile = os.path.join(os.getcwd(), file_name) gdxHandle = gdxcc.new_gdxHandle_tp() rc = gdxcc.gdxCreate(gdxHandle, gdxcc.GMS_SSSIZE) assert rc[0],rc[1] assert gdxcc.gdxOpenRead(gdxHandle, gdxFile)[0] nrUels = gdxcc.gdxUMUelInfo(gdxHandle)[1] uelMap = [] for i in range(nrUels+1): uelMap.append(gdxcc.gdxUMUelGet(gdxHandle, i)[1]) ret, symNr = gdxcc.gdxFindSymbol(gdxHandle, param_name) assert ret, param_name + " parameter not found" ret, nrRecs = gdxcc.gdxDataReadRawStart(gdxHandle, symNr) assert ret, "Error in gdxDataReadRawStart: " + gdxcc.gdxErrorStr(gdxHandle, gdxcc.gdxGetLastError(gdxHandle))[1] ls = [] for i in range(nrRecs): ret = gdxcc.gdxDataReadRaw(gdxHandle) sets = [uelMap[x] for x in ret[1]] val = ret[2][gdxcc.GMS_VAL_LEVEL] if val == 5e300: val = 0 ls.append(sets+[val]) assert not gdxcc.gdxClose(gdxHandle) assert gdxcc.gdxFree(gdxHandle) df = pd.DataFrame(ls) return df
def _find_symbol(self, name: str) -> int: """Find symbol number by name Args: name: Symbol name """ ret, symno = gdxcc.gdxFindSymbol(self._h, name) if ret > 0: return symno else: return None
def gdx_to_list(gams_dir, filename, varname='all', verbose=False): """ This function loads the gdx with the results of the simulation All results are stored in an unordered list :param gams_dir: Gams working directory :param filename: Path to the gdx file to be read :param varname: In case online one variable is needed, specify it name (otherwise specify 'all') :returns: Dictionary with all the collected values (within lists) """ from gdxcc import gdxSymbolInfo, gdxCreateD, gdxOpenRead, GMS_SSSIZE, gdxDataReadDone, new_gdxHandle_tp, \ gdxDataReadStr, gdxFindSymbol, gdxErrorStr, gdxDataReadStrStart, gdxGetLastError out = {} tgdx = tm.time() gdxHandle = new_gdxHandle_tp() gdxCreateD(gdxHandle, gams_dir, GMS_SSSIZE) # make sure the file path is properly formatted: filename = filename.replace('/', os.path.sep).replace( '\\\\', os.path.sep).replace('\\', os.path.sep) filename = str(filename) # removing possible unicode formatting if not os.path.isfile(filename): logging.critical('Gdx file "' + filename + '" does not exist') sys.exit(1) gdxOpenRead(gdxHandle, filename) if varname == 'all': # go through all the symbols one by one and add their data to the dict symNr = 0 SymbolInfo = gdxSymbolInfo(gdxHandle, 0) while SymbolInfo[0] > 0: ret, nrRecs = gdxDataReadStrStart(gdxHandle, symNr) assert ret, "Error in gdx data string" + gdxErrorStr( gdxHandle, gdxGetLastError(gdxHandle))[1] res = [] for i in range(nrRecs): ret, elements, values, afdim = gdxDataReadStr(gdxHandle) res.append(elements + [values[0]]) out[SymbolInfo[1]] = res symNr += 1 SymbolInfo = gdxSymbolInfo(gdxHandle, symNr) else: # find the number of the required symbol: ret, symNr = gdxFindSymbol(gdxHandle, varname) assert ret, "Symbol not found" ret, nrRecs = gdxDataReadStrStart(gdxHandle, symNr) assert ret, "Error in gdx data string" + gdxErrorStr( gdxHandle, gdxGetLastError(gdxHandle))[1] res = [] for i in range(nrRecs): ret, elements, values, afdim = gdxDataReadStr(gdxHandle) res.append(elements + [values[0]]) out[varname] = res gdxDataReadDone(gdxHandle) if verbose: logging.info("Loading gdx file " + filename + " took {}s".format(tm.time() - tgdx)) return out
def query(self, name, reshape=RESHAPE_DEFAULT, gamsdir=None, filt=None, idval=None): if __gdxpy_mode__ == GDX_MODE_API: gdxHandle = self.gdxHandle ret, symNr = gdxcc.gdxFindSymbol(gdxHandle, name) assert ret, "Symbol '%s' not found in GDX '%s'" % (name,self.internal_filename) sinfo = self.get_sid_info(symNr) dim = sinfo['dim'] # assert dim>0, "Symbol '%s' is a scalar, not supported" % (name) symType = sinfo['stype'] ret, nrRecs = gdxcc.gdxDataReadStrStart(gdxHandle, symNr) assert ret, "Error in gdxDataReadStrStart: "+gdxcc.gdxErrorStr(gdxHandle,gdxGetLastError(gdxHandle))[1] if idval is None: if symType == gdxcc.GMS_DT_EQU: idval = gdxcc.GMS_VAL_MARGINAL else: idval = gdxcc.GMS_VAL_LEVEL ifilt = None vtable = [] rcounter = 0 rowtype = None if filt != None: if isinstance(filt,list): filt = '^({0})$'.format('|'.join([re.escape(x) for x in filt])) filt_regex = re.compile(filt, re.IGNORECASE) for i in range(nrRecs): vrow = [None]*(dim+1) ret, elements, values, afdim = gdxcc.gdxDataReadStr(gdxHandle) assert ret, "Error in gdxDataReadStr: "+gdxcc.gdxErrorStr(gdxHandle,gdxGetLastError(gdxHandle))[1] if (filt != None): match_filt = False for e in elements: m = filt_regex.match(e) if m != None: match_filt = True break if not match_filt: continue d = -1 for d in range(dim): try: vrow[d] = int(elements[d]) except: vrow[d] = elements[d].lower() vrow[d+1] = values[idval] vtable.append(vrow) rcounter += 1 gdxcc.gdxDataReadDone(gdxHandle) # ifilt = 1 cols = ['s%d' % x for x in range(dim)]+['val',] #print vtable[:5] #print cols df = pd.DataFrame(vtable,columns=cols) #print df #if ifilt != None: # df = df.drop(df.columns[ifilt], axis=1) #print "%d / %d records read from <%s>" % (rcounter, nrRecs, self.internal_filename) elif __gdxpy_mode__ == GDX_MODE_SHELL: cmdline = r'gdxdump.exe {0} symb={1} Format=csv NoHeader'.format(self.internal_filename, name) p = subprocess.Popen(cmdline, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False) # TODO check for return code # p = subprocess.Popen(cmdline +' | tr "[:upper:]" "[:lower:]"', stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) strdata = p.communicate()[0] #.replace("'.","',").replace(', \r\n','\r\n').replace(" ",",") sepchar = ',' strdata = strdata.replace('eps','1e-16') csvfile = StringIO.StringIO(strdata) #print strdata[:500] df = pd.read_csv(csvfile,sep=sepchar,quotechar='"',prefix='s',header=None,error_bad_lines=False).dropna() else: raise Exception('Function "get_symbols_list" not available outside GDX API/SHELL mode') #raise e, None, sys.exc_info()[2] #print_traceback(e) #print df.columns.values #print df.columns.values[-1], list(df.columns.values[:-2]), df.columns.values[-2] #print df if symType == gdxcc.GMS_DT_SET: reshape = RESHAPE_SERIES df = dfreshape(df, reshape) if symType == gdxcc.GMS_DT_SET: df = df.index self.data = df return df
def has_symbol(self,name): if __gdxpy_mode__ != GDX_MODE_API: raise Exception('Function "get_sid_info" not available outside GDX API mode') ret, symNr = gdxcc.gdxFindSymbol(self.gdxHandle, name) return ret
def query(self, name, reshape=RESHAPE_DEFAULT, filt=None, idval=None, idxlower=True): ''' Query attribute `idval` from symbol `name`, and return a data structure shaped according to `reshape`. ''' gdx_handle = self.gdx_handle ret, symNr = gdxcc.gdxFindSymbol(gdx_handle, name) assert ret, "Symbol '%s' not found in GDX '%s'" % (name, self.internal_filename) sinfo = self.get_sid_info(symNr) dim = sinfo['dim'] symType = sinfo['stype'] ret, nrRecs = gdxcc.gdxDataReadStrStart(gdx_handle, symNr) assert ret, get_last_error('gdxDataReadStrStart', gdx_handle) if idval is None: if symType == gdxcc.GMS_DT_EQU: idval = gdxcc.GMS_VAL_MARGINAL else: idval = gdxcc.GMS_VAL_LEVEL ifilt = None vtable = [None]*(nrRecs) rcounter = 0 rowtype = None if filt != None: if isinstance(filt,list): filt = '^({0})$'.format('|'.join([re.escape(x) for x in filt])) if isinstance(filt, str): filt_func = re.compile(filt, re.IGNORECASE).match else: filt_func = filt for i in range(nrRecs): vrow = [None]*(dim+1) ret, elements, values, afdim = gdxcc.gdxDataReadStr(gdx_handle) assert ret, get_last_error('gdxDataReadStr', gdx_handle) if (filt != None): match_filt = False for e in elements: m = filt_func(e) if m != None: match_filt = True break if not match_filt: continue d = -1 for d in range(dim): try: vrow[d] = int(elements[d]) except: vrow[d] = elements[d].lower() if idxlower else elements[d] vrow[d+1] = values[idval] vtable[rcounter] = vrow rcounter += 1 gdxcc.gdxDataReadDone(gdx_handle) cols = ['s%d' % x for x in range(dim)]+['val',] df = pd.DataFrame(vtable[:rcounter],columns=cols) logger.debug("%d / %d records read from <%s>" % (rcounter, nrRecs, self.internal_filename)) if symType == gdxcc.GMS_DT_SET: reshape = RESHAPE_SERIES df = dfreshape(df, reshape) if symType == gdxcc.GMS_DT_SET: df = df.index self.data = df return df
def has_symbol(self,name): ret, symNr = gdxcc.gdxFindSymbol(self.gdx_handle, name) return ret