def load(self): if self.loaded: logger.info("Nothing to do. Symbol already loaded.") return if not self.file: raise Error( "Cannot load {} because there is no file pointer".format( repr(self))) if not self.index: raise Error( "Cannot load {} because there is no symbol index".format( repr(self))) if self.data_type == GamsDataType.Parameter and HAVE_GDX2PY: self.dataframe = gdx2py.par2list(self.file.filename, self.name) self._loaded = True return data = [] ret, records = gdxcc.gdxDataReadStrStart(self.file.H, self.index) for i in range(records): ret, elements, values, afdim = gdxcc.gdxDataReadStr(self.file.H) # make sure we pick value columns up correctly data.append( elements + [values[col_ind] for col_name, col_ind in self.value_cols]) if self.data_type == GamsDataType.Set: data[-1][-1] = True # gdxdict called gdxGetElemText here, but I do not currently see value in doing that self.dataframe = data if not self.data_type == GamsDataType.Set: self.dataframe = convert_gdx_to_np_svs(self.dataframe, self.num_dims, self.file) self._loaded = True return
def dims(self, value): if (self._dims is not None) and (self.loaded and ((self.num_dims > 0) or (self.num_records > 0))): if not isinstance(value, list) or len(value) != self.num_dims: raise Error( f"Cannot set dims to {value}, because the number of " "dimensions has already been set to {self.num_dims}.") if isinstance(value, int): self._dims = ['*'] * value self._init_dataframe() return if not isinstance(value, list): raise Error( 'dims must be an int or a list. Was passed {} of type {}.'. format(value, type(value))) for dim in value: if not isinstance(dim, str): raise Error( 'Individual dimensions must be denoted by strings. Was passed {} as element of {}.' .format(dim, value)) assert (self._dims is None) or ( self.loaded and (self.num_dims == 0) and (self.num_records == 0)) or (len(value) == self.num_dims) self._dims = value if self.loaded and self.num_records > 0: self._dataframe.columns = self.dims + self.value_col_names return self._init_dataframe()
def dims(self, value): if self.loaded and self.num_records > 0: if not isinstance(value, list) or len(value) != self.num_dims: logger.warn( "Cannot set dims to {}, because dataframe with dims {} already contains data." .format(value, self.dims)) if isinstance(value, int): self._dims = ['*'] * value self._init_dataframe() return if not isinstance(value, list): raise Error( 'dims must be an int or a list. Was passed {} of type {}.'. format(value, type(value))) for dim in value: if not isinstance(dim, string_types): raise Error( 'Individual dimensions must be denoted by strings. Was passed {} as element of {}.' .format(dim, value)) if self.num_dims > 0 and self.num_dims != len(value): logger.warn( "{}'s number of dimensions is changing from {} to {}.".format( self.name, self.num_dims, len(value))) self._dims = value if self.loaded and self.num_records > 0: self._dataframe.columns = self.dims + self.value_col_names return self._init_dataframe()
def _check_insert_setitem(self,key,value): if not isinstance(value,GdxSymbol): raise Error("GdxFiles only contain GdxSymbols. GdxFile was given a {}.".format(type(value))) if not isinstance(key,int): raise Error("When adding or replacing GdxSymbols in GdxFiles, only integer, not name indices, may be used.") if key > len(self): raise Error("Invalid key, {}".format(key)) return
def write(self,index=None): if not self.loaded: raise Error("Cannot write unloaded symbol {}.".format(repr(self.name))) if self.data_type == GamsDataType.Set: self._fixup_set_value() if index is not None: self._index = index if self.index == 0: # universal set gdxcc.gdxUELRegisterRawStart(self.file.H) gdxcc.gdxUELRegisterRaw(self.file.H,self.name) gdxcc.gdxUELRegisterDone(self.file.H) return # write the data userinfo = 0 if self.variable_type is not None: userinfo = self.variable_type.value elif self.equation_type is not None: userinfo = self.equation_type.value if not gdxcc.gdxDataWriteStrStart(self.file.H, self.name, self.description, self.num_dims, self.data_type.value, userinfo): raise GdxError(self.file.H,"Could not start writing data for symbol {}".format(repr(self.name))) # set domain information if self.num_dims > 0: if self.index: if not gdxcc.gdxSymbolSetDomainX(self.file.H,self.index,self.dims): raise GdxError(self.file.H,"Could not set domain information for {}. Domains are {}".format(repr(self.name),repr(self.dims))) else: logger.info("Not writing domain information because symbol index is unknown.") values = gdxcc.doubleArray(gdxcc.GMS_VAL_MAX) # make sure index is clean -- needed for merging in convert_np_to_gdx_svs self.dataframe = self.dataframe.reset_index(drop=True) for row in convert_np_to_gdx_svs(self.dataframe,self.num_dims,self.file).itertuples(index=False,name=None): dims = [str(x) for x in row[:self.num_dims]] vals = row[self.num_dims:] for col_name, col_ind in self.value_cols: values[col_ind] = float(0.0) try: if isinstance(vals[col_ind],Number): values[col_ind] = float(vals[col_ind]) except: raise Error("Unable to set element {} from {}.".format(col_ind,vals)) gdxcc.gdxDataWriteStr(self.file.H,dims,values) gdxcc.gdxDataWriteDone(self.file.H) return
def write(self, filename): # only write if all symbols loaded for symbol in self: if not symbol.loaded: raise Error( "All symbols must be loaded before this file can be written." ) ret = gdxcc.gdxOpenWrite(self.H, filename, "gdxpds") if not ret: raise GdxError( self.H, "Could not open {} for writing. Consider cloning this file (.clone()) before trying to write" .format(repr(filename))) self._filename = filename # set special values ret = gdxcc.gdxSetSpecialValues(self.H, self.special_values) if ret == 0: raise GdxError(self.H, "Unable to set special values") # write the universal set self.universal_set.write() for i, symbol in enumerate(self, start=1): try: symbol.write(index=i) except: logger.error("Unable to write {} to {}".format( symbol, filename)) raise gdxcc.gdxClose(self.H)
def data_type(self, value): if not self.loaded or self.num_records > 0: raise Error("Cannot change the data_type of a GdxSymbol that is yet to be read for file or contains records.") self._data_type = GamsDataType(value) self.variable_type = None self.equation_type = None self._init_dataframe() return
def read(self, filename): """ Opens gdx file at filename and reads meta-data. If not self.lazy_load, also loads all symbols. Throws an Error if not self.empty. Throws a GdxError if any calls to gdxcc fail. """ if not self.empty: raise Error( "GdxFile.read can only be used if the GdxFile is .empty") # open the file rc = gdxcc.gdxOpenRead(self.H, filename) if not rc[0]: raise GdxError(self.H, "Could not open '{}'".format(filename)) self._filename = filename # read in meta-data ... # ... for the file ret, self._version, self._producer = gdxcc.gdxFileVersion(self.H) if ret != 1: raise GdxError(self.H, "Could not get file version") ret, symbol_count, element_count = gdxcc.gdxSystemInfo(self.H) logger.debug( "Opening '{}' with {} symbols and {} elements with lazy_load = {}." .format(filename, symbol_count, element_count, self.lazy_load)) # ... for the symbols ret, name, dims, data_type = gdxcc.gdxSymbolInfo(self.H, 0) if ret != 1: raise GdxError(self.H, "Could not get symbol info for the universal set") self.universal_set = GdxSymbol(name, data_type, dims=dims, file=self, index=0) for i in range(symbol_count): index = i + 1 ret, name, dims, data_type = gdxcc.gdxSymbolInfo(self.H, index) if ret != 1: raise GdxError( self.H, "Could not get symbol info for symbol {}".format(index)) self.append( GdxSymbol(name, data_type, dims=dims, file=self, index=index)) # read all symbols if not lazy_load if not self.lazy_load: for symbol in self: symbol.load() return
def clone(self): if not self.loaded: raise Error("Symbol {} cannot be cloned because it is not yet loaded.".format(repr(self.name))) assert self.loaded result = GdxSymbol(self.name,self.data_type, dims=self.dims, description=self.description, variable_type=self.variable_type, equation_type=self.equation_type) result.dataframe = copy.deepcopy(self.dataframe) assert result.loaded return result
def load(self): if self.loaded: logger.info("Nothing to do. Symbol already loaded.") return if not self.file: raise Error( "Cannot load {} because there is no file pointer".format( repr(self))) if not self.index: raise Error( "Cannot load {} because there is no symbol index".format( repr(self))) if self.data_type == GamsDataType.Parameter and HAVE_GDX2PY: self.dataframe = gdx2py.par2list(self.file.filename, self.name) self._loaded = True return _ret, records = gdxcc.gdxDataReadStrStart(self.file.H, self.index) def reader(): handle = self.file.H for i in range(records): yield gdxcc.gdxDataReadStr(handle) vc = self.value_cols # do this for speed in the next line data = [ elements + [values[col_ind] for col_name, col_ind in vc] for ret, elements, values, afdim in reader() ] # gdxdict called gdxGetElemText here, but I do not currently see value in doing that self.dataframe = data if not self.data_type == GamsDataType.Set: self.dataframe = special.convert_gdx_to_np_svs( self.dataframe, self.num_dims) self._loaded = True return
def get_value_col_default(self,value_col_name): if not value_col_name in self.value_col_names: raise Error("{} is not one of the value columns for this GdxSymbol, which is a {}".format(value_col_name,self.data_type)) value_col = GamsValueType(value_col_name) if self.data_type == GamsDataType.Set: assert value_col == GamsValueType.Level return c_bool(True) if (self.data_type == GamsDataType.Variable) and ( (value_col == GamsValueType.Lower) or (value_col == GamsValueType.Upper)): lb_default, ub_default = GAMS_VARIABLE_DEFAULT_LOWER_UPPER_BOUNDS[self.variable_type] if value_col == GamsValueType.Lower: return lb_default else: assert value_col == GamsValueType.Upper return ub_default return GAMS_VALUE_DEFAULTS[value_col]
def dataframe(self, data): try: # get data in common format and start dealing with dimensions if isinstance(data, pds.DataFrame): df = data.copy() has_col_names = True else: df = pds.DataFrame(data) has_col_names = False if df.empty: # clarify dimensionality, as needed for loading empty GdxSymbols df = pds.DataFrame(data, columns=self.dims + self.value_cols) # finish handling dimensions n = len(df.columns) if (self.num_dims > 0) or (self.num_records > 0): if not ((n == self.num_dims) or (n == self.num_dims + len(self.value_cols))): raise Error("Cannot set dataframe to {} because the number ".format(df.head()) + \ "of dimensions would change. This symbol has {} ".format(self.num_dims) + \ "dimensions, currently represented by {}.".format(self.dims)) num_dims = self.num_dims else: # num_dims not explicitly established yet. in this case we must # assume value columns have been provided or dimensionality is 0 num_dims = max(n - len(self.value_cols), 0) if (num_dims == 0) and (n < len(self.value_cols)): raise Error("Cannot set dataframe to {} because the number ".format(df.head()) + \ "of dimensions cannot be established consistent with {}.".format(self)) if self.loaded and (num_dims > 0): logger.warning( 'Inferring {} to have {} dimensions. '.format( self.name, num_dims) + 'Recommended practice is to explicitly set gdxpds.gdx.GdxSymbol dims in the constructor.' ) replace_dims = True if has_col_names: dim_cols = list(df.columns)[:num_dims] elif self.num_dims == num_dims: dim_cols = self.dims replace_dims = False else: dim_cols = ['*'] * num_dims for col in dim_cols: if not isinstance(col, str): replace_dims = False logger.info( "Not using dataframe column names to set dimensions because {} is not a string." .format(col)) if num_dims != self.num_dims: self.dims = num_dims break if replace_dims: self.dims = dim_cols # all done establishing dimensions assert self.num_dims == num_dims # finalize the dataframe if n == self.num_dims: self._append_default_values(df) df.columns = self.dims + self.value_col_names self._dataframe = df except Exception: logger.error( "Unable to set dataframe for {} to\n{}\n\nIn process dataframe: {}" .format(self, data, self._dataframe)) raise if self.data_type == GamsDataType.Set: self._fixup_set_value() return