def add_sensitivity_filter(self, key,
filters): # This overwrites a parent function
db = get_database()
tbl_name = self._table_name
tbl = db.get_table(tbl_name)
if 'sensitivity' in tbl.data.columns:
# check to see if we have a matching sensitivity in our scenario
if tbl_name in self._scenario._sensitivities and key in self._scenario._sensitivities[
tbl_name]:
filters['sensitivity'] = self._scenario._sensitivities[
tbl_name][key]
else:
filters['sensitivity'] = '_reference_'
return filters
def timeseries_cleanup(self,
timeseries): # This overwrites a parent function
db = get_database()
tbl_name = self._table_name
tbl = db.get_table(tbl_name)
md = tbl.metadata
# drop any columns that are all NaN
timeseries = timeseries.loc[:, ~timeseries.isnull().all()]
if 'sensitivity' in timeseries.columns: #We've filtered sensitivities in an earlier step, for EP we drop them
del timeseries['sensitivity']
index_cols = [
c for c in timeseries.columns if c not in md.df_value_col
]
timeseries = timeseries.set_index(index_cols).sort_index()
return timeseries
def csv_read_table(table_name, column_names=None, return_unique=False, return_iterable=False, **filters):
"""
Get data from a table filtering by columns.
Key word arguments give column name, column criteria pairs.
Example:
util.csv_read_table('DemandDrivers', 'name', driver='oil and gas mining VOS')
"""
db = get_database()
tbl = db.get_table(table_name)
df = tbl.data
if filters:
verify_columns(table_name, df, filters.keys())
df = filter_query(df, filters)
if column_names:
column_names = ensure_iterable(column_names)
verify_columns(table_name, df, column_names)
df = df[column_names]
if return_unique:
df.drop_duplicates(inplace=True)
rows, cols = df.shape
if rows == 0:
data = [None]
elif cols == 1:
data = df.iloc[:, 0].tolist()
else:
data = list(df.itertuples(index=False, name=None))
# pull out the first element if length is 1 and we don't want to return an iterable
if len(data) == 0 or data == [None]:
return [] if return_iterable else None
elif len(data) == 1:
return data if return_iterable else data[0]
else:
return data
def table_data(table_name, column_names=None, as_tup_iter=False):
"""
Get the DataFrame holding the data for the given table_name. The
database must have already been opened (with explicit path) so that
the call here returns the singleton instance.
:param table_name: (str) the name of a table
:param column_names: (list of str) the columns to return
:param as_tup_iter: (bool) if True, return a tuple iterator, else a DataFrame
:return: the DataFrame or tuple iterator providing the selected data
"""
db = get_database()
tbl = db.get_table(table_name)
df = tbl.data
if column_names:
verify_columns(table_name, df, column_names)
df = df[column_names]
return df.itertuples(index=False, name=None) if as_tup_iter else df
def object_att_from_table(tablename, id, primary_key='name'):
db = get_database()
tbl = db.get_table(tablename)
table_headers = [h for h in tbl.get_columns() if h != primary_key]
# table_headers = [h for h in sql_read_headers(tablename) if h != primary_key]
if not len(table_headers):
return []
attributes = csv_read_table(tablename, column_names=table_headers, **{primary_key : id})
if attributes is None:
return None
native_tuples = [(table_headers, attributes)] if len(table_headers)==1 else zip(table_headers, attributes)
named_tuples = []
for t in native_tuples:
col_name = id_to_name(id_col=t[0], id_num=t[1], return_type='tuple')
if col_name is not None:
named_tuples.append(col_name)
return native_tuples + named_tuples
def create_index_levels_new(self):
if self._child_data is None:
return None
data = self._child_data
data_cols = data.columns
elements = {col: sorted(data[col].unique()) for col in data_cols}
if not DataObject._index_level_tups:
# collect the tuples on the first call and cache them in a class variable
db = get_database()
levels_tbl = db.get_table('IndexLevels')
levels = levels_tbl.data
DataObject._index_level_tups = [(row.index_level,
row.data_column_name)
for idx, row in levels.iterrows()]
level_tups = DataObject._index_level_tups
tups = [(level, col) for level, col in level_tups
if (col in data_cols and not _isListOfNoneOrNan(elements[col]))
]
#col not in self.data_id_key and # TODO: Was this really intended to be a substring test?
# OrderedDict in necessary the way this currently works
self.column_names = OrderedDict(tups)
target_cols = self.column_names.values()
tups = [(level, elements[col]) for level, col in level_tups
if col in target_cols]
index_levels = self.index_levels = OrderedDict(tups)
self.df_index_names = [
getattr(self, level, level) for level in index_levels
]
return index_levels
def __init__(self, database_path):
# Initiate pint for unit conversions
self.ureg = pint.UnitRegistry()
self.cfg_energy_unit = cfg.getParam('calculation_energy_unit')
self.cfg_currency = cfg.getParam('currency_name')
self.cfg_currency_year = cfg.getParamAsInt('currency_year')
db = get_database(database_path)
self.currency_table = db.get_table("CurrenciesConversion").data
self.currency_table = self.currency_table.set_index(
['currency', 'year']).sort_index()
self.inflation_table = db.get_table("InflationConversion").data
self.inflation_table = self.inflation_table.set_index(
['currency', 'year']).sort_index()
for unit_def in UnitConverter._unit_defs:
unit_name = unit_def.split(' = ')[0]
if hasattr(self.ureg, unit_name):
logging.debug(
'pint already has unit {}, unit is not being redefined'.
format(unit_name))
continue
self.ureg.define(unit_def)
def _load_sensitivities(self, sensitivities):
if not isinstance(sensitivities, list):
raise ValueError("The 'Sensitivities' for a scenario should be a list of objects containing "
"the keys 'table', 'name' and 'sensitivity'.")
db = get_database()
for sensitivity_spec in sensitivities:
table = sensitivity_spec['table']
name = sensitivity_spec['name']
sensitivity = sensitivity_spec['sensitivity']
if name in self._sensitivities[table]:
raise ValueError("Scenario specifies sensitivity for {} {} more than once".format(table, name))
# Check that the sensitivity actually exists in the database before using it
md = db.table_metadata(table)
filters = {'sensitivity' : sensitivity, md.key_col : name}
data = csv_read_table(table, **filters)
if data is None or len(data) == 0:
raise ValueError("Could not find sensitivity '{}' for {} {}.".format(sensitivity, table, name))
self._sensitivities[table][name] = sensitivity