FILES = [
('bau', 'result_urban_exo.gdx'),
('3', 'result_cint_n_3.gdx'),
('4', 'result_cint_n_4.gdx'),
('5', 'result_cint_n_5.gdx'),
('bau_lo', 'result_urban_exo_lessGDP.gdx'),
('3_lo', 'result_cint_n_3_lessGDP.gdx'),
('4_lo', 'result_cint_n_4_lessGDP.gdx'),
('5_lo', 'result_cint_n_5_lessGDP.gdx'),
]
raw = OrderedDict()
extra = dict()
for case, fn in FILES:
raw[case] = gdx.File(join(GDX_DIR, fn))
extra[case] = gdx.File(join(GDX_DIR, fn.replace('.gdx', '_extra.gdx')))
CREM = raw['bau']
cases = pd.Index(raw.keys(), name='case')
time = pd.Index(filter(lambda t: int(t) <= 2030, CREM.set('t')))
# Cell:
# List all the parameters available in each file
#CREM.parameters()
# Temporary container for read-in data
arrays = {}
def import_data(data_path, metadata_path):
# # Construct the countries-regions mapping
# country_map(join(path, '..', 'downscale'))
# Read the metadata
with open(join(metadata_path, "variables.yaml")) as vars_f:
vars = yaml.safe_load(vars_f)
ureg = pint.UnitRegistry()
for unit in vars["_units"]:
ureg.define(unit)
# Read the raw data
try:
import gdx
if gdx.__version__ > "4":
f = gdx.open_dataset(data_path)
else:
f = gdx.File(data_path)
except ImportError:
# py-gdx not installed
return None, None
data = []
# Iterate over variables
for var in sorted(vars.keys()):
if var.startswith("_"):
continue
else:
attrs = vars[var]
log(" loading: %s" % var)
# Attributes of the variable to import
# Unit conversion
convert = tuple(map(ureg, attrs.get("convert", [0, 0])))
exponent = 1
if not convert[0].dimensionless:
log(" converting %s → %s" % convert)
if convert[0].dimensionality**-1 == convert[1].dimensionality:
exponent = -1
log(" values to be inverted")
factor = (convert[0]**exponent / convert[1]).to_base_units()
log(" values to be multiplied by %f" % factor)
# Sub-selection from a larger variable
select = attrs.get("select", {})
# Time and region dimensions
time_dim = attrs.get("time", "T")
region_dim = attrs.get("region", "R")
if isinstance(region_dim, int):
region_dim = "_%s_%d" % (var, region_dim)
# Aggregation
agg = attrs.get("agg", "sum")
# Extract the variable from the GDX file as an xr.DataArray and change
# units
da = f[var]**exponent * factor
if len(select):
# Select a subset of dimensions
log(" selecting: " +
", ".join(["%s=%s" % kv for kv in select.items()]))
da = da.sel(**select)
# Determine one or more quantities to unpack from *da*
qties = attrs["item"]["quantity"]
try:
dim = qties.pop("_dim")
quantities = [({dim: k}, v) for k, v in qties.items()]
agg = False
except TypeError:
quantities = [({}, qties)]
# Retrieve quantities
for sub, qty in quantities:
log(" %sto %s" %
(["%s=%s " % kv
for kv in sub.items()][0] if len(sub) else "", qty))
# Convert to pd.DataFrame, unstack the time dimension to columns,
# convert the region dimension to a column named 'Region'
df = (da.sel(**sub).to_dataframe()[var].unstack(
time_dim).reset_index().rename(columns={region_dim: "region"}))
if agg is not False:
# Aggregate this set of rows, e.g. take a sum
log(" aggregating using .%s()" % agg)
aggregate = getattr(df, agg)()
aggregate["region"] = "Global"
df = df.append(aggregate, ignore_index=True)
# Fill the other index columns
df["variable"], df["mode"], df["technology"], df["fuel"] = qty
df["unit"] = attrs["item"]["unit"]
data.append(df)
# Combine all data
data = pd.concat(data)
data["Model"] = "EPPA5"
data["Scenario"] = "Outlook 2015"
return data, None
# named `*foo*_extra.gdx` with the pyGDX-friendly variables `ptcarb_t(t)`,
# `pe_t(e,r,t)` and `cons_t(r,t)`.
# 3. Read the GDX files
raw = OrderedDict()
extra = dict()
for case, fn, label in CASE_INFO:
print(case, '--', label)
# Preprocess
fn = GDX_DIR / fn
run(['gams', 'pre.gms', '--file={}'.format(fn)])
# Read the GDX files
fn = fn.with_suffix('.gdx')
try:
raw[case] = gdx.File(fn)
extra[case] = gdx.File(str(fn).replace('.gdx', '_extra.gdx'))
except FileNotFoundError:
continue
# Use the BAU file to reference dimensions, sets etc.
CREM = raw['bau']
# Indices for result objects
cases = pd.Index(raw.keys(), name='case')
time = pd.Index(filter(lambda t: int(t) <= 2030, CREM.set('t')))
# Description of the scenarios being loaded
scenarios_desc = pd.Series({case[0]: case[2] for case in CASE_INFO})[cases]
# for debugging:
def gdxfile_explicit(rawgdx):
"""A gdx.File fixture, instantiated with implicit=False."""
return gdx.File(rawgdx, implicit=False)
def gdxfile(rawgdx):
"""A gdx.File fixture."""
return gdx.File(rawgdx)
def test_info2(self, rawgdx):
# Use a File where p1 is guaranteed to not have been loaded:
assert (gdx.File(rawgdx).info('p1') == 'unknown parameter p1(s), 1 '
'records: Example parameter with animal data')
def test_init(self, rawgdx):
gdx.File(rawgdx)
gdx.File(rawgdx, lazy=False)
with pytest.raises(FileNotFoundError):
gdx.File('nonexistent.gdx')