def test_interpolate_full_example():
cols = ['model_a', 'scen_a', 'World']
df = IamDataFrame(
pd.DataFrame(
[
cols + ['all', 'EJ/yr', 0, 1, 6., 10],
cols + ['last', 'EJ/yr', 0, 0.5, 3, np.nan],
cols + ['first', 'EJ/yr', 0, np.nan, 2, 7],
cols + ['middle', 'EJ/yr', 0, 1, np.nan, 7],
cols + ['first two', 'EJ/yr', 0, np.nan, np.nan, 7],
cols + ['last two', 'EJ/yr', 0, 1, np.nan, np.nan],
],
columns=IAMC_IDX + [2000, 2005, 2010, 2017],
))
exp = IamDataFrame(
pd.DataFrame(
[
cols + ['all', 'EJ/yr', 0, 1, 6., 7.142857, 10],
cols + ['last', 'EJ/yr', 0, 0.5, 3, np.nan, np.nan],
cols + ['first', 'EJ/yr', 0, 1., 2, 3.428571, 7],
cols + ['middle', 'EJ/yr', 0, 1, np.nan, 4.5, 7],
cols +
['first two', 'EJ/yr', 0, 2.058824, np.nan, 4.941176, 7],
cols + ['last two', 'EJ/yr', 0, 1, np.nan, np.nan, np.nan],
],
columns=IAMC_IDX + [2000, 2005, 2010, 2012, 2017],
))
obs = df.interpolate([2005, 2012], inplace=False)
assert_iamframe_equal(obs, exp)
def test_unfccc_tier1():
# test that UNFCCC API returns expected data and units
exp = IamDataFrame(
UNFCCC_DF,
**INDEX_ARGS,
region="DEU",
variable="Emissions|CH4|Agriculture",
unit="kt CH4",
)
obs = read_unfccc(party_code="DEU", gases=["CH4"], tier=1)
# assert that the data is similar
horizon = [1990, 1991, 1992]
assert_iamframe_equal(obs.filter(year=horizon, variable="*Agri*"), exp)
# assert that variables are similar
types = [
"Agriculture",
"Energy",
"Industrial Processes and Product Use",
"Land Use, Land-Use Change and Forestry",
"Waste",
]
assert obs.variable == [f"Emissions|CH4|{i}" for i in types]
# assert that the unit is merged as expected
assert obs.unit == ["kt CH4"]
def test_divide_variable(test_df_year, arg, df_func, fillna, append,
ignore_units):
"""Verify that in-dataframe addition works on the default `variable` axis"""
# note that dividing with pint reformats the unit
if ignore_units:
# change one unit to make ignore-units strictly necessary
test_df_year.rename(
variable={"Primary Energy": "Primary Energy"},
unit={"EJ/yr": "custom_unit"},
inplace=True,
)
unit = ignore_units
else:
unit = "EJ / a" if isinstance(arg, int) else ""
exp = df_func(operator.truediv, "Ratio", unit=unit, meta=test_df_year.meta)
args = ("Primary Energy", arg, "Ratio")
kwds = dict(ignore_units=ignore_units, fillna=fillna)
if append:
obs = test_df_year.copy()
obs.divide(*args, **kwds, append=True)
assert_iamframe_equal(test_df_year.append(exp), obs)
else:
# check that incompatible units raise the expected error
if ignore_units:
with pytest.raises(pint.UndefinedUnitError):
test_df_year.add(*args, fillna=fillna, ignore_units=False)
assert_iamframe_equal(exp, test_df_year.divide(*args, **kwds))
def test_interpolate_extra_cols():
# check that interpolation with non-matching extra_cols has no effect
# (#351)
EXTRA_COL_DF = pd.DataFrame(
[
["foo", 2005, 1],
["foo", 2010, 2],
["bar", 2005, 2],
["bar", 2010, 3],
],
columns=["extra_col", "year", "value"],
)
df = IamDataFrame(
EXTRA_COL_DF,
model="model_a",
scenario="scen_a",
region="World",
variable="Primary Energy",
unit="EJ/yr",
)
# create a copy, interpolate
df2 = df.copy()
df2.interpolate(2007)
# interpolate should work as if extra_cols is in the _data index
assert_iamframe_equal(df, df2.filter(year=2007, keep=False))
obs = df2.filter(year=2007)["value"]
exp = pd.Series([2.4, 1.4], name="value")
pd.testing.assert_series_equal(obs, exp)
def test_interpolate_full_example():
cols = ["model_a", "scen_a", "World"]
df = IamDataFrame(
pd.DataFrame(
[
cols + ["all", "EJ/yr", 0, 1, 6.0, 10],
cols + ["last", "EJ/yr", 0, 0.5, 3, np.nan],
cols + ["first", "EJ/yr", 0, np.nan, 2, 7],
cols + ["middle", "EJ/yr", 0, 1, np.nan, 7],
cols + ["first two", "EJ/yr", 0, np.nan, np.nan, 7],
cols + ["last two", "EJ/yr", 0, 1, np.nan, np.nan],
],
columns=IAMC_IDX + [2000, 2005, 2010, 2017],
))
exp = IamDataFrame(
pd.DataFrame(
[
cols + ["all", "EJ/yr", 0, 1, 6.0, 7.142857, 10],
cols + ["last", "EJ/yr", 0, 0.5, 3, np.nan, np.nan],
cols + ["first", "EJ/yr", 0, 1.0, 2, 3.428571, 7],
cols + ["middle", "EJ/yr", 0, 1, np.nan, 4.5, 7],
cols +
["first two", "EJ/yr", 0, 2.058824, np.nan, 4.941176, 7],
cols + ["last two", "EJ/yr", 0, 1, np.nan, np.nan, np.nan],
],
columns=IAMC_IDX + [2000, 2005, 2010, 2012, 2017],
))
obs = df.interpolate([2005, 2012], inplace=False)
assert_iamframe_equal(obs, exp)
def test_add_variable(test_df_year, arg, df_func, fillna, ignore_units,
append):
"""Verify that in-dataframe addition works on the default `variable` axis"""
# change one unit to make ignore-units strictly necessary
if ignore_units:
test_df_year.rename(
variable={"Primary Energy": "Primary Energy"},
unit={"EJ/yr": "custom_unit"},
inplace=True,
)
unit = "EJ/yr" if ignore_units is False else ignore_units
exp = df_func(operator.add, "Sum", unit=unit, meta=test_df_year.meta)
args = ("Primary Energy", arg, "Sum")
kwds = dict(ignore_units=ignore_units, fillna=fillna)
if append:
obs = test_df_year.copy()
obs.add(*args, **kwds, append=True)
assert_iamframe_equal(test_df_year.append(exp), obs)
else:
# check that incompatible units raise the expected error
if ignore_units:
with pytest.raises(pint.UndefinedUnitError):
test_df_year.add(*args, fillna=fillna, ignore_units=False)
assert_iamframe_equal(exp, test_df_year.add(*args, **kwds))
def test_diff(test_df_year, periods, year, append):
"""Test `diff` method including non-default periods argument"""
exp = IamDataFrame(
pd.DataFrame(
[
["model_a", "scen_a", "World", "foo", "EJ/yr", 5],
["model_a", "scen_a", "World", "bar", "EJ/yr", 2.5],
["model_a", "scen_b", "World", "foo", "EJ/yr", 5],
],
columns=IAMC_IDX + [year],
),
meta=test_df_year.meta,
)
# values are negative if computing diff in a negative direction
if year == 2005:
exp._data = -exp._data
mapping = {"Primary Energy": "foo", "Primary Energy|Coal": "bar"}
if append:
obs = test_df_year.copy()
obs.diff(mapping=mapping, append=True, **periods)
assert_iamframe_equal(test_df_year.append(exp), obs)
else:
obs = test_df_year.diff(mapping=mapping, **periods)
assert_iamframe_equal(exp, obs)
def test_apply_variable(test_df_year, append):
"""Verify that in-dataframe apply works on the default `variable` axis"""
def custom_func(a, b, c, d):
return a * b + c * d
v = "new variable"
exp = IamDataFrame(
pd.DataFrame([custom_func(1, 2, 0.5, 3),
custom_func(6, 2, 3, 3)],
index=[2005, 2010]).T,
**DF_ARGS,
scenario="scen_a",
variable=v,
unit="EJ / a", # applying operations with pint reformats the unit
meta=test_df_year.meta,
)
args = ["Primary Energy", 2]
kwds = dict(d=3, c="Primary Energy|Coal")
if append:
obs = test_df_year.copy()
obs.apply(custom_func,
name="new variable",
append=True,
args=args,
**kwds)
assert_iamframe_equal(test_df_year.append(exp), obs)
else:
obs = test_df_year.apply(custom_func, name=v, args=args, **kwds)
assert_iamframe_equal(exp, obs)
def test_interpolate_extra_cols():
# check that interpolation with non-matching extra_cols has no effect
# (#351)
EXTRA_COL_DF = pd.DataFrame(
[
['foo', 2005, 1],
['foo', 2010, 2],
['bar', 2005, 2],
['bar', 2010, 3],
],
columns=['extra_col', 'year', 'value'],
)
df = IamDataFrame(EXTRA_COL_DF,
model='model_a',
scenario='scen_a',
region='World',
variable='Primary Energy',
unit='EJ/yr')
# create a copy, interpolate
df2 = df.copy()
df2.interpolate(2007)
# interpolate should work as if extra_cols is in the _data index
assert_iamframe_equal(df, df2.filter(year=2007, keep=False))
obs = df2.filter(year=2007)['value']
exp = pd.Series([2.4, 1.4], name='value')
pd.testing.assert_series_equal(obs, exp)
def test_equal_meta_nan_col(test_df_year):
"""Test that a meta-column with only np.nan is seen as equal"""
# https://github.com/IAMconsortium/pyam/issues/515
df = test_df_year.copy()
df.set_meta(meta=np.nan,
name="nan-column") # add a column of np.nan's to `meta`
assert_iamframe_equal(test_df_year, df)
def test_worldbank():
try:
obs = read_worldbank(model="foo",
indicator={"NY.GDP.PCAP.PP.KD": "GDP"})
exp = IamDataFrame(WB_DF)
# test data with 5% relative tolerance to guard against minor data changes
assert_iamframe_equal(obs, exp, rtol=5.0e-2)
except ReadTimeout:
logger.error("Timeout when reading from WorldBank API!")
def test_io_datapackage(test_df, tmpdir):
# add column to `meta` and write to datapackage
file = Path(tmpdir) / "foo.zip"
test_df.set_meta(["a", "b"], "string")
test_df.to_datapackage(file)
# read from csv assert that IamDataFrame instances are equal
import_df = read_datapackage(file)
assert_iamframe_equal(test_df, import_df)
def test_aggregate_region(simple_df, variable):
# check that `variable` is a a direct sum across regions
exp = simple_df.filter(variable=variable, region="World")
assert_iamframe_equal(simple_df.aggregate_region(variable), exp)
# check custom `region` (will include `World`, so double-count values)
foo = exp.rename(region={"World": "foo"})
foo._data = foo._data * 2
assert_iamframe_equal(simple_df.aggregate_region(variable, region="foo"), foo)
def test_learning_rate(append):
"""Check computing the learning rate"""
if append:
obs = TEST_DF.copy()
obs.compute.learning_rate("Learning Rate", "Cost", "Cap", append=True)
assert_iamframe_equal(TEST_DF.append(EXP_DF), obs)
else:
obs = TEST_DF.compute.learning_rate("Learning Rate", "Cost", "Cap")
assert_iamframe_equal(EXP_DF, obs)
def test_learning_rate_empty(append):
"""Assert that computing the learning rate with invalid variables returns empty"""
if append:
obs = TEST_DF.copy()
obs.compute.learning_rate("Learning Rate", "foo", "Cap", append=True)
assert_iamframe_equal(TEST_DF, obs) # assert that no data was added
else:
obs = TEST_DF.compute.learning_rate("Learning Rate", "foo", "Cap")
assert obs.empty
def test_load_meta_empty_rows(test_df_year, tmpdir):
"""Loading empty meta table (columns but no rows) from xlsx file"""
exp = test_df_year.copy() # loading empty file has no effect
# write empty meta frame to file, then load to the IamDataFrame
file = tmpdir / "testing_meta_empty.xlsx"
pd.DataFrame(columns=META_IDX).to_excel(file, index=False)
test_df_year.load_meta(file)
assert_iamframe_equal(test_df_year, exp)
def test_aggregate_region_with_other_method(simple_df, variable, data):
# use other method (max) both as string and passing the function
_df = data.copy()
if simple_df.time_col == 'time':
_df.year = _df.year.replace(DTS_MAPPING)
_df.rename({'year': 'time'}, axis='columns', inplace=True)
exp = IamDataFrame(_df).filter(region='World')
for m in ['max', np.max]:
assert_iamframe_equal(simple_df.aggregate_region(variable, method=m),
exp)
def test_growth_rate_empty(test_df_year, append):
"""Assert that computing the growth rate with invalid variables returns empty"""
if append:
obs = test_df_year.copy()
obs.compute.growth_rate({"foo": "bar"}, append=True)
assert_iamframe_equal(test_df_year,
obs) # assert that no data was added
else:
obs = test_df_year.compute.growth_rate({"foo": "bar"})
assert obs.empty
def test_aggregate_region_with_other_method(simple_df, variable, data):
# use other method (max) both as string and passing the function
_df = data.copy()
if simple_df.time_col == "time":
_df.year = _df.year.replace(DTS_MAPPING)
_df.rename({"year": "time"}, axis="columns", inplace=True)
exp = IamDataFrame(_df, meta=simple_df.meta).filter(region="World")
for m in ["max", np.max]:
obs = simple_df.aggregate_region(variable, method=m)
assert_iamframe_equal(obs, exp)
def test_growth_rate(test_df_year, append):
"""Check computing the growth rate from an IamDataFrame"""
if append:
obs = test_df_year.copy()
obs.compute.growth_rate({"Primary Energy": "Growth Rate"}, append=True)
assert_iamframe_equal(test_df_year.append(EXP_DF), obs)
else:
obs = test_df_year.compute.growth_rate(
{"Primary Energy": "Growth Rate"})
assert_iamframe_equal(EXP_DF, obs)
def test_query_with_meta_false(conn, test_pd_df, kwargs):
# test reading timeseries data (including subannual data)
exp = IamDataFrame(test_pd_df, subannual='Year')\
.append(MODEL_B_DF, model='model_b', scenario='scen_a', region='World')
# test method via Connection
df = conn.query(meta=False, **kwargs)
assert_iamframe_equal(df, exp.filter(**kwargs))
# test top-level method
df = read_iiasa(TEST_API, meta=False, **kwargs)
assert_iamframe_equal(df, exp.filter(**kwargs))
def test_diff_empty(test_df_year, append):
"""Assert that `diff` with only one time period returns empty"""
df = test_df_year.filter(year=2005)
mapping = {"Primary Energy": "foo", "Primary Energy|Coal": "bar"}
if append:
obs = df.copy()
obs.diff(mapping=mapping, append=True)
assert_iamframe_equal(df, obs) # assert that no data was added
else:
obs = df.diff(mapping=mapping)
assert obs.empty
def test_io_xlsx(test_df, meta_args, tmpdir):
# write to xlsx (direct file name and ExcelWriter, see #300)
file = tmpdir / "testing_io_write_read.xlsx"
for f in [file, pd.ExcelWriter(file)]:
test_df.to_excel(f, **meta_args[0])
if isinstance(f, pd.ExcelWriter):
f.close()
# read from xlsx
import_df = IamDataFrame(file, **meta_args[1])
# assert that IamDataFrame instances are equal
assert_iamframe_equal(test_df, import_df)
def test_query_year(conn, test_df_year, kwargs):
# test reading timeseries data (`model_a` has only yearly data)
exp = test_df_year.copy()
for i in ['version'] + META_COLS:
exp.set_meta(META_DF.iloc[[0, 1]][i])
# test method via Connection
df = conn.query(model='model_a', **kwargs)
assert_iamframe_equal(df, exp.filter(**kwargs))
# test top-level method
df = read_iiasa(TEST_API, model='model_a', **kwargs)
assert_iamframe_equal(df, exp.filter(**kwargs))
def test_downscale_region_with_proxy(simple_df, variable):
simple_df.set_meta([1], name='test')
regions = ['reg_a', 'reg_b']
# return as new IamDataFrame
obs = simple_df.downscale_region(variable, proxy='Population')
exp = simple_df.filter(variable=variable, region=regions)
assert_iamframe_equal(exp, obs)
# append to `self` (after removing to-be-downscaled timeseries)
inplace = simple_df.filter(variable=variable, region=regions, keep=False)
inplace.downscale_region(variable, proxy='Population', append=True)
assert_iamframe_equal(inplace, simple_df)
def test_aggregate(simple_df, variable, data):
# check that `variable` is a a direct sum and matches given total
exp = simple_df.filter(variable=variable)
assert_iamframe_equal(simple_df.aggregate(variable), exp)
# use other method (max) both as string and passing the function
_df = data.copy()
if simple_df.time_col == "time":
_df.year = _df.year.replace(DTS_MAPPING)
_df.rename({"year": "time"}, axis="columns", inplace=True)
exp = IamDataFrame(_df, meta=simple_df.meta)
for m in ["max", np.max]:
assert_iamframe_equal(simple_df.aggregate(variable, method=m), exp)
def test_query_with_meta_arg(conn, test_pd_df, kwargs):
# test reading timeseries data (including subannual data)
exp = IamDataFrame(test_pd_df, subannual='Year')\
.append(MODEL_B_DF, model='model_b', scenario='scen_a', region='World')
for i in ['version', 'string']:
exp.set_meta(META_DF.iloc[[0, 1, 3]][i])
# test method via Connection
df = conn.query(meta=['string'], **kwargs)
assert_iamframe_equal(df, exp.filter(**kwargs))
# test top-level method
df = read_iiasa(TEST_API, meta=['string'], **kwargs)
assert_iamframe_equal(df, exp.filter(**kwargs))
def test_io_xlsx_multiple_data_sheets(test_df, sheets, sheetname, tmpdir):
# write data to separate sheets in excel file
file = tmpdir / "testing_io_write_read.xlsx"
xl = pd.ExcelWriter(file)
for i, (model, scenario) in enumerate(test_df.index):
test_df.filter(scenario=scenario).to_excel(xl, sheet_name=sheets[i])
test_df.export_meta(xl)
xl.close()
# read from xlsx
import_df = IamDataFrame(file, **sheetname)
# assert that IamDataFrame instances are equal
assert_iamframe_equal(test_df, import_df)
def test_aggregate_recursive(recursive_df):
# use the feature `recursive=True`
# create object without variables to be aggregated
v = "Secondary Energy|Electricity"
agg_vars = [f"{v}{i}" for i in ["", "|Wind"]]
df_minimal = recursive_df.filter(variable=agg_vars, keep=False)
# return recursively aggregated data as new object
obs = df_minimal.aggregate(variable=v, recursive=True)
assert_iamframe_equal(obs, recursive_df.filter(variable=agg_vars))
# append to `self`
df_minimal.aggregate(variable=v, recursive=True, append=True)
assert_iamframe_equal(df_minimal, recursive_df)
def test_io_datapackage(test_df):
file = 'foo.zip'
# add column to `meta`
test_df.set_meta(['a', 'b'], 'string')
# write to datapackage
test_df.to_datapackage(file)
# read from csv
import_df = read_datapackage(file)
# assert that IamDataFrame instances are equal and delete file
assert_iamframe_equal(test_df, import_df)
os.remove(file)