def test_append_time_domain(test_pd_df, test_df_mixed, other, time, inplace):
df_year = IamDataFrame(test_pd_df[IAMC_IDX + [2005]],
meta=test_df_mixed.meta)
df_time = IamDataFrame(test_pd_df[IAMC_IDX + [2010]].rename(
{2010: time}, axis="columns"))
# append `df_time` to `df_year`
if other == "time":
if inplace:
obs = df_year.copy()
obs.append(df_time, inplace=True)
else:
obs = df_year.append(df_time)
# assert that original object was not modified
assert df_year.year == [2005]
# append `df_year` to `df_time`
else:
if inplace:
obs = df_time.copy()
obs.append(df_year, inplace=True)
else:
obs = df_time.append(df_year)
# assert that original object was not modified
assert df_time.time == pd.Index([datetime(2010, 7, 21)])
assert_iamframe_equal(obs, test_df_mixed)
def test_append_incompatible_col_raises(test_pd_df, inplace):
# Merging objects with different data index dimensions raises an error
df = IamDataFrame(test_pd_df)
test_pd_df["foo"] = "baz"
other = IamDataFrame(test_pd_df)
with pytest.raises(ValueError, match="Incompatible timeseries data index"):
df.append(other=other, inplace=inplace)
def test_numerical_relationship(self, use_ratio):
# Calculate the values using the cruncher for a fairly detailed dataset
larger_db = IamDataFrame(self.large_db)
larger_db["model"] = larger_db["model"] + "_2"
larger_db["value"] = larger_db["value"] - 0.5
larger_db.append(IamDataFrame(self.large_db), inplace=True)
larger_db = IamDataFrame(larger_db.data)
tcruncher = self.tclass(larger_db)
res = tcruncher.derive_relationship("Emissions|CH4", ["Emissions|CO2"],
use_ratio=use_ratio)
assert callable(res)
to_find = IamDataFrame(self.small_db.copy())
crunched = res(to_find)
# Calculate the same values numerically
xs = larger_db.filter(variable="Emissions|CO2")["value"].values
ys = larger_db.filter(variable="Emissions|CH4")["value"].values
if use_ratio:
ys = ys / xs
quantile_expected = silicone.stats.rolling_window_find_quantiles(
xs, ys, [0.5])
interpolate_fn = scipy.interpolate.interp1d(
np.array(quantile_expected.index),
quantile_expected.values.squeeze(),
)
xs_to_interp = to_find.filter(variable="Emissions|CO2")["value"].values
if use_ratio:
expected = interpolate_fn(xs_to_interp) * xs_to_interp
else:
expected = interpolate_fn(xs_to_interp)
assert all(crunched["value"].values == expected)
def recursive_df(request):
data = (RECURSIVE_DF if request.param == "year" else RECURSIVE_DF.rename(
DTS_MAPPING, axis="columns"))
df = IamDataFrame(data, model="model_a", scenario="scen_a", region="World")
df2 = df.rename(scenario={"scen_a": "scen_b"})
df2._data *= 2
df.append(df2, inplace=True)
yield df
def test_append_extra_col(test_df, shuffle_cols):
base_data = test_df.data.copy()
base_data["col_1"] = "hi"
base_data["col_2"] = "bye"
base_df = IamDataFrame(base_data)
other_data = base_data[base_data["variable"] == "Primary Energy"].copy()
other_data["variable"] = "Primary Energy|Gas"
other_df = IamDataFrame(other_data)
if shuffle_cols:
c1_idx = other_df._LONG_IDX.index("col_1")
c2_idx = other_df._LONG_IDX.index("col_2")
other_df._LONG_IDX[c1_idx] = "col_2"
other_df._LONG_IDX[c2_idx] = "col_1"
res = base_df.append(other_df)
def check_meta_is(iamdf, meta_col, val):
for checker in [iamdf.timeseries().reset_index(), iamdf.data]:
meta_vals = checker[meta_col].unique()
assert len(meta_vals) == 1, meta_vals
assert meta_vals[0] == val, meta_vals
# ensure meta merged correctly
check_meta_is(res, "col_1", "hi")
check_meta_is(res, "col_2", "bye")
def test_relationship_usage(self, timecol):
# Define a regular set of values and check that we return the correct quantiles
# of it
if timecol == "year":
dates = [2010, 2020, 2030]
else:
dates = [
datetime(year=2010, month=6, day=6),
datetime(year=2020, month=6, day=6),
datetime(year=2030, month=6, day=6),
]
regular_db = pd.DataFrame(
[[_ma, _sa + str(val), "World", _eco2, _gtc, val, val, val]
for val in range(11)],
columns=_msrvu + dates,
)
regular_data = IamDataFrame(regular_db)
regular_data["variable"] = _ech4
regular_data["value"] = 1
regular_data.append(IamDataFrame(regular_db), inplace=True)
regular_data = IamDataFrame(regular_data.data)
tcruncher = self.tclass(regular_data)
follow = _eco2
formatted_dates = list(regular_data[regular_data.time_col].unique())
quant = {
formatted_dates[0]: 0.4,
formatted_dates[1]: 0.5,
formatted_dates[2]: 0.6,
}
res = tcruncher.derive_relationship(
follow,
[_ech4],
time_quantile_dict=quant,
nwindows=2,
)
to_infill = regular_data.filter(variable=_ech4)
returned = res(to_infill)
for time, quantile in quant.items():
if timecol == "year":
filtered_ans = returned.filter(year=int(time))["value"]
else:
filtered_ans = returned.filter(
time=tcruncher._convert_dt64_todt(time))["value"]
assert np.allclose(filtered_ans, 11 * (quantile - 1 / 22))
def test_aggregate_recursive(time_col):
# use the feature `recursive=True`
data = (RECURSIVE_DF if time_col == "year" else RECURSIVE_DF.rename(
DTS_MAPPING, axis="columns"))
df = IamDataFrame(data, model="model_a", scenario="scen_a", region="World")
df2 = df.rename(scenario={"scen_a": "scen_b"})
df2.data.value *= 2
df.append(df2, inplace=True)
# create object without variables to be aggregated
v = "Secondary Energy|Electricity"
agg_vars = [f"{v}{i}" for i in ["", "|Wind"]]
df_minimal = df.filter(variable=agg_vars, keep=False)
# return recursively aggregated data as new object
obs = df_minimal.aggregate(variable=v, recursive=True)
assert_iamframe_equal(obs, df.filter(variable=agg_vars))
# append to `self`
df_minimal.aggregate(variable=v, recursive=True, append=True)
assert_iamframe_equal(df_minimal, df)
def test_aggregate_recursive(time_col):
# use the feature `recursive=True`
data = RECURSIVE_DF if time_col == 'year' \
else RECURSIVE_DF.rename(DTS_MAPPING, axis='columns')
df = IamDataFrame(data, model='model_a', scenario='scen_a', region='World')
df2 = df.rename(scenario={'scen_a': 'scen_b'})
df2.data.value *= 2
df.append(df2, inplace=True)
# create object without variables to be aggregated
v = 'Secondary Energy|Electricity'
agg_vars = [f'{v}{i}' for i in ['', '|Wind']]
df_minimal = df.filter(variable=agg_vars, keep=False)
# return recursively aggregated data as new object
obs = df_minimal.aggregate(variable=v, recursive=True)
assert_iamframe_equal(obs, df.filter(variable=agg_vars))
# append to `self`
df_minimal.aggregate(variable=v, recursive=True, append=True)
assert_iamframe_equal(df_minimal, df)
def test_relationship_usage(self, test_downscale_df, add_col):
units = "new units"
tcruncher = self.tclass()
test_downscale_df = test_downscale_df.filter(year=[2010, 2015])
if add_col:
# what should happen if there's more than one value in the `add_col`?
add_col_val = "blah"
test_downscale_df[add_col] = add_col_val
test_downscale_df = IamDataFrame(test_downscale_df.data)
assert test_downscale_df.extra_cols[0] == add_col
lead = ["Emissions|HFC|C2F6"]
follow = "Emissions|HFC|C5F12"
filler = tcruncher.derive_relationship(follow,
lead,
ratio=2,
units=units)
res = filler(test_downscale_df)
exp = test_downscale_df.filter(variable=lead)
exp.data["variable"] = follow
exp.data["value"] = exp.data["value"] * 2
exp.data["unit"] = units
pd.testing.assert_frame_equal(res.timeseries(),
exp.timeseries(),
check_like=True)
# comes back on input timepoints
np.testing.assert_array_equal(
res.timeseries().columns.values.squeeze(),
test_downscale_df.timeseries().columns.values.squeeze(),
)
# Test we can append the results correctly
append_df = test_downscale_df.append(res)
assert append_df.filter(variable=follow).equals(res)
if add_col:
assert all(append_df.filter(variable=lead)[add_col] == add_col_val)
