def test_append_time_domain(test_pd_df, test_df_mixed, other, time, inplace): df_year = IamDataFrame(test_pd_df[IAMC_IDX + [2005]], meta=test_df_mixed.meta) df_time = IamDataFrame(test_pd_df[IAMC_IDX + [2010]].rename( {2010: time}, axis="columns")) # append `df_time` to `df_year` if other == "time": if inplace: obs = df_year.copy() obs.append(df_time, inplace=True) else: obs = df_year.append(df_time) # assert that original object was not modified assert df_year.year == [2005] # append `df_year` to `df_time` else: if inplace: obs = df_time.copy() obs.append(df_year, inplace=True) else: obs = df_time.append(df_year) # assert that original object was not modified assert df_time.time == pd.Index([datetime(2010, 7, 21)]) assert_iamframe_equal(obs, test_df_mixed)
def test_append_incompatible_col_raises(test_pd_df, inplace): # Merging objects with different data index dimensions raises an error df = IamDataFrame(test_pd_df) test_pd_df["foo"] = "baz" other = IamDataFrame(test_pd_df) with pytest.raises(ValueError, match="Incompatible timeseries data index"): df.append(other=other, inplace=inplace)
def test_numerical_relationship(self, use_ratio): # Calculate the values using the cruncher for a fairly detailed dataset larger_db = IamDataFrame(self.large_db) larger_db["model"] = larger_db["model"] + "_2" larger_db["value"] = larger_db["value"] - 0.5 larger_db.append(IamDataFrame(self.large_db), inplace=True) larger_db = IamDataFrame(larger_db.data) tcruncher = self.tclass(larger_db) res = tcruncher.derive_relationship("Emissions|CH4", ["Emissions|CO2"], use_ratio=use_ratio) assert callable(res) to_find = IamDataFrame(self.small_db.copy()) crunched = res(to_find) # Calculate the same values numerically xs = larger_db.filter(variable="Emissions|CO2")["value"].values ys = larger_db.filter(variable="Emissions|CH4")["value"].values if use_ratio: ys = ys / xs quantile_expected = silicone.stats.rolling_window_find_quantiles( xs, ys, [0.5]) interpolate_fn = scipy.interpolate.interp1d( np.array(quantile_expected.index), quantile_expected.values.squeeze(), ) xs_to_interp = to_find.filter(variable="Emissions|CO2")["value"].values if use_ratio: expected = interpolate_fn(xs_to_interp) * xs_to_interp else: expected = interpolate_fn(xs_to_interp) assert all(crunched["value"].values == expected)
def recursive_df(request): data = (RECURSIVE_DF if request.param == "year" else RECURSIVE_DF.rename( DTS_MAPPING, axis="columns")) df = IamDataFrame(data, model="model_a", scenario="scen_a", region="World") df2 = df.rename(scenario={"scen_a": "scen_b"}) df2._data *= 2 df.append(df2, inplace=True) yield df
def test_append_extra_col(test_df, shuffle_cols): base_data = test_df.data.copy() base_data["col_1"] = "hi" base_data["col_2"] = "bye" base_df = IamDataFrame(base_data) other_data = base_data[base_data["variable"] == "Primary Energy"].copy() other_data["variable"] = "Primary Energy|Gas" other_df = IamDataFrame(other_data) if shuffle_cols: c1_idx = other_df._LONG_IDX.index("col_1") c2_idx = other_df._LONG_IDX.index("col_2") other_df._LONG_IDX[c1_idx] = "col_2" other_df._LONG_IDX[c2_idx] = "col_1" res = base_df.append(other_df) def check_meta_is(iamdf, meta_col, val): for checker in [iamdf.timeseries().reset_index(), iamdf.data]: meta_vals = checker[meta_col].unique() assert len(meta_vals) == 1, meta_vals assert meta_vals[0] == val, meta_vals # ensure meta merged correctly check_meta_is(res, "col_1", "hi") check_meta_is(res, "col_2", "bye")
def test_relationship_usage(self, timecol): # Define a regular set of values and check that we return the correct quantiles # of it if timecol == "year": dates = [2010, 2020, 2030] else: dates = [ datetime(year=2010, month=6, day=6), datetime(year=2020, month=6, day=6), datetime(year=2030, month=6, day=6), ] regular_db = pd.DataFrame( [[_ma, _sa + str(val), "World", _eco2, _gtc, val, val, val] for val in range(11)], columns=_msrvu + dates, ) regular_data = IamDataFrame(regular_db) regular_data["variable"] = _ech4 regular_data["value"] = 1 regular_data.append(IamDataFrame(regular_db), inplace=True) regular_data = IamDataFrame(regular_data.data) tcruncher = self.tclass(regular_data) follow = _eco2 formatted_dates = list(regular_data[regular_data.time_col].unique()) quant = { formatted_dates[0]: 0.4, formatted_dates[1]: 0.5, formatted_dates[2]: 0.6, } res = tcruncher.derive_relationship( follow, [_ech4], time_quantile_dict=quant, nwindows=2, ) to_infill = regular_data.filter(variable=_ech4) returned = res(to_infill) for time, quantile in quant.items(): if timecol == "year": filtered_ans = returned.filter(year=int(time))["value"] else: filtered_ans = returned.filter( time=tcruncher._convert_dt64_todt(time))["value"] assert np.allclose(filtered_ans, 11 * (quantile - 1 / 22))
def test_aggregate_recursive(time_col): # use the feature `recursive=True` data = (RECURSIVE_DF if time_col == "year" else RECURSIVE_DF.rename( DTS_MAPPING, axis="columns")) df = IamDataFrame(data, model="model_a", scenario="scen_a", region="World") df2 = df.rename(scenario={"scen_a": "scen_b"}) df2.data.value *= 2 df.append(df2, inplace=True) # create object without variables to be aggregated v = "Secondary Energy|Electricity" agg_vars = [f"{v}{i}" for i in ["", "|Wind"]] df_minimal = df.filter(variable=agg_vars, keep=False) # return recursively aggregated data as new object obs = df_minimal.aggregate(variable=v, recursive=True) assert_iamframe_equal(obs, df.filter(variable=agg_vars)) # append to `self` df_minimal.aggregate(variable=v, recursive=True, append=True) assert_iamframe_equal(df_minimal, df)
def test_aggregate_recursive(time_col): # use the feature `recursive=True` data = RECURSIVE_DF if time_col == 'year' \ else RECURSIVE_DF.rename(DTS_MAPPING, axis='columns') df = IamDataFrame(data, model='model_a', scenario='scen_a', region='World') df2 = df.rename(scenario={'scen_a': 'scen_b'}) df2.data.value *= 2 df.append(df2, inplace=True) # create object without variables to be aggregated v = 'Secondary Energy|Electricity' agg_vars = [f'{v}{i}' for i in ['', '|Wind']] df_minimal = df.filter(variable=agg_vars, keep=False) # return recursively aggregated data as new object obs = df_minimal.aggregate(variable=v, recursive=True) assert_iamframe_equal(obs, df.filter(variable=agg_vars)) # append to `self` df_minimal.aggregate(variable=v, recursive=True, append=True) assert_iamframe_equal(df_minimal, df)
def test_relationship_usage(self, test_downscale_df, add_col): units = "new units" tcruncher = self.tclass() test_downscale_df = test_downscale_df.filter(year=[2010, 2015]) if add_col: # what should happen if there's more than one value in the `add_col`? add_col_val = "blah" test_downscale_df[add_col] = add_col_val test_downscale_df = IamDataFrame(test_downscale_df.data) assert test_downscale_df.extra_cols[0] == add_col lead = ["Emissions|HFC|C2F6"] follow = "Emissions|HFC|C5F12" filler = tcruncher.derive_relationship(follow, lead, ratio=2, units=units) res = filler(test_downscale_df) exp = test_downscale_df.filter(variable=lead) exp.data["variable"] = follow exp.data["value"] = exp.data["value"] * 2 exp.data["unit"] = units pd.testing.assert_frame_equal(res.timeseries(), exp.timeseries(), check_like=True) # comes back on input timepoints np.testing.assert_array_equal( res.timeseries().columns.values.squeeze(), test_downscale_df.timeseries().columns.values.squeeze(), ) # Test we can append the results correctly append_df = test_downscale_df.append(res) assert append_df.filter(variable=follow).equals(res) if add_col: assert all(append_df.filter(variable=lead)[add_col] == add_col_val)