def data():
"""Generate data for tests"""
df = gen_sliced_df()
df = df[["x", "z_categ", "y", "residual"]]
new_df = df.iloc[[
1, 100, 150, 200, 250, 300, 305, 400, 405, 500, 550, 609
]].copy()
return {"df": df, "new_df": new_df}
def test_normal_quantiles_df():
"""Testing calculating normal quantiles for a sliced dataframe."""
df = gen_sliced_df()
model_dict = estimate_empirical_distribution(df=df,
value_col="y",
quantile_grid_size=None,
quantiles=[0.025, 0.975],
conditional_cols=["x"])
ecdf_df = model_dict["ecdf_df"]
# check the case with regular mean
quantiles_df = normal_quantiles_df(df=ecdf_df,
mean_col="y_mean",
std_col="y_std",
quantiles=(0.025, 0.975))
quantiles_df = quantiles_df[["x", "normal_quantiles"]].copy()
quantiles_df["normal_quantiles"] = quantiles_df["normal_quantiles"].apply(
lambda x: tuple(e.round(2) for e in x))
expected_df = pd.DataFrame()
expected_df["x"] = ["a", "b", "c", "d", "e"]
expected_df["normal_quantiles"] = [(89.94, 102.01), (190.2, 201.9),
(290.25, 301.91), (479.37, 561.93),
(-14.15, 6.43)]
assert expected_df.equals(
quantiles_df), "quantiles are not calculated correctly"
# check the case with fixed_mean
quantiles_df = normal_quantiles_df(df=ecdf_df,
mean_col=None,
std_col="y_std",
fixed_mean=0,
quantiles=(0.025, 0.975))
quantiles_df = quantiles_df[["x", "normal_quantiles"]].copy()
quantiles_df["normal_quantiles"] = quantiles_df["normal_quantiles"].apply(
lambda x: tuple(e.round(2) for e in x))
expected_df = pd.DataFrame()
expected_df["x"] = ["a", "b", "c", "d", "e"]
expected_df["normal_quantiles"] = [(-6.04, 6.04), (-5.85, 5.85),
(-5.83, 5.83), (-41.28, 41.28),
(-10.29, 10.29)]
assert expected_df.equals(
quantiles_df), "quantiles are not calculated correctly"
def test_plot_flexible_grouping_evaluation():
"""Tests plot_flexible_grouping_evaluation function"""
df = gen_sliced_df(sample_size_dict={"a": 300, "b": 200, "c": 300, "d": 80, "e": 300})
actual_col = "y"
predicted_col = "y_hat"
groupby_col = "x"
groupby_col2 = "z"
df = df[[actual_col, predicted_col, groupby_col, groupby_col2]]
df[cst.TIME_COL] = pd.date_range(start="2020-01-01", periods=df.shape[0], freq="D")
end_index = math.floor(df.shape[0] * 0.8)
forecast = UnivariateForecast(
df,
train_end_date=df[cst.TIME_COL][end_index],
time_col=cst.TIME_COL,
actual_col=actual_col,
predicted_col=predicted_col,
predicted_lower_col=None,
predicted_upper_col=None,
null_model_predicted_col=None)
# MSE and quantiles of squared error
metric_col = "squared_err"
map_func_dict = {metric_col: ElementwiseEvaluationMetricEnum.SquaredError.name}
agg_kwargs = {f"Q{quantile}": pd.NamedAgg(column=metric_col, aggfunc=partial(np.nanquantile, q=quantile)) for quantile in [0.1, 0.9]}
agg_kwargs.update({"mean": pd.NamedAgg(column=metric_col, aggfunc=np.nanmean)})
# group by "dom", "auto-fill" styling
fig = forecast.plot_flexible_grouping_evaluation(
which="train",
groupby_time_feature="dom",
groupby_sliding_window_size=None,
groupby_custom_column=None,
map_func_dict=map_func_dict,
agg_kwargs=agg_kwargs,
extend_col_names=False,
y_col_style_dict="auto-fill",
default_color="rgba(0, 145, 202, 1.0)",
xlabel=None,
ylabel=metric_col,
title=None,
showlegend=True)
assert [fig.data[i].name for i in range(len(fig.data))] == ["Q0.1", "mean", "Q0.9"]
assert fig.layout.xaxis.title.text == "dom"
assert fig.layout.yaxis.title.text == metric_col
assert fig.layout.title.text == f"{metric_col} vs dom"
assert fig.data[0].x.shape[0] == 31 # 31 unique days in month
assert fig.data[1].line["color"] == "rgba(0, 145, 202, 1.0)"
assert fig.data[1].fill == "tonexty" # from auto-fill
assert fig.layout.showlegend
# group by sliding window, "auto" styling
# provide default color, xlabel, hide legend
fig = forecast.plot_flexible_grouping_evaluation(
which="train",
groupby_time_feature=None,
groupby_sliding_window_size=7,
groupby_custom_column=None,
map_func_dict=map_func_dict,
agg_kwargs=agg_kwargs,
extend_col_names=False,
y_col_style_dict="auto",
default_color="rgba(145, 0, 202, 1.0)",
xlabel="ts",
ylabel=None,
title=None,
showlegend=False)
assert [fig.data[i].name for i in range(len(fig.data))] == ["Q0.1", "mean", "Q0.9"]
assert fig.layout.xaxis.title.text == "ts"
assert fig.layout.yaxis.title.text is None
assert fig.layout.title.text is None
assert fig.data[0].x[0] == datetime.datetime(2020, 1, 1, 0, 0)
assert fig.data[1].line["color"] == "rgba(145, 0, 202, 1.0)"
assert fig.data[1].fill is None
assert not fig.layout.showlegend
# custom groups, "plotly" styling, provide ylabel, title
fig = forecast.plot_flexible_grouping_evaluation(
which="train",
groupby_time_feature=None,
groupby_sliding_window_size=None,
groupby_custom_column=forecast.df_train["x"],
map_func_dict=map_func_dict,
agg_kwargs=agg_kwargs,
extend_col_names=False,
y_col_style_dict="plotly",
default_color=None,
xlabel=None,
ylabel=metric_col,
title="custom title",
showlegend=True)
assert [fig.data[i].name for i in range(len(fig.data))] == ["Q0.1", "Q0.9", "mean"] # not sorted
assert fig.layout.xaxis.title.text == "x"
assert fig.layout.yaxis.title.text == metric_col
assert fig.layout.title.text == "custom title"
assert list(fig.data[0].x) == list("abcde")
assert fig.data[0].line["color"] is None # color is up to plotly
assert fig.data[1].fill is None
assert fig.layout.showlegend
# test set, absolute percent error, custom `y_col_style_dict` styling
metric_col = "squared_error"
map_func_dict = {
metric_col: ElementwiseEvaluationMetricEnum.AbsolutePercentError.name
}
agg_kwargs = {
"median": pd.NamedAgg(column=metric_col, aggfunc=np.nanmedian),
"mean": pd.NamedAgg(column=metric_col, aggfunc=np.nanmean),
}
y_col_style_dict = {
"median": {
"mode": "lines+markers",
"line": {
"color": "rgba(202, 145, 0, 0.5)"
}
},
"mean": {
"mode": "lines+markers",
"line": {
"color": "rgba(0, 145, 202, 1.0)"
}
},
}
with pytest.warns(UserWarning, match="true_val is less than 1e-8"):
fig = forecast.plot_flexible_grouping_evaluation(
which="test",
groupby_time_feature="dow",
groupby_sliding_window_size=None,
groupby_custom_column=None,
map_func_dict=map_func_dict,
agg_kwargs=agg_kwargs,
extend_col_names=False,
y_col_style_dict=y_col_style_dict,
xlabel="x value",
ylabel="y value",
title="error plot",
showlegend=True)
assert [fig.data[i].name for i in range(len(fig.data))] == ["median", "mean"] # not sorted
assert fig.layout.xaxis.title.text == "x value"
assert fig.layout.yaxis.title.text == "y value"
assert fig.layout.title.text == "error plot"
assert len(fig.data[0].x) == 7
assert fig.data[0].mode == "lines+markers"
assert fig.data[1].mode == "lines+markers"
assert fig.data[0].line["color"] == y_col_style_dict["median"]["line"]["color"]
assert fig.data[1].line["color"] == y_col_style_dict["mean"]["line"]["color"]
assert fig.data[1].fill is None
assert fig.layout.showlegend
# median actual vs forecast value by group
agg_kwargs = {
"y_median": pd.NamedAgg(column="y", aggfunc=np.nanmedian),
"y_hat_median": pd.NamedAgg(column="y_hat", aggfunc=np.nanmedian),
}
fig = forecast.plot_flexible_grouping_evaluation(
which="train",
groupby_time_feature="dow",
groupby_sliding_window_size=None,
groupby_custom_column=None,
map_func_dict=None,
agg_kwargs=agg_kwargs,
extend_col_names=True,
y_col_style_dict="plotly",
xlabel=None,
ylabel=forecast.ylabel,
title="true vs actual by dow",
showlegend=True)
assert [fig.data[i].name for i in range(len(fig.data))] == ["y_median", "y_hat_median"]
assert fig.layout.xaxis.title.text == "dow"
assert fig.layout.yaxis.title.text == "y"
assert fig.layout.title.text == "true vs actual by dow"
assert len(fig.data[0].x) == 7
assert fig.layout.showlegend
def test_fit_ml_model_with_evaluation_with_uncertainty():
"""Tests fit_ml_model_with_evaluation with uncertainty intervals"""
df = gen_sliced_df(sample_size_dict={
"a": 200,
"b": 340,
"c": 300,
"d": 8,
"e": 800
},
seed_dict={
"a": 301,
"b": 167,
"c": 593,
"d": 893,
"e": 191,
"z": 397
},
err_magnitude_coef=8.0)
df = df[["x", "z_categ", "y_hat"]]
df.rename(columns={"y_hat": "y"}, inplace=True)
model_formula_str = "y~x+z_categ"
y_col = "y"
# test_df
fut_df = df.copy()
# we change the name of the column of true values in fut_df
# to be able to keep track of true values later
fut_df.rename(columns={"y": "y_true"}, inplace=True)
y_test = fut_df["y_true"]
# create a small dataframe for testing values only
small_sample_index = [1, 500, 750, 1000]
trained_model = fit_ml_model_with_evaluation(
df=df,
model_formula_str=model_formula_str,
uncertainty_dict={
"uncertainty_method": "simple_conditional_residuals",
"params": {
"quantiles": [0.025, 0.975],
"quantile_estimation_method": "normal_fit",
"sample_size_thresh": 10,
"small_sample_size_method": "std_quantiles",
"small_sample_size_quantile": 0.8
}
})
y_test_pred = predict_ml(fut_df=fut_df, trained_model=trained_model)[y_col]
y_test_pred_small = y_test_pred[small_sample_index]
# testing predictions
err = calc_pred_err(y_test, y_test_pred)
enum = EvaluationMetricEnum.MeanAbsoluteError
assert err[enum.get_metric_name()] < 10.0
enum = EvaluationMetricEnum.RootMeanSquaredError
assert err[enum.get_metric_name()] < 10.0
enum = EvaluationMetricEnum.Correlation
assert err[enum.get_metric_name()] > 0.5
# testing actual values for a smaller set
assert list(
y_test_pred_small.round(1)) == [99.7, 201.5, 303.5,
7.3], ("predictions are not correct")
# testing uncertainty
# assign the predicted y to the response in fut_df
fut_df["y"] = y_test_pred
new_df_with_uncertainty = predict_ci(fut_df,
trained_model["uncertainty_model"])
assert list(new_df_with_uncertainty.columns) == [
"y_quantile_summary", ERR_STD_COL
], ("column names are not as expected")
fut_df["y_quantile_summary"] = new_df_with_uncertainty[
"y_quantile_summary"]
# calculate coverage of the CI
fut_df["inside_95_ci"] = fut_df.apply(lambda row: (
(row["y_true"] <= row["y_quantile_summary"][1]) and
(row["y_true"] >= row["y_quantile_summary"][0])),
axis=1)
ci_coverage = 100.0 * fut_df["inside_95_ci"].mean()
assert ci_coverage > 94.0 and ci_coverage < 96.0, (
"95 percent CI coverage is not between 94 and 96")
# testing uncertainty_method not being implemented but passed
with pytest.raises(
Exception,
match="uncertainty method: non_existing_method is not implemented"
):
fit_ml_model_with_evaluation(df=df,
model_formula_str=model_formula_str,
uncertainty_dict={
"uncertainty_method":
"non_existing_method",
"params": {
"quantiles": [0.025, 0.975],
"quantile_estimation_method":
"normal_fit",
"sample_size_thresh": 10,
"small_sample_size_method":
"std_quantiles",
"small_sample_size_quantile": 0.8
}
})