def test_forecast_pipeline_rolling_evaluation_error():
"""Checks errors of forecast_pipeline_rolling_evaluation"""
data = generate_df_for_tests(freq="D", periods=30)
df = data["df"]
tscv = RollingTimeSeriesSplit(forecast_horizon=7,
periods_between_train_test=7,
max_splits=1)
# Different forecast_horizon in pipeline_params and tscv
with pytest.raises(ValueError,
match="Forecast horizon in 'pipeline_params' "
"does not match that of the 'tscv'."):
pipeline_params = mock_pipeline(df=df, forecast_horizon=15)
forecast_pipeline_rolling_evaluation(pipeline_params=pipeline_params,
tscv=tscv)
with pytest.raises(
ValueError,
match="'periods_between_train_test' in 'pipeline_params' "
"does not match that of the 'tscv'."):
pipeline_params = mock_pipeline(df=df,
forecast_horizon=7,
periods_between_train_test=2)
forecast_pipeline_rolling_evaluation(pipeline_params=pipeline_params,
tscv=tscv)
def test_forecast_pipeline_rolling_evaluation_prophet():
"""Checks the output rolling evaluation with Prophet template"""
data = generate_df_with_reg_for_tests(freq="D",
periods=30,
remove_extra_cols=True,
mask_test_actuals=True)
reg_cols = ["regressor1", "regressor2", "regressor3"]
keep_cols = [TIME_COL, VALUE_COL] + reg_cols
df = data["df"][keep_cols]
hyperparameter_grid = {
"estimator__weekly_seasonality": [True],
"estimator__daily_seasonality": [True, False],
"estimator__n_changepoints":
[0], # to speed up test case, remove for better fit
"estimator__uncertainty_samples": [10], # to speed up test case
"estimator__add_regressor_dict": [{
"regressor1": {
"prior_scale": 10,
"standardize": True,
"mode": 'additive'
},
"regressor2": {
"prior_scale": 15,
"standardize": False,
"mode": 'additive'
},
"regressor3": {}
}]
}
pipeline_params = mock_pipeline(
df=df,
forecast_horizon=3,
regressor_cols=["regressor1", "regressor2", "regressor3"],
estimator=ProphetEstimator(),
hyperparameter_grid=hyperparameter_grid)
tscv = RollingTimeSeriesSplit(forecast_horizon=3,
expanding_window=True,
max_splits=1)
rolling_evaluation = forecast_pipeline_rolling_evaluation(
pipeline_params=pipeline_params, tscv=tscv)
expected_splits_n = tscv.max_splits
assert len(rolling_evaluation.keys()) == expected_splits_n
assert set(rolling_evaluation.keys()) == {"split_0"}
split0_output = rolling_evaluation["split_0"]
assert round(split0_output["runtime_sec"],
3) == split0_output["runtime_sec"]
pipeline_result = split0_output["pipeline_result"]
# Calculates expected pipeline
train, test = list(tscv.split(X=df))[0]
df_train = df.loc[train]
pipeline_params_updated = pipeline_params
pipeline_params_updated["test_horizon"] = 0
pipeline_params_updated["df"] = df_train
expected_pipeline_result = forecast_pipeline(**pipeline_params_updated)
assert pipeline_result.backtest is None
# Checks output is identical when there is only 1 split
pipeline_grid_search = summarize_grid_search_results(
pipeline_result.grid_search,
score_func=EvaluationMetricEnum.MeanAbsolutePercentError.name)
expected_grid_search = summarize_grid_search_results(
expected_pipeline_result.grid_search,
score_func=EvaluationMetricEnum.MeanAbsolutePercentError.name)
assert_equal(pipeline_grid_search["mean_test_MAPE"],
expected_grid_search["mean_test_MAPE"])
assert_equal(pipeline_result.grid_search.cv.__dict__,
expected_pipeline_result.grid_search.cv.__dict__)
# Checks forecast df has the correct number of rows
expected_rows = pipeline_result.timeseries.fit_df.shape[
0] + tscv.forecast_horizon
assert pipeline_result.forecast.df.shape[0] == expected_rows
def test_forecast_pipeline_rolling_evaluation_silverkite():
"""Checks the output rolling evaluation with Silverkite template"""
data = generate_df_with_reg_for_tests(
freq="1D",
periods=20 * 7, # short-term: 20 weeks of data
remove_extra_cols=True,
mask_test_actuals=True)
regressor_cols = ["regressor1", "regressor2", "regressor_categ"]
keep_cols = [TIME_COL, VALUE_COL] + regressor_cols
df = data["df"][keep_cols]
coverage = 0.1
hyperparameter_grid = {
"estimator__origin_for_time_vars":
[None], # inferred from training data
"estimator__fs_components_df": [
pd.DataFrame({
"name": ["tow"],
"period": [7.0],
"order": [3],
"seas_names": ["weekly"]
})
],
"estimator__extra_pred_cols":
[regressor_cols, regressor_cols + ["ct_sqrt"]
], # two cases: no growth term and single growth term
"estimator__fit_algorithm_dict": [{
"fit_algorithm": "linear"
}]
}
pipeline_params = mock_pipeline(
df=df,
time_col=TIME_COL,
value_col=VALUE_COL,
date_format=None, # not recommended, but possible to specify
freq=None,
regressor_cols=regressor_cols,
estimator=SilverkiteEstimator(),
hyperparameter_grid=hyperparameter_grid,
hyperparameter_budget=1,
n_jobs=1,
forecast_horizon=2 * 7,
coverage=coverage,
test_horizon=2 * 7,
periods_between_train_test=2 * 7,
agg_periods=7,
agg_func=np.mean,
score_func=mean_absolute_error, # callable score_func
null_model_params=None,
cv_horizon=1 * 7,
cv_expanding_window=True,
cv_min_train_periods=8 * 7,
cv_periods_between_splits=7,
cv_periods_between_train_test=3 * 7,
cv_max_splits=2)
tscv = RollingTimeSeriesSplit(forecast_horizon=2 * 7,
min_train_periods=10 * 7,
expanding_window=True,
use_most_recent_splits=True,
periods_between_splits=2 * 7,
periods_between_train_test=2 * 7,
max_splits=3)
rolling_evaluation = forecast_pipeline_rolling_evaluation(
pipeline_params=pipeline_params, tscv=tscv)
expected_splits_n = tscv.max_splits
assert len(rolling_evaluation.keys()) == expected_splits_n
assert set(rolling_evaluation.keys()) == {"split_0", "split_1", "split_2"}
time_col = pipeline_params["time_col"]
for split_num, (train, test) in enumerate(tscv.split(X=df)):
split_output = rolling_evaluation[f"split_{split_num}"]
assert round(split_output["runtime_sec"],
3) == split_output["runtime_sec"]
pipeline_result = split_output["pipeline_result"]
# Checks every split uses all the available data for training
ts = pipeline_result.timeseries
train_end_date = df.iloc[train[-1]][time_col]
assert ts.train_end_date == train_end_date
assert pipeline_result.backtest is None
# Checks every split has forecast for train+test periods passed by tscv
forecast = pipeline_result.forecast
assert forecast.df.shape[0] == ts.fit_df.shape[
0] + tscv.periods_between_train_test + tscv.forecast_horizon