def test_run_template_1():
"""Runs default template"""
data = generate_df_for_tests(
freq="H",
periods=700 * 24)
df = data["train_df"]
forecast_horizon = data["test_df"].shape[0]
config = ForecastConfig(
model_template=ModelTemplateEnum.SK.name,
forecast_horizon=forecast_horizon,
)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
result = Forecaster().run_forecast_config(
df=df,
config=config,
)
rmse = EvaluationMetricEnum.RootMeanSquaredError.get_metric_name()
q80 = EvaluationMetricEnum.Quantile80.get_metric_name()
assert result.backtest.test_evaluation[rmse] == pytest.approx(2.037, rel=1e-2)
assert result.backtest.test_evaluation[q80] == pytest.approx(0.836, rel=1e-2)
assert result.forecast.train_evaluation[rmse] == pytest.approx(2.004, rel=1e-2)
assert result.forecast.train_evaluation[q80] == pytest.approx(0.800, rel=1e-2)
check_forecast_pipeline_result(
result,
coverage=None,
strategy=None,
score_func=EvaluationMetricEnum.MeanAbsolutePercentError.name,
greater_is_better=False)
def test_run_forecast_json():
"""Tests:
- no coverage
- hourly data (2+ years)
- default `hyperparameter_grid` (all interaction terms enabled)
"""
# sets random state for consistent comparison
data = generate_df_for_tests(freq="H", periods=700 * 24)
df = data["train_df"]
json_str = """{
"model_template": "SILVERKITE",
"forecast_horizon": 3359,
"model_components_param": {
"custom": {
"fit_algorithm_dict": {
"fit_algorithm": "linear"
}
}
}
}"""
with warnings.catch_warnings():
warnings.simplefilter("ignore")
forecaster = Forecaster()
result = forecaster.run_forecast_json(df=df, json_str=json_str)
mse = EvaluationMetricEnum.RootMeanSquaredError.get_metric_name()
q80 = EvaluationMetricEnum.Quantile80.get_metric_name()
assert result.backtest.test_evaluation[mse] == pytest.approx(2.120,
rel=0.03)
assert result.backtest.test_evaluation[q80] == pytest.approx(0.863,
rel=0.02)
assert result.forecast.train_evaluation[mse] == pytest.approx(1.975,
rel=0.02)
assert result.forecast.train_evaluation[q80] == pytest.approx(0.786,
rel=1e-2)
check_forecast_pipeline_result(
result,
coverage=None,
strategy=None,
score_func=EvaluationMetricEnum.MeanAbsolutePercentError.name,
greater_is_better=False)
def test_run_template_4():
"""Runs custom template with monthly data and auto-regression"""
data = generate_df_with_reg_for_tests(
freq="MS",
periods=48,
remove_extra_cols=True,
mask_test_actuals=True)
reg_cols = ["regressor1", "regressor2", "regressor_categ"]
keep_cols = [TIME_COL, VALUE_COL] + reg_cols
df = data["df"][keep_cols]
forecast_horizon = data["test_df"].shape[0]
model_components = ModelComponentsParam(
custom=dict(
fit_algorithm_dict=dict(fit_algorithm="linear"),
extra_pred_cols=["ct2"]),
autoregression=dict(autoreg_dict=dict(lag_dict=dict(orders=[1]))),
uncertainty=dict(uncertainty_dict=None))
config = ForecastConfig(
model_template=ModelTemplateEnum.SK.name,
forecast_horizon=forecast_horizon,
coverage=0.9,
model_components_param=model_components,
)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
result = Forecaster().run_forecast_config(
df=df,
config=config,
)
rmse = EvaluationMetricEnum.RootMeanSquaredError.get_metric_name()
assert result.backtest.test_evaluation[rmse] == pytest.approx(4.95, rel=1e-1)
check_forecast_pipeline_result(
result,
coverage=0.9,
strategy=None,
score_func=EvaluationMetricEnum.MeanAbsolutePercentError.name,
greater_is_better=False)
def test_run_forecast_config():
"""Tests `run_forecast_config`"""
data = generate_df_for_tests(freq="H", periods=14 * 24)
df = data["df"]
# Checks if exception is raised
with pytest.raises(ValueError, match="is not recognized"):
forecaster = Forecaster()
forecaster.run_forecast_config(
df=df, config=ForecastConfig(model_template="unknown_template"))
with pytest.raises(ValueError, match="is not recognized"):
forecaster = Forecaster()
forecaster.run_forecast_json(
df=df, json_str="""{ "model_template": "unknown_template" }""")
# All run_forecast_config* functions return the same result for the default config,
# call forecast_pipeline, and return a result with the proper format.
np.random.seed(123)
forecaster = Forecaster()
default_result = forecaster.run_forecast_config(df=df)
score_func = EvaluationMetricEnum.MeanAbsolutePercentError.name
check_forecast_pipeline_result(default_result,
coverage=None,
strategy=None,
score_func=score_func,
greater_is_better=False)
assert_equal(forecaster.forecast_result, default_result)
np.random.seed(123)
forecaster = Forecaster()
json_result = forecaster.run_forecast_json(df=df)
check_forecast_pipeline_result(json_result,
coverage=None,
strategy=None,
score_func=score_func,
greater_is_better=False)
assert_forecast_pipeline_result_equal(json_result,
default_result,
rel=0.02)
def test_run_template_2():
"""Runs custom template with all options"""
data = generate_df_with_reg_for_tests(
freq="D",
periods=400,
remove_extra_cols=True,
mask_test_actuals=True)
reg_cols = ["regressor1", "regressor2", "regressor_categ"]
keep_cols = [TIME_COL, VALUE_COL] + reg_cols
df = data["df"][keep_cols]
forecast_horizon = data["test_df"].shape[0]
daily_event_df_dict = generate_holiday_events(
countries=["UnitedStates"],
holidays_to_model_separately=["New Year's Day"],
year_start=2017,
year_end=2022,
pre_num=2,
post_num=2)
event_pred_cols = get_event_pred_cols(daily_event_df_dict)
model_components = ModelComponentsParam(
seasonality={
"fs_components_df": pd.DataFrame({
"name": ["tow", "tom", "toq", "toy"],
"period": [7.0, 1.0, 1.0, 1.0],
"order": [2, 1, 1, 5],
"seas_names": ["weekly", "monthly", "quarterly", "yearly"]
})
},
events={
"daily_event_df_dict": daily_event_df_dict
},
changepoints={
"changepoints_dict": {
"method": "auto",
"yearly_seasonality_order": 3,
"regularization_strength": 0.5,
"resample_freq": "14D",
"potential_changepoint_distance": "56D",
"no_changepoint_proportion_from_end": 0.2
},
"seasonality_changepoints_dict": {
"potential_changepoint_distance": "60D",
"regularization_strength": 0.5,
"no_changepoint_proportion_from_end": 0.2
},
},
autoregression=None,
uncertainty={
"uncertainty_dict": None,
},
custom={
"origin_for_time_vars": None,
"extra_pred_cols": [["ct1"] + reg_cols + event_pred_cols], # growth, regressors, events
"fit_algorithm_dict": {
"fit_algorithm": "ridge",
"fit_algorithm_params": {"cv": 2}
},
"min_admissible_value": min(df[VALUE_COL]) - abs(max(df[VALUE_COL])),
"max_admissible_value": max(df[VALUE_COL]) * 2,
}
)
config = ForecastConfig(
model_template=ModelTemplateEnum.SK.name,
forecast_horizon=forecast_horizon,
coverage=0.9,
model_components_param=model_components,
)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
result = Forecaster().run_forecast_config(
df=df,
config=config,
)
rmse = EvaluationMetricEnum.RootMeanSquaredError.get_metric_name()
q80 = EvaluationMetricEnum.Quantile80.get_metric_name()
assert result.backtest.test_evaluation[rmse] == pytest.approx(2.692, rel=1e-2)
assert result.backtest.test_evaluation[q80] == pytest.approx(1.531, rel=1e-2)
assert result.backtest.test_evaluation[PREDICTION_BAND_COVERAGE] == pytest.approx(0.823, rel=1e-2)
assert result.forecast.train_evaluation[rmse] == pytest.approx(2.304, rel=1e-2)
assert result.forecast.train_evaluation[q80] == pytest.approx(0.921, rel=1e-2)
assert result.forecast.train_evaluation[PREDICTION_BAND_COVERAGE] == pytest.approx(0.897, rel=1e-2)
check_forecast_pipeline_result(
result,
coverage=0.9,
strategy=None,
score_func=EvaluationMetricEnum.MeanAbsolutePercentError.name,
greater_is_better=False)
def test_run_template_5():
"""Runs custom template with monthly data, auto-regression and lagged regressors"""
data = generate_df_with_reg_for_tests(
freq="MS",
periods=48,
remove_extra_cols=True,
mask_test_actuals=True)
reg_cols_all = ["regressor1", "regressor2", "regressor_categ"]
reg_cols = ["regressor1"]
keep_cols = [TIME_COL, VALUE_COL] + reg_cols_all
df = data["df"][keep_cols]
forecast_horizon = data["test_df"].shape[0]
model_components = ModelComponentsParam(
custom=dict(
fit_algorithm_dict=dict(fit_algorithm="linear"),
extra_pred_cols=reg_cols),
autoregression=dict(autoreg_dict=dict(lag_dict=dict(orders=[1]))),
lagged_regressors={
"lagged_regressor_dict": [
{"regressor2": "auto"},
{"regressor_categ": {"lag_dict": {"orders": [5]}}}
]},
uncertainty=dict(uncertainty_dict=None))
config = ForecastConfig(
model_template=ModelTemplateEnum.SK.name,
forecast_horizon=forecast_horizon,
coverage=0.9,
model_components_param=model_components,
)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
result = Forecaster().run_forecast_config(
df=df,
config=config,
)
rmse = EvaluationMetricEnum.RootMeanSquaredError.get_metric_name()
assert result.backtest.test_evaluation[rmse] == pytest.approx(4.46, rel=1e-1)
check_forecast_pipeline_result(
result,
coverage=0.9,
strategy=None,
score_func=EvaluationMetricEnum.MeanAbsolutePercentError.name,
greater_is_better=False)
# Checks lagged regressor columns
actual_pred_cols = set(result.model[-1].model_dict["pred_cols"])
actual_x_mat_cols = set(result.model[-1].model_dict["x_mat"].columns)
expected_pred_cols = {
'regressor1',
'y_lag1',
'regressor_categ_lag5'
}
expected_x_mat_cols = {
'regressor1',
'y_lag1',
'regressor_categ_lag5[T.c2]',
'regressor_categ_lag5[T.c2]'
}
assert expected_pred_cols.issubset(actual_pred_cols)
assert expected_x_mat_cols.issubset(actual_x_mat_cols)
def test_run_forecast_config_custom():
"""Tests `run_forecast_config` on weekly data with custom config:
- numeric and categorical regressors
- coverage
- null model
"""
data = generate_df_with_reg_for_tests(freq="W-MON",
periods=140,
remove_extra_cols=True,
mask_test_actuals=True)
reg_cols = ["regressor1", "regressor2", "regressor_categ"]
keep_cols = [TIME_COL, VALUE_COL] + reg_cols
df = data["df"][keep_cols]
metric = EvaluationMetricEnum.MeanAbsoluteError
evaluation_metric = EvaluationMetricParam(cv_selection_metric=metric.name,
agg_periods=7,
agg_func=np.max,
null_model_params={
"strategy": "quantile",
"constant": None,
"quantile": 0.5
})
evaluation_period = EvaluationPeriodParam(test_horizon=10,
periods_between_train_test=5,
cv_horizon=4,
cv_min_train_periods=80,
cv_expanding_window=False,
cv_periods_between_splits=20,
cv_periods_between_train_test=3,
cv_max_splits=3)
model_components = ModelComponentsParam(
regressors={"regressor_cols": reg_cols},
custom={
"fit_algorithm_dict": {
"fit_algorithm": "ridge",
"fit_algorithm_params": {
"cv": 2
}
}
})
computation = ComputationParam(verbose=2)
forecast_horizon = 27
coverage = 0.90
forecast_config = ForecastConfig(
model_template=ModelTemplateEnum.SILVERKITE.name,
computation_param=computation,
coverage=coverage,
evaluation_metric_param=evaluation_metric,
evaluation_period_param=evaluation_period,
forecast_horizon=forecast_horizon,
model_components_param=model_components)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
forecaster = Forecaster()
result = forecaster.run_forecast_config(df=df, config=forecast_config)
mse = EvaluationMetricEnum.RootMeanSquaredError.get_metric_name()
q80 = EvaluationMetricEnum.Quantile80.get_metric_name()
assert result.backtest.test_evaluation[mse] == pytest.approx(2.976,
rel=1e-2)
assert result.backtest.test_evaluation[q80] == pytest.approx(1.360,
rel=1e-2)
assert result.forecast.train_evaluation[mse] == pytest.approx(2.224,
rel=1e-2)
assert result.forecast.train_evaluation[q80] == pytest.approx(0.941,
rel=1e-2)
check_forecast_pipeline_result(result,
coverage=coverage,
strategy=None,
score_func=metric.name,
greater_is_better=False)
with pytest.raises(KeyError, match="missing_regressor"):
model_components = ModelComponentsParam(
regressors={"regressor_cols": ["missing_regressor"]})
forecaster = Forecaster()
result = forecaster.run_forecast_config(
df=df,
config=ForecastConfig(
model_template=ModelTemplateEnum.SILVERKITE.name,
model_components_param=model_components))
check_forecast_pipeline_result(result,
coverage=None,
strategy=None,
score_func=metric.get_metric_func(),
greater_is_better=False)