def test_benchmark_silverkite_template_with_simulated_data():
# setting every list to 1 item to speed up test case
forecast_horizons = [30]
max_cvs = [3]
fit_algorithms = ["linear"]
metric = EvaluationMetricEnum.MeanSquaredError
evaluation_metric = EvaluationMetricParam(cv_selection_metric=metric.name)
# Simulated data
data_name = "daily_simulated"
train_period = 365
data = generate_df_for_tests(freq="D", periods=train_period)
df = data["df"]
time_col, value_col = df.columns
metadata = MetadataParam(time_col=time_col, value_col=value_col, freq="D")
result_silverkite_simulated = benchmark_silverkite_template(
data_name=data_name,
df=df,
metadata=metadata,
evaluation_metric=evaluation_metric,
forecast_horizons=forecast_horizons,
fit_algorithms=fit_algorithms,
max_cvs=max_cvs)
result_silverkite_simulated = result_silverkite_simulated[0]
assert result_silverkite_simulated["data_name"] == data_name
assert result_silverkite_simulated["forecast_model_name"] == "silverkite_linear"
assert result_silverkite_simulated["train_period"] == train_period
assert result_silverkite_simulated["forecast_horizon"] == 30
assert result_silverkite_simulated["cv_folds"] == 3
def test_gen_moving_timeseries_forecast_with_regressors():
"""Test for `gen_moving_timeseries_forecast` with regressors"""
data = generate_df_for_tests(freq="1H", periods=1000)
df = data["df"]
df["x"] = 40
# A simple train-forecast function which always uses last available
# value, then adds the regressor value -> forecasted value
def train_forecast_func(
df,
value_col,
time_col,
forecast_horizon,
new_external_regressor_df):
# Simply get last observed value and offer as forecast
value = df[value_col].values[-1]
forecasted_values = np.repeat(a=value, repeats=forecast_horizon)
fut_df = pd.DataFrame({value_col: forecasted_values})
# Adds regerssor value
fut_df[value_col] = fut_df[value_col] + new_external_regressor_df["x"]
return {"fut_df": fut_df}
compare_df = gen_moving_timeseries_forecast(
df=df,
time_col="ts",
value_col="y",
train_move_ahead=10,
forecast_horizon=7,
train_forecast_func=train_forecast_func,
min_training_end_point=100,
regressor_cols=["x"])["compare_df"]
assert list(compare_df.head(5)["y_hat"].round(1).values) == [39.4] * 5
def test_run_template_1():
"""Runs default template"""
data = generate_df_for_tests(
freq="H",
periods=700 * 24)
df = data["train_df"]
forecast_horizon = data["test_df"].shape[0]
config = ForecastConfig(
model_template=ModelTemplateEnum.SK.name,
forecast_horizon=forecast_horizon,
)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
result = Forecaster().run_forecast_config(
df=df,
config=config,
)
rmse = EvaluationMetricEnum.RootMeanSquaredError.get_metric_name()
q80 = EvaluationMetricEnum.Quantile80.get_metric_name()
assert result.backtest.test_evaluation[rmse] == pytest.approx(2.037, rel=1e-2)
assert result.backtest.test_evaluation[q80] == pytest.approx(0.836, rel=1e-2)
assert result.forecast.train_evaluation[rmse] == pytest.approx(2.004, rel=1e-2)
assert result.forecast.train_evaluation[q80] == pytest.approx(0.800, rel=1e-2)
check_forecast_pipeline_result(
result,
coverage=None,
strategy=None,
score_func=EvaluationMetricEnum.MeanAbsolutePercentError.name,
greater_is_better=False)
def hourly_data():
"""Generate 500 days of hourly data for tests"""
return generate_df_for_tests(freq="H",
periods=24 * 500,
train_start_date=datetime.datetime(
2018, 7, 1),
conti_year_origin=2018)
def test_plot_grouping_evaluation():
"""Tests plot_grouping_evaluation function"""
df = generate_df_for_tests(freq="D", periods=20)["df"]
df.rename(columns={
TIME_COL: "custom_time_column",
VALUE_COL: "custom_value_column"
},
inplace=True)
ts = UnivariateTimeSeries()
ts.load_data(df,
time_col="custom_time_column",
value_col="custom_value_column")
# groupby_time_feature
fig = ts.plot_grouping_evaluation(aggregation_func=np.mean,
aggregation_func_name="mean",
groupby_time_feature="dow")
assert fig.data[0].name == f"mean of {VALUE_COL}"
assert fig.layout.xaxis.title.text == "dow"
assert fig.layout.yaxis.title.text == f"mean of {VALUE_COL}"
assert fig.layout.title.text == f"mean of {VALUE_COL} vs dow"
assert fig.data[0].x.shape[0] == 7
# groupby_sliding_window_size
fig = ts.plot_grouping_evaluation(
aggregation_func=np.max,
aggregation_func_name="max",
groupby_sliding_window_size=7
) # there are 20 training points, so this creates groups of size (6, 7, 7)
assert fig.data[0].name == f"max of {VALUE_COL}"
assert fig.layout.xaxis.title.text == f"{TIME_COL}_downsample"
assert fig.layout.yaxis.title.text == f"max of {VALUE_COL}"
assert fig.layout.title.text == f"max of {VALUE_COL} vs {TIME_COL}_downsample"
assert fig.data[0].x.shape[0] == 3
# groupby_custom_column
custom_groups = pd.Series(["g1", "g2", "g3", "g4", "g5"],
name="custom_groups").repeat(4)
fig = ts.plot_grouping_evaluation(aggregation_func=np.min,
aggregation_func_name="min",
groupby_custom_column=custom_groups)
assert fig.data[0].name == f"min of {VALUE_COL}"
assert fig.layout.xaxis.title.text == "custom_groups"
assert fig.layout.yaxis.title.text == f"min of {VALUE_COL}"
assert fig.layout.title.text == f"min of {VALUE_COL} vs custom_groups"
assert fig.data[0].x.shape[0] == 5
# custom xlabel, ylabel and title
fig = ts.plot_grouping_evaluation(aggregation_func=np.mean,
aggregation_func_name="mean",
groupby_time_feature="dow",
xlabel="Day of Week",
ylabel="Average of y",
title="Average of y by Day of week")
assert fig.layout.xaxis.title.text == "Day of Week"
assert fig.layout.yaxis.title.text == "Average of y"
assert fig.layout.title.text == "Average of y by Day of week"
def test_prophet_template_default():
"""Tests prophet_template with default values, for limited data"""
# prepares input data
num_days = 10
data = generate_df_for_tests(freq="D",
periods=num_days,
train_start_date="2018-01-01")
df = data["df"]
template = ProphetTemplate()
config = ForecastConfig(model_template="PROPHET")
params = template.apply_template_for_pipeline_params(df=df, config=config)
# not modified
assert config == ForecastConfig(model_template="PROPHET")
# checks result
metric = EvaluationMetricEnum.MeanAbsolutePercentError
pipeline = params.pop("pipeline", None)
expected_params = dict(
df=df,
time_col=cst.TIME_COL,
value_col=cst.VALUE_COL,
date_format=None,
freq=None,
train_end_date=None,
anomaly_info=None,
# model
regressor_cols=None,
lagged_regressor_cols=None,
estimator=None,
hyperparameter_grid=template.hyperparameter_grid,
hyperparameter_budget=None,
n_jobs=COMPUTATION_N_JOBS,
verbose=1,
# forecast
forecast_horizon=None,
coverage=None,
test_horizon=None,
periods_between_train_test=None,
agg_periods=None,
agg_func=None,
# evaluation
score_func=metric.name,
score_func_greater_is_better=metric.get_metric_greater_is_better(),
cv_report_metrics=CV_REPORT_METRICS_ALL,
null_model_params=None,
relative_error_tolerance=None,
# CV
cv_horizon=None,
cv_min_train_periods=None,
cv_expanding_window=True,
cv_periods_between_splits=None,
cv_periods_between_train_test=None,
cv_max_splits=3)
assert_basic_pipeline_equal(pipeline, template.pipeline)
assert_equal(params, expected_params)
def test_gen_moving_timeseries_forecast():
"""Basic test for `gen_moving_timeseries_forecast`"""
data = generate_df_for_tests(freq="1H", periods=1000)
df = data["df"]
# A simple train-forecast function which always uses last available
# value as the forecasted value
def train_forecast_func(
df,
value_col,
time_col=None,
forecast_horizon=1):
# Simply get last observed value and offer as forecast
value = df[value_col].values[df.shape[0] - 1]
forecasted_values = np.repeat(a=value, repeats=forecast_horizon)
fut_df = pd.DataFrame({value_col: forecasted_values})
return {"fut_df": fut_df}
compare_df = gen_moving_timeseries_forecast(
df=df,
time_col="ts",
value_col="y",
train_forecast_func=train_forecast_func,
train_move_ahead=10,
forecast_horizon=7,
min_training_end_point=100)["compare_df"]
assert list(compare_df.head(5)["y_hat"].round(1).values) == [-0.6] * 5
# Input start and end times
compare_df = gen_moving_timeseries_forecast(
df=df,
time_col="ts",
value_col="y",
train_forecast_func=train_forecast_func,
train_move_ahead=10,
forecast_horizon=7,
min_training_end_point=None,
min_training_end_timestamp="2018-07-15",
max_forecast_end_point=None,
max_forecast_end_timestamp="2018-08-01")["compare_df"]
assert list(compare_df.head(5)["y_hat"].round(1).values) == [-3.5] * 5
expected_match = "No reasonble train test period is found for validation"
with pytest.raises(ValueError, match=expected_match):
gen_moving_timeseries_forecast(
df=df,
time_col="ts",
value_col="y",
train_forecast_func=train_forecast_func,
train_move_ahead=10,
forecast_horizon=700,
min_training_end_point=1000)["compare_df"]
def test_apply_template_decorator():
data = generate_df_for_tests(freq="D", periods=10)
df = data["df"]
template = ProphetTemplate()
with pytest.raises(
ValueError,
match=
"ProphetTemplate only supports config.model_template='PROPHET', found 'UNKNOWN'"
):
template.apply_template_for_pipeline_params(
df=df, config=ForecastConfig(model_template="UNKNOWN"))
def test_generate_df_for_tests():
"""Test generate_df_for_tests"""
data = generate_df_for_tests(freq="H",
periods=24 * 10,
train_start_date=datetime.datetime(
2018, 1, 1),
train_frac=0.9,
remove_extra_cols=False)
assert data["df"].shape == (24 * 10, 48) # Contains time_feature columns
assert not data["train_df"].isna().any().any()
assert not data["test_df"][TIME_COL].isna().any().any()
def test_silverkite_template():
"""Tests test_silverkite_template with default config"""
data = generate_df_for_tests(freq="D", periods=10)
df = data["df"]
template = SilverkiteTemplate()
config = ForecastConfig(model_template="SK")
params = template.apply_template_for_pipeline_params(
df=df,
config=config
)
assert config == ForecastConfig(model_template="SK") # not modified
pipeline = params.pop("pipeline", None)
metric = EvaluationMetricEnum.MeanAbsolutePercentError
expected_params = dict(
df=df,
time_col=TIME_COL,
value_col=VALUE_COL,
date_format=None,
freq=None,
train_end_date=None,
anomaly_info=None,
# model
regressor_cols=None,
estimator=None,
hyperparameter_grid=template.hyperparameter_grid,
hyperparameter_budget=None,
n_jobs=COMPUTATION_N_JOBS,
verbose=1,
# forecast
forecast_horizon=None,
coverage=None,
test_horizon=None,
periods_between_train_test=None,
agg_periods=None,
agg_func=None,
# evaluation
score_func=metric.name,
score_func_greater_is_better=metric.get_metric_greater_is_better(),
cv_report_metrics=CV_REPORT_METRICS_ALL,
null_model_params=None,
relative_error_tolerance=None,
# CV
cv_horizon=None,
cv_min_train_periods=None,
cv_expanding_window=True,
cv_periods_between_splits=None,
cv_periods_between_train_test=None,
cv_max_splits=3
)
assert_basic_pipeline_equal(pipeline, template.pipeline)
assert_equal(params, expected_params)
def test_forecast_via_prophet_freq():
"""Tests prophet model at different frequencies"""
holidays = pd.DataFrame({
"ds":
pd.to_datetime(["2018-12-25", "2019-12-25", "2020-12-25"]),
"holiday":
"christmas",
"lower_window":
-2,
"upper_window":
2,
})
params = dict(coverage=0.9,
growth="linear",
n_changepoints=2,
changepoint_range=0.9,
yearly_seasonality="auto",
weekly_seasonality="auto",
daily_seasonality="auto",
holidays=holidays,
seasonality_mode="additive",
seasonality_prior_scale=5.0,
holidays_prior_scale=5.0,
changepoint_prior_scale=0.10,
mcmc_samples=0,
uncertainty_samples=10)
# A wide variety of frequencies listed here:
# https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#offset-aliases
frequencies = [
"B", "W", "W-SAT", "W-TUE", "M", "SM", "MS", "SMS", "CBMS", "BM", "B",
"Q", "QS", "BQS", "BQ-AUG", "Y", "YS", "AS-SEP", "H", "BH", "T", "S"
]
for freq in frequencies:
df = generate_df_for_tests(freq=freq, periods=50)
train_df = df["train_df"]
test_df = df["test_df"]
# tests model fit and predict work without error
model = ProphetEstimator(**params)
try:
model.fit(train_df, time_col=TIME_COL, value_col=VALUE_COL)
pred = model.predict(test_df)
except Exception:
print(f"Failed for frequency {freq}")
raise
assert list(pred.columns) == [
TIME_COL, PREDICTED_COL, PREDICTED_LOWER_COL, PREDICTED_UPPER_COL
]
assert pred[TIME_COL].equals(test_df[TIME_COL])
model.summary()
def test_get_basic_pipeline_apply():
"""Tests get_basic_pipeline fit and predict methods on a dataset without regressors"""
df = generate_df_for_tests("D", 50)
pipeline = get_basic_pipeline(
estimator=ProphetEstimator(),
score_func=EvaluationMetricEnum.MeanSquaredError.name,
score_func_greater_is_better=False,
agg_periods=None,
agg_func=None,
relative_error_tolerance=None,
coverage=0.95,
null_model_params=None)
pipeline.fit(df["train_df"])
predictions = pipeline.predict(df["test_df"])
assert predictions.shape[0] == df["test_df"].shape[0]
def test_run_forecast_json():
"""Tests:
- no coverage
- hourly data (2+ years)
- default `hyperparameter_grid` (all interaction terms enabled)
"""
# sets random state for consistent comparison
data = generate_df_for_tests(freq="H", periods=700 * 24)
df = data["train_df"]
json_str = """{
"model_template": "SILVERKITE",
"forecast_horizon": 3359,
"model_components_param": {
"custom": {
"fit_algorithm_dict": {
"fit_algorithm": "linear"
}
}
}
}"""
with warnings.catch_warnings():
warnings.simplefilter("ignore")
forecaster = Forecaster()
result = forecaster.run_forecast_json(df=df, json_str=json_str)
mse = EvaluationMetricEnum.RootMeanSquaredError.get_metric_name()
q80 = EvaluationMetricEnum.Quantile80.get_metric_name()
assert result.backtest.test_evaluation[mse] == pytest.approx(2.120,
rel=0.03)
assert result.backtest.test_evaluation[q80] == pytest.approx(0.863,
rel=0.02)
assert result.forecast.train_evaluation[mse] == pytest.approx(1.975,
rel=0.02)
assert result.forecast.train_evaluation[q80] == pytest.approx(0.786,
rel=1e-2)
check_forecast_pipeline_result(
result,
coverage=None,
strategy=None,
score_func=EvaluationMetricEnum.MeanAbsolutePercentError.name,
greater_is_better=False)
def test_forecast_via_arima_freq(params):
frequencies = ["H", "D", "M"]
for freq in frequencies:
df = generate_df_for_tests(freq=freq, periods=50)
train_df = df["train_df"]
test_df = df["test_df"]
# tests model fit and predict work without error
model = AutoArimaEstimator(**params)
try:
model.fit(train_df, time_col=TIME_COL, value_col=VALUE_COL)
pred = model.predict(test_df)
except Exception:
print(f"Failed for frequency {freq}")
raise
assert list(pred.columns) == [
TIME_COL, PREDICTED_COL, PREDICTED_LOWER_COL, PREDICTED_UPPER_COL
]
assert pred[TIME_COL].equals(test_df[TIME_COL])
model.summary()
def test_run_forecast_config():
"""Tests `run_forecast_config`"""
data = generate_df_for_tests(freq="H", periods=14 * 24)
df = data["df"]
# Checks if exception is raised
with pytest.raises(ValueError, match="is not recognized"):
forecaster = Forecaster()
forecaster.run_forecast_config(
df=df, config=ForecastConfig(model_template="unknown_template"))
with pytest.raises(ValueError, match="is not recognized"):
forecaster = Forecaster()
forecaster.run_forecast_json(
df=df, json_str="""{ "model_template": "unknown_template" }""")
# All run_forecast_config* functions return the same result for the default config,
# call forecast_pipeline, and return a result with the proper format.
np.random.seed(123)
forecaster = Forecaster()
default_result = forecaster.run_forecast_config(df=df)
score_func = EvaluationMetricEnum.MeanAbsolutePercentError.name
check_forecast_pipeline_result(default_result,
coverage=None,
strategy=None,
score_func=score_func,
greater_is_better=False)
assert_equal(forecaster.forecast_result, default_result)
np.random.seed(123)
forecaster = Forecaster()
json_result = forecaster.run_forecast_json(df=df)
check_forecast_pipeline_result(json_result,
coverage=None,
strategy=None,
score_func=score_func,
greater_is_better=False)
assert_forecast_pipeline_result_equal(json_result,
default_result,
rel=0.02)
def test_forecast_pipeline_rolling_evaluation_error():
"""Checks errors of forecast_pipeline_rolling_evaluation"""
data = generate_df_for_tests(freq="D", periods=30)
df = data["df"]
tscv = RollingTimeSeriesSplit(forecast_horizon=7,
periods_between_train_test=7,
max_splits=1)
# Different forecast_horizon in pipeline_params and tscv
with pytest.raises(ValueError,
match="Forecast horizon in 'pipeline_params' "
"does not match that of the 'tscv'."):
pipeline_params = mock_pipeline(df=df, forecast_horizon=15)
forecast_pipeline_rolling_evaluation(pipeline_params=pipeline_params,
tscv=tscv)
with pytest.raises(
ValueError,
match="'periods_between_train_test' in 'pipeline_params' "
"does not match that of the 'tscv'."):
pipeline_params = mock_pipeline(df=df,
forecast_horizon=7,
periods_between_train_test=2)
forecast_pipeline_rolling_evaluation(pipeline_params=pipeline_params,
tscv=tscv)
def daily_data():
return generate_df_for_tests(freq="D", periods=500, conti_year_origin=2018)
def test_forecast_similarity_based():
""" Testing the function: forecast_similarity_based in various examples
"""
data = generate_df_for_tests(freq="D", periods=30 * 8) # 8 months
df = data["df"]
train_df = data["train_df"]
test_df = data["test_df"]
df["z"] = df["y"] + 1
train_df["z"] = train_df["y"] + 1
test_df["z"] = test_df["y"] + 1
train_df = train_df[["ts", "y", "z"]]
test_df = test_df[["ts", "y", "z"]]
res = forecast_similarity_based(df=train_df,
time_col="ts",
value_cols=["y", "z"],
agg_method="median",
agg_func=None,
match_cols=["dow"])
# forecast using predict
fdf_median = res["predict"](test_df)
assert (fdf_median["z"] - fdf_median["y"] - 1.0).abs().max().round(2) == 0.0, \
"forecast for z must be forecast for y + 1 at each timestamp"
err = calc_pred_err(test_df["y"], fdf_median["y"])
enum = EvaluationMetricEnum.Correlation
assert err[enum.get_metric_name()] > 0.3
# forecast using predict_n
fdf_median = res["predict_n"](test_df.shape[0])
err = calc_pred_err(test_df["y"], fdf_median["y"])
assert err[enum.get_metric_name()] > 0.3
res = forecast_similarity_based(df=train_df,
time_col="ts",
value_cols=["y", "z"],
agg_method="mean",
agg_func=None,
match_cols=["dow"])
# forecast using the mean of all similar times
fdf_mean = res["predict"](test_df)
err = calc_pred_err(test_df["y"], fdf_mean["y"])
assert err[enum.get_metric_name()] > 0.3
res = forecast_similarity_based(df=train_df,
time_col="ts",
value_cols=["y", "z"],
agg_method="most_recent",
agg_func=None,
match_cols=["dow"],
recent_k=3)
# forecast using the mean of 3 recent times similar to the given time
fdf_recent3_mean = res["predict"](test_df)
err = calc_pred_err(test_df["y"], fdf_recent3_mean["y"])
assert err[enum.get_metric_name()] > 0.3
plt.plot(df["ts"].dt.strftime('%Y-%m-%d'),
df["y"],
label="true",
alpha=0.5)
plt.plot(fdf_median["ts"].dt.strftime('%Y-%m-%d'),
fdf_median["y"],
alpha=0.5,
label="median pred wrt dow")
plt.plot(fdf_mean["ts"].dt.strftime('%Y-%m-%d'),
fdf_mean["y"],
alpha=0.5,
label="mean pred wrt dow")
plt.plot(fdf_recent3_mean["ts"].dt.strftime('%Y-%m-%d'),
fdf_recent3_mean["y"],
alpha=0.5,
label="mean recent 3 wrt dow")
plt.xticks(rotation=15)
plt.legend()
def test_run_forecast_config_with_single_simple_silverkite_template():
# The generic name of single simple silverkite templates are not added to `ModelTemplateEnum`,
# therefore we test if these are recognized.
data = generate_df_for_tests(freq="D", periods=365)
df = data["df"]
metric = EvaluationMetricEnum.MeanAbsoluteError
evaluation_metric = EvaluationMetricParam(cv_selection_metric=metric.name,
agg_periods=7,
agg_func=np.max,
null_model_params={
"strategy": "quantile",
"constant": None,
"quantile": 0.5
})
evaluation_period = EvaluationPeriodParam(test_horizon=10,
periods_between_train_test=5,
cv_horizon=4,
cv_min_train_periods=80,
cv_expanding_window=False,
cv_periods_between_splits=20,
cv_periods_between_train_test=3,
cv_max_splits=2)
model_components = ModelComponentsParam(
hyperparameter_override=[{
"estimator__yearly_seasonality": 1
}, {
"estimator__yearly_seasonality": 2
}])
computation = ComputationParam(verbose=2)
forecast_horizon = 27
coverage = 0.90
single_template_class = SimpleSilverkiteTemplateOptions(
freq=SILVERKITE_COMPONENT_KEYWORDS.FREQ.value.DAILY,
seas=SILVERKITE_COMPONENT_KEYWORDS.SEAS.value.NONE)
forecast_config = ForecastConfig(model_template=[
single_template_class, "DAILY_ALGO_SGD", "SILVERKITE_DAILY_90"
],
computation_param=computation,
coverage=coverage,
evaluation_metric_param=evaluation_metric,
evaluation_period_param=evaluation_period,
forecast_horizon=forecast_horizon,
model_components_param=model_components)
forecaster = Forecaster()
result = forecaster.run_forecast_config(df=df, config=forecast_config)
summary = summarize_grid_search_results(result.grid_search)
# single_template_class is 1 template,
# "DAILY_ALGO_SGD" is 1 template and "SILVERKITE_DAILY_90" has 4 templates.
# With 2 items in `hyperparameter_override, there should be a total of 12 cases.
assert summary.shape[0] == 12
# Tests functionality for single template class only.
forecast_config = ForecastConfig(model_template=single_template_class,
computation_param=computation,
coverage=coverage,
evaluation_metric_param=evaluation_metric,
evaluation_period_param=evaluation_period,
forecast_horizon=forecast_horizon)
forecaster = Forecaster()
pipeline_parameters = forecaster.apply_forecast_config(
df=df, config=forecast_config)
assert_equal(actual=pipeline_parameters["hyperparameter_grid"],
expected={
"estimator__time_properties": [None],
"estimator__origin_for_time_vars": [None],
"estimator__train_test_thresh": [None],
"estimator__training_fraction": [None],
"estimator__fit_algorithm_dict": [{
"fit_algorithm":
"linear",
"fit_algorithm_params":
None
}],
"estimator__holidays_to_model_separately": [[]],
"estimator__holiday_lookup_countries": [[]],
"estimator__holiday_pre_num_days": [0],
"estimator__holiday_post_num_days": [0],
"estimator__holiday_pre_post_num_dict": [None],
"estimator__daily_event_df_dict": [None],
"estimator__changepoints_dict": [None],
"estimator__seasonality_changepoints_dict": [None],
"estimator__yearly_seasonality": [0],
"estimator__quarterly_seasonality": [0],
"estimator__monthly_seasonality": [0],
"estimator__weekly_seasonality": [0],
"estimator__daily_seasonality": [0],
"estimator__max_daily_seas_interaction_order": [0],
"estimator__max_weekly_seas_interaction_order": [2],
"estimator__autoreg_dict": [None],
"estimator__min_admissible_value": [None],
"estimator__max_admissible_value": [None],
"estimator__uncertainty_dict": [None],
"estimator__growth_term": ["linear"],
"estimator__regressor_cols": [[]],
"estimator__feature_sets_enabled": [False],
"estimator__extra_pred_cols": [[]]
},
ignore_keys={"estimator__time_properties": None})
def pipeline_results():
"""Runs forecast_pipeline three times to get
grid search results"""
pipeline_results = {}
data = generate_df_for_tests(freq="1D", periods=20 * 7)
df = data["df"]
with warnings.catch_warnings():
warnings.simplefilter("ignore")
hyperparameter_grid = [{
"estimator__strategy": ["quantile"],
"estimator__quantile": [0.9]
}, {
"estimator__strategy": ["mean"]
}, {
"estimator__strategy": ["constant"],
"estimator__constant": [1.0, 2.0]
}]
pipeline = Pipeline([("estimator", DummyEstimator())])
# Tests MAPE `score_func`, list `cv_report_metrics`
metric = EvaluationMetricEnum.MeanAbsolutePercentError
pipeline_results["1"] = forecast_pipeline(
df,
pipeline=pipeline,
hyperparameter_grid=hyperparameter_grid,
n_jobs=-1,
forecast_horizon=20,
coverage=None,
agg_periods=7,
agg_func=np.sum,
score_func=metric.name,
score_func_greater_is_better=metric.get_metric_greater_is_better(),
cv_report_metrics=[
EvaluationMetricEnum.MeanAbsoluteError.name,
EvaluationMetricEnum.MeanSquaredError.name,
EvaluationMetricEnum.MedianAbsolutePercentError.name,
],
null_model_params=None)
# Tests FRACTION_OUTSIDE_TOLERANCE `score_func`, all `cv_report_metrics`
pipeline = Pipeline([("estimator", DummyEstimator())])
pipeline_results["2"] = forecast_pipeline(
df,
pipeline=pipeline,
hyperparameter_grid=hyperparameter_grid,
n_jobs=-1,
forecast_horizon=20,
coverage=None,
score_func=FRACTION_OUTSIDE_TOLERANCE,
score_func_greater_is_better=False,
cv_report_metrics=CV_REPORT_METRICS_ALL,
null_model_params=None,
relative_error_tolerance=0.02)
# Tests callable `score_func`, greater_is_better=True, no `cv_report_metrics`
fs1 = pd.DataFrame({
"name": ["tow", "conti_year"],
"period": [7.0, 1.0],
"order": [3, 3],
"seas_names": ["weekly", None]
})
fs2 = pd.DataFrame({
"name": ["tow"],
"period": [7.0],
"order": [3],
"seas_names": ["weekly"]
})
hyperparameter_grid = {
"estimator__origin_for_time_vars": [2018],
"estimator__extra_pred_cols": [["ct1"], ["ct2"]],
"estimator__fit_algorithm_dict": [{
"fit_algorithm": "linear"
}],
"estimator__fs_components_df": [fs1, fs2],
}
cv_max_splits = 2
pipeline_results["3"] = forecast_pipeline(
df,
estimator=SilverkiteEstimator(),
hyperparameter_grid=hyperparameter_grid,
hyperparameter_budget=4,
n_jobs=1,
forecast_horizon=3 * 7,
test_horizon=2 * 7,
score_func=mean_absolute_error, # callable score_func
score_func_greater_is_better=
True, # Not really True, only for the sake of testing
null_model_params=None,
cv_horizon=1 * 7,
cv_expanding_window=True,
cv_min_train_periods=7 * 7,
cv_periods_between_splits=7,
cv_periods_between_train_test=3 * 7,
cv_max_splits=cv_max_splits)
return pipeline_results
def test_get_forecast_time_properties():
"""Tests get_forecast_time_properties"""
num_training_points = 365 # one year of daily data
data = generate_df_for_tests(freq="D", periods=num_training_points)
df = data["df"]
result = get_forecast_time_properties(df,
time_col=TIME_COL,
value_col=VALUE_COL,
freq="D",
forecast_horizon=0)
default_origin = get_default_origin_for_time_vars(df, TIME_COL)
assert result == {
"period": TimeEnum.ONE_DAY_IN_SECONDS.value,
"simple_freq": SimpleTimeFrequencyEnum.DAY,
"num_training_points": num_training_points,
"num_training_days": num_training_points,
"days_per_observation": 1,
"forecast_horizon": 0,
"forecast_horizon_in_timedelta": timedelta(days=0),
"forecast_horizon_in_days": 0,
"start_year": 2018,
"end_year": 2019,
"origin_for_time_vars": default_origin
}
# longer forecast_horizon
result = get_forecast_time_properties(df,
time_col=TIME_COL,
value_col=VALUE_COL,
freq="D",
forecast_horizon=365)
default_origin = get_default_origin_for_time_vars(df, TIME_COL)
assert result == {
"period": TimeEnum.ONE_DAY_IN_SECONDS.value,
"simple_freq": SimpleTimeFrequencyEnum.DAY,
"num_training_points": num_training_points,
"num_training_days": num_training_points,
"days_per_observation": 1,
"forecast_horizon": 365,
"forecast_horizon_in_timedelta": timedelta(days=365),
"forecast_horizon_in_days": 365,
"start_year": 2018,
"end_year": 2020,
"origin_for_time_vars": default_origin
}
# two years of hourly data
num_training_points = 2 * 365 * 24
data = generate_df_for_tests(freq="H", periods=num_training_points)
df = data["df"]
result = get_forecast_time_properties(df,
time_col=TIME_COL,
value_col=VALUE_COL,
freq="H",
forecast_horizon=0)
default_origin = get_default_origin_for_time_vars(df, TIME_COL)
assert result == {
"period": TimeEnum.ONE_HOUR_IN_SECONDS.value,
"simple_freq": SimpleTimeFrequencyEnum.HOUR,
"num_training_points": num_training_points,
"num_training_days": num_training_points / 24,
"days_per_observation": 1 / 24,
"forecast_horizon": 0,
"forecast_horizon_in_timedelta": timedelta(days=0),
"forecast_horizon_in_days": 0,
"start_year": 2018,
"end_year": 2020,
"origin_for_time_vars": default_origin
}
# longer forecast_horizon
result = get_forecast_time_properties(df,
time_col=TIME_COL,
value_col=VALUE_COL,
freq="H",
forecast_horizon=365 * 24)
default_origin = get_default_origin_for_time_vars(df, TIME_COL)
assert result == {
"period": TimeEnum.ONE_HOUR_IN_SECONDS.value,
"simple_freq": SimpleTimeFrequencyEnum.HOUR,
"num_training_points": num_training_points,
"num_training_days": num_training_points / 24,
"days_per_observation": 1 / 24,
"forecast_horizon": 365 * 24,
"forecast_horizon_in_timedelta": timedelta(days=365),
"forecast_horizon_in_days": 365,
"start_year": 2018,
"end_year": 2021,
"origin_for_time_vars": default_origin
}
# ``forecast_horizon=None``
result = get_forecast_time_properties(df,
time_col=TIME_COL,
value_col=VALUE_COL,
freq="H",
forecast_horizon=None)
default_origin = get_default_origin_for_time_vars(df, TIME_COL)
assert result == {
"period": TimeEnum.ONE_HOUR_IN_SECONDS.value,
"simple_freq": SimpleTimeFrequencyEnum.HOUR,
"num_training_points": num_training_points,
"num_training_days": num_training_points / 24,
"days_per_observation": 1 / 24,
"forecast_horizon": 24,
"forecast_horizon_in_timedelta": timedelta(days=1),
"forecast_horizon_in_days": 1,
"start_year": 2018,
"end_year": 2020,
"origin_for_time_vars": default_origin
}
# weekly df with regressors
num_training_points = 50
data = generate_df_with_reg_for_tests(freq="W-SUN",
periods=num_training_points,
train_start_date=datetime.datetime(
2018, 11, 30),
remove_extra_cols=True,
mask_test_actuals=True)
df = data["df"]
train_df = data["train_df"]
forecast_horizon = data["fut_time_num"]
regressor_cols = [
col for col in df.columns if col not in [TIME_COL, VALUE_COL]
]
result = get_forecast_time_properties(df,
time_col=TIME_COL,
value_col=VALUE_COL,
freq="W-SUN",
regressor_cols=regressor_cols,
forecast_horizon=forecast_horizon)
default_origin = get_default_origin_for_time_vars(df, TIME_COL)
assert result == {
"period": TimeEnum.ONE_WEEK_IN_SECONDS.value,
"simple_freq": SimpleTimeFrequencyEnum.WEEK,
"num_training_points": train_df.shape[0], # size of training set
"num_training_days": train_df.shape[0] * 7,
"days_per_observation": 7,
"forecast_horizon": 9,
"forecast_horizon_in_timedelta": timedelta(days=63),
"forecast_horizon_in_days": 63.0,
"start_year": 2018,
"end_year": 2019,
"origin_for_time_vars": default_origin
}
# checks `num_training_days` with `train_end_date`
data = generate_df_with_reg_for_tests(freq="H",
periods=300 * 24,
train_start_date=datetime.datetime(
2018, 7, 1),
remove_extra_cols=True,
mask_test_actuals=True)
df = data["df"]
train_end_date = datetime.datetime(2019, 2, 1)
result = get_forecast_time_properties(
df=df,
time_col=TIME_COL,
value_col=VALUE_COL,
freq="H",
regressor_cols=data["regressor_cols"],
train_end_date=train_end_date,
forecast_horizon=forecast_horizon)
period = 3600 # seconds between observations
time_delta = (train_end_date - df[TIME_COL].min()
) # train end - train start
num_training_days = (
time_delta.days +
(time_delta.seconds + period) / TimeEnum.ONE_DAY_IN_SECONDS.value)
assert result["num_training_days"] == num_training_days
# checks `num_training_days` without `train_end_date`
result = get_forecast_time_properties(
df=df,
time_col=TIME_COL,
value_col=VALUE_COL,
freq="H",
regressor_cols=data["regressor_cols"],
train_end_date=None,
forecast_horizon=forecast_horizon)
time_delta = (
datetime.datetime(2019, 2, 26) - df[TIME_COL].min()
) # by default, train end is the last date with nonnull value_col
num_training_days = (
time_delta.days +
(time_delta.seconds + period) / TimeEnum.ONE_DAY_IN_SECONDS.value)
assert result["num_training_days"] == num_training_days
def hourly_data():
"""Generate 500 days of hourly data for tests"""
return generate_df_for_tests(freq="H", periods=24 * 500)
def daily_data():
return generate_df_for_tests(freq="D",
periods=730,
train_start_date=datetime.datetime(
2018, 1, 1),
conti_year_origin=2018)
