def apply_evaluation_period_defaults(
evaluation: Optional[EvaluationPeriodParam] = None
) -> EvaluationPeriodParam:
"""Applies the default EvaluationPeriodParam values to the given object.
If an expected attribute value is provided, the value is unchanged. Otherwise the default value for it is used.
Other attributes are untouched.
If the input object is None, it creates a EvaluationPeriodParam object.
Parameters
----------
evaluation : `~greykite.framework.templates.autogen.forecast_config.EvaluationPeriodParam` or None
The EvaluationMetricParam object.
Returns
-------
evaluation : `~greykite.framework.templates.autogen.forecast_config.EvaluationPeriodParam`
Valid EvaluationPeriodParam object with the provided attribute values and the default attribute values if not.
"""
if evaluation is None:
evaluation = EvaluationPeriodParam()
if evaluation.cv_max_splits is None:
evaluation.cv_max_splits = EVALUATION_PERIOD_CV_MAX_SPLITS
if evaluation.cv_periods_between_train_test is None:
evaluation.cv_periods_between_train_test = evaluation.periods_between_train_test
if evaluation.cv_expanding_window is None:
# NB: subclass may want to override.
evaluation.cv_expanding_window = True # good for long-term forecasts, or when data are limited
return evaluation
def df_config():
data = generate_df_with_reg_for_tests(freq="W-MON",
periods=140,
remove_extra_cols=True,
mask_test_actuals=True)
reg_cols = ["regressor1", "regressor2", "regressor_categ"]
keep_cols = [TIME_COL, VALUE_COL] + reg_cols
df = data["df"][keep_cols]
model_template = "SILVERKITE"
evaluation_metric = EvaluationMetricParam(
cv_selection_metric=EvaluationMetricEnum.MeanAbsoluteError.name,
agg_periods=7,
agg_func=np.max,
null_model_params={
"strategy": "quantile",
"constant": None,
"quantile": 0.5
})
evaluation_period = EvaluationPeriodParam(test_horizon=10,
periods_between_train_test=5,
cv_horizon=4,
cv_min_train_periods=80,
cv_expanding_window=False,
cv_periods_between_splits=20,
cv_periods_between_train_test=3,
cv_max_splits=3)
model_components = ModelComponentsParam(
regressors={"regressor_cols": reg_cols},
custom={
"fit_algorithm_dict": {
"fit_algorithm": "ridge",
"fit_algorithm_params": {
"cv": 2
}
}
})
computation = ComputationParam(verbose=2)
forecast_horizon = 27
coverage = 0.90
config = ForecastConfig(model_template=model_template,
computation_param=computation,
coverage=coverage,
evaluation_metric_param=evaluation_metric,
evaluation_period_param=evaluation_period,
forecast_horizon=forecast_horizon,
model_components_param=model_components)
return {
"df": df,
"config": config,
"model_template": model_template,
"reg_cols": reg_cols,
}
def test_forecast_config():
"""Tests ForecastConfig dataclass"""
config = ForecastConfig(
model_template=ModelTemplateEnum.SILVERKITE.name,
metadata_param=MetadataParam(time_col="custom_time_col",
anomaly_info=[{
"key": "value"
}, {
"key2": "value2"
}]),
evaluation_period_param=EvaluationPeriodParam(
test_horizon=10,
periods_between_train_test=5,
cv_min_train_periods=20),
evaluation_metric_param=EvaluationMetricParam(
cv_selection_metric=EvaluationMetricEnum.MeanSquaredError.name,
cv_report_metrics=[
EvaluationMetricEnum.MeanAbsoluteError.name,
EvaluationMetricEnum.MeanAbsolutePercentError.name
],
relative_error_tolerance=0.02),
model_components_param=ModelComponentsParam(
autoregression={"autoreg_dict": {
"autoreg_param": 0
}},
changepoints=None,
custom={"custom_param": 1},
growth={"growth_param": 2},
events={"events_param": 3},
hyperparameter_override=[{
"h1": 4
}, {
"h2": 5
}, None],
regressors={"names": ["regressor1", "regressor2"]},
lagged_regressors={"lagged_regressor_dict": {
"lag_reg_param": 0
}},
seasonality={"seas_param": 6},
uncertainty={"uncertainty_param": 7}),
computation_param=ComputationParam(n_jobs=None))
assert_forecast_config(config)
# Tests a string passed to `cv_report_metrics`
assert ForecastConfig(evaluation_metric_param=EvaluationMetricParam(
cv_report_metrics=CV_REPORT_METRICS_ALL), ).to_dict()
def test_prophet_template_custom():
"""Tests prophet_template with custom values, with long range input"""
# prepares input data
data = generate_df_with_reg_for_tests(freq="H",
periods=300 * 24,
remove_extra_cols=True,
mask_test_actuals=True)
df = data["df"]
time_col = "some_time_col"
value_col = "some_value_col"
df.rename({
cst.TIME_COL: time_col,
cst.VALUE_COL: value_col
},
axis=1,
inplace=True)
# prepares params and calls template
metric = EvaluationMetricEnum.MeanAbsoluteError
# anomaly adjustment adds 10.0 to every record
adjustment_size = 10.0
anomaly_df = pd.DataFrame({
cst.START_DATE_COL: [df[time_col].min()],
cst.END_DATE_COL: [df[time_col].max()],
cst.ADJUSTMENT_DELTA_COL: [adjustment_size],
cst.METRIC_COL: [value_col]
})
anomaly_info = {
"value_col": cst.VALUE_COL,
"anomaly_df": anomaly_df,
"start_date_col": cst.START_DATE_COL,
"end_date_col": cst.END_DATE_COL,
"adjustment_delta_col": cst.ADJUSTMENT_DELTA_COL,
"filter_by_dict": {
cst.METRIC_COL: cst.VALUE_COL
},
"adjustment_method": "add"
}
metadata = MetadataParam(
time_col=time_col,
value_col=value_col,
freq="H",
date_format="%Y-%m-%d-%H",
train_end_date=datetime.datetime(2019, 7, 1),
anomaly_info=anomaly_info,
)
evaluation_metric = EvaluationMetricParam(
cv_selection_metric=metric.name,
cv_report_metrics=[
EvaluationMetricEnum.MedianAbsolutePercentError.name
],
agg_periods=24,
agg_func=np.max,
null_model_params={
"strategy": "quantile",
"constant": None,
"quantile": 0.8
},
relative_error_tolerance=0.01)
evaluation_period = EvaluationPeriodParam(test_horizon=1,
periods_between_train_test=2,
cv_horizon=3,
cv_min_train_periods=4,
cv_expanding_window=True,
cv_periods_between_splits=5,
cv_periods_between_train_test=6,
cv_max_splits=7)
model_components = ModelComponentsParam(
seasonality={
"yearly_seasonality": [True],
"weekly_seasonality": [False],
"daily_seasonality": [4],
"add_seasonality_dict": [{
"yearly": {
"period": 365.25,
"fourier_order": 20,
"prior_scale": 20.0
},
"quarterly": {
"period": 365.25 / 4,
"fourier_order": 15
},
"weekly": {
"period": 7,
"fourier_order": 35,
"prior_scale": 30.0
}
}]
},
growth={"growth_term": "linear"},
events={
"holiday_lookup_countries":
["UnitedStates", "UnitedKingdom", "India"],
"holiday_pre_num_days": [2],
"holiday_post_num_days": [3],
"holidays_prior_scale": [5.0]
},
regressors={
"add_regressor_dict": [{
"regressor1": {
"prior_scale": 10.0,
"mode": 'additive'
},
"regressor2": {
"prior_scale": 20.0,
"mode": 'multiplicative'
},
}]
},
changepoints={
"changepoint_prior_scale": [0.05],
"changepoints": [None],
"n_changepoints": [50],
"changepoint_range": [0.9]
},
uncertainty={
"mcmc_samples": [500],
"uncertainty_samples": [2000]
},
hyperparameter_override={
"input__response__null__impute_algorithm":
"ts_interpolate",
"input__response__null__impute_params": {
"orders": [7, 14]
},
"input__regressors_numeric__normalize__normalize_algorithm":
"RobustScaler",
})
computation = ComputationParam(hyperparameter_budget=10,
n_jobs=None,
verbose=1)
forecast_horizon = 20
coverage = 0.7
config = ForecastConfig(model_template=ModelTemplateEnum.PROPHET.name,
metadata_param=metadata,
forecast_horizon=forecast_horizon,
coverage=coverage,
evaluation_metric_param=evaluation_metric,
evaluation_period_param=evaluation_period,
model_components_param=model_components,
computation_param=computation)
template = ProphetTemplate()
params = template.apply_template_for_pipeline_params(df=df, config=config)
pipeline = params.pop("pipeline", None)
# Adding start_year and end_year based on the input df
model_components.events["start_year"] = df[time_col].min().year
model_components.events["end_year"] = df[time_col].max().year
expected_params = dict(
df=df,
time_col=time_col,
value_col=value_col,
date_format=metadata.date_format,
freq=metadata.freq,
train_end_date=metadata.train_end_date,
anomaly_info=metadata.anomaly_info,
# model
regressor_cols=template.regressor_cols,
estimator=None,
hyperparameter_grid=template.hyperparameter_grid,
hyperparameter_budget=computation.hyperparameter_budget,
n_jobs=computation.n_jobs,
verbose=computation.verbose,
# forecast
forecast_horizon=forecast_horizon,
coverage=coverage,
test_horizon=evaluation_period.test_horizon,
periods_between_train_test=evaluation_period.
periods_between_train_test,
agg_periods=evaluation_metric.agg_periods,
agg_func=evaluation_metric.agg_func,
# evaluation
score_func=metric.name,
score_func_greater_is_better=metric.get_metric_greater_is_better(),
cv_report_metrics=evaluation_metric.cv_report_metrics,
null_model_params=evaluation_metric.null_model_params,
relative_error_tolerance=evaluation_metric.relative_error_tolerance,
# CV
cv_horizon=evaluation_period.cv_horizon,
cv_min_train_periods=evaluation_period.cv_min_train_periods,
cv_expanding_window=evaluation_period.cv_expanding_window,
cv_periods_between_splits=evaluation_period.cv_periods_between_splits,
cv_periods_between_train_test=evaluation_period.
cv_periods_between_train_test,
cv_max_splits=evaluation_period.cv_max_splits)
assert_basic_pipeline_equal(pipeline, template.pipeline)
assert_equal(params, expected_params)
def test_run_prophet_template_custom():
"""Tests running prophet template through the pipeline"""
data = generate_df_with_reg_for_tests(freq="D",
periods=50,
train_frac=0.8,
conti_year_origin=2018,
remove_extra_cols=True,
mask_test_actuals=True)
# select relevant columns for testing
relevant_cols = [
cst.TIME_COL, cst.VALUE_COL, "regressor1", "regressor2", "regressor3"
]
df = data["df"][relevant_cols]
forecast_horizon = data["fut_time_num"]
# Model components - custom holidays; other params as defaults
model_components = ModelComponentsParam(
seasonality={
"seasonality_mode": ["additive"],
"yearly_seasonality": ["auto"],
"weekly_seasonality": [True],
"daily_seasonality": ["auto"],
},
growth={"growth_term": ["linear"]},
events={
"holiday_pre_num_days": [1],
"holiday_post_num_days": [1],
"holidays_prior_scale": [1.0]
},
changepoints={
"changepoint_prior_scale": [0.05],
"n_changepoints": [1],
"changepoint_range": [0.5],
},
regressors={
"add_regressor_dict": [{
"regressor1": {
"prior_scale": 10,
"standardize": True,
"mode": "additive"
},
"regressor2": {
"prior_scale": 15,
"standardize": False,
"mode": "additive"
},
"regressor3": {}
}]
},
uncertainty={"uncertainty_samples": [10]})
metadata = MetadataParam(
time_col=cst.TIME_COL,
value_col=cst.VALUE_COL,
freq="D",
)
evaluation_period = EvaluationPeriodParam(
test_horizon=5, # speeds up test case
periods_between_train_test=5,
cv_horizon=0, # speeds up test case
)
config = ForecastConfig(
model_template=ModelTemplateEnum.PROPHET.name,
metadata_param=metadata,
forecast_horizon=forecast_horizon,
coverage=0.95,
model_components_param=model_components,
evaluation_period_param=evaluation_period,
)
result = Forecaster().run_forecast_config(
df=df,
config=config,
)
forecast_df = result.forecast.df_test.reset_index(drop=True)
expected_cols = [
"ts", "actual", "forecast", "forecast_lower", "forecast_upper"
]
assert list(forecast_df.columns) == expected_cols
assert result.backtest.coverage == 0.95, "coverage is not correct"
# NB: coverage is poor because of very small dataset size and low uncertainty_samples
assert result.backtest.train_evaluation[cst.PREDICTION_BAND_COVERAGE] == pytest.approx(0.677, rel=1e-3), \
"training coverage is None or less than expected"
assert result.backtest.test_evaluation[cst.PREDICTION_BAND_COVERAGE] == pytest.approx(0.800, rel=1e-3), \
"testing coverage is None or less than expected"
assert result.backtest.train_evaluation["MSE"] == pytest.approx(3.7849, rel=1e-3), \
"training MSE is None or more than expected"
assert result.backtest.test_evaluation["MSE"] == pytest.approx(2.9609, rel=1e-3), \
"testing MSE is None or more than expected"
assert result.forecast.train_evaluation[cst.PREDICTION_BAND_COVERAGE] == pytest.approx(0.7805, rel=1e-3), \
"forecast coverage is None or less than expected"
assert result.forecast.train_evaluation["MSE"] == pytest.approx(4.1806, rel=1e-3), \
"forecast MSE is None or more than expected"
# ensure regressors were used in the model
prophet_estimator = result.model.steps[-1][-1]
regressors = prophet_estimator.model.extra_regressors
assert regressors.keys() == {"regressor1", "regressor2", "regressor3"}
assert regressors["regressor1"]["prior_scale"] == 10.0
assert regressors["regressor1"]["standardize"] is True
assert regressors["regressor1"]["mode"] == "additive"
assert regressors["regressor2"]["prior_scale"] == 15.0
assert regressors["regressor3"]["standardize"] == "auto"
def test_silverkite_template_custom(model_components_param):
""""Tests simple_silverkite_template with custom parameters,
and data that has regressors"""
data = generate_df_with_reg_for_tests(
freq="H",
periods=300*24,
remove_extra_cols=True,
mask_test_actuals=True)
df = data["df"]
time_col = "some_time_col"
value_col = "some_value_col"
df.rename({
TIME_COL: time_col,
VALUE_COL: value_col
}, axis=1, inplace=True)
metric = EvaluationMetricEnum.MeanAbsoluteError
# anomaly adjustment adds 10.0 to every record
adjustment_size = 10.0
anomaly_df = pd.DataFrame({
START_DATE_COL: [df[time_col].min()],
END_DATE_COL: [df[time_col].max()],
ADJUSTMENT_DELTA_COL: [adjustment_size],
METRIC_COL: [value_col]
})
anomaly_info = {
"value_col": VALUE_COL,
"anomaly_df": anomaly_df,
"start_date_col": START_DATE_COL,
"end_date_col": END_DATE_COL,
"adjustment_delta_col": ADJUSTMENT_DELTA_COL,
"filter_by_dict": {METRIC_COL: VALUE_COL},
"adjustment_method": "add"
}
metadata = MetadataParam(
time_col=time_col,
value_col=value_col,
freq="H",
date_format="%Y-%m-%d-%H",
train_end_date=datetime.datetime(2019, 7, 1),
anomaly_info=anomaly_info
)
evaluation_metric = EvaluationMetricParam(
cv_selection_metric=metric.name,
cv_report_metrics=[EvaluationMetricEnum.MedianAbsolutePercentError.name],
agg_periods=24,
agg_func=np.max,
null_model_params={
"strategy": "quantile",
"constant": None,
"quantile": 0.8
},
relative_error_tolerance=0.01
)
evaluation_period = EvaluationPeriodParam(
test_horizon=1,
periods_between_train_test=2,
cv_horizon=3,
cv_min_train_periods=4,
cv_expanding_window=True,
cv_periods_between_splits=5,
cv_periods_between_train_test=6,
cv_max_splits=7
)
computation = ComputationParam(
hyperparameter_budget=10,
n_jobs=None,
verbose=1
)
forecast_horizon = 20
coverage = 0.7
template = SilverkiteTemplate()
params = template.apply_template_for_pipeline_params(
df=df,
config=ForecastConfig(
model_template=ModelTemplateEnum.SK.name,
metadata_param=metadata,
forecast_horizon=forecast_horizon,
coverage=coverage,
evaluation_metric_param=evaluation_metric,
evaluation_period_param=evaluation_period,
model_components_param=model_components_param,
computation_param=computation
)
)
pipeline = params.pop("pipeline", None)
expected_params = dict(
df=df,
time_col=time_col,
value_col=value_col,
date_format=metadata.date_format,
freq=metadata.freq,
train_end_date=metadata.train_end_date,
anomaly_info=metadata.anomaly_info,
# model
regressor_cols=template.regressor_cols,
estimator=None,
hyperparameter_grid=template.hyperparameter_grid,
hyperparameter_budget=computation.hyperparameter_budget,
n_jobs=computation.n_jobs,
verbose=computation.verbose,
# forecast
forecast_horizon=forecast_horizon,
coverage=coverage,
test_horizon=evaluation_period.test_horizon,
periods_between_train_test=evaluation_period.periods_between_train_test,
agg_periods=evaluation_metric.agg_periods,
agg_func=evaluation_metric.agg_func,
relative_error_tolerance=evaluation_metric.relative_error_tolerance,
# evaluation
score_func=metric.name,
score_func_greater_is_better=metric.get_metric_greater_is_better(),
cv_report_metrics=evaluation_metric.cv_report_metrics,
null_model_params=evaluation_metric.null_model_params,
# CV
cv_horizon=evaluation_period.cv_horizon,
cv_min_train_periods=evaluation_period.cv_min_train_periods,
cv_expanding_window=evaluation_period.cv_expanding_window,
cv_periods_between_splits=evaluation_period.cv_periods_between_splits,
cv_periods_between_train_test=evaluation_period.cv_periods_between_train_test,
cv_max_splits=evaluation_period.cv_max_splits
)
assert_basic_pipeline_equal(pipeline, template.pipeline)
assert_equal(params, expected_params)
# The forecast model's performance will be the average of the three evaluations
# on the forecasts.
#
# By default, the results returned by :py:meth:`~greykite.framework.templates.forecaster.Forecaster.run_forecast_config`
# also runs time series cross-validation internally.
# You are allowed to configure the cross-validation splits, as shown below.
# Here note that the ``test_horizon`` are reserved from the back of
# the data and not used for cross-validation.
# This part of testing data can further evaluate the model performance
# besides the cross-validation result, and is available for plotting.
# Defines the cross-validation config
evaluation_period = EvaluationPeriodParam(
test_horizon=365, # leaves 365 days as testing data
cv_horizon=365, # each cv test size is 365 days (same as forecast horizon)
cv_max_splits=3, # 3 folds cv
cv_min_train_periods=365 *
4 # uses at least 4 years for training because we have 8 years data
)
# Runs the forecast
result = forecaster.run_forecast_config(
df=df,
config=ForecastConfig(
model_template=ModelTemplateEnum.SILVERKITE.name,
forecast_horizon=365, # forecasts 365 steps ahead
coverage=0.95, # 95% prediction intervals
metadata_param=metadata,
evaluation_period_param=evaluation_period))
# Summarizes the cv result
def valid_configs():
metadata = MetadataParam(time_col=TIME_COL, value_col=VALUE_COL, freq="D")
computation = ComputationParam(hyperparameter_budget=10,
n_jobs=None,
verbose=1)
forecast_horizon = 2 * 7
coverage = 0.90
evaluation_metric = EvaluationMetricParam(
cv_selection_metric=EvaluationMetricEnum.MeanAbsoluteError.name,
cv_report_metrics=None,
agg_periods=7,
agg_func=np.mean,
null_model_params=None)
evaluation_period = EvaluationPeriodParam(test_horizon=2 * 7,
periods_between_train_test=2 * 7,
cv_horizon=1 * 7,
cv_min_train_periods=8 * 7,
cv_expanding_window=True,
cv_periods_between_splits=7,
cv_periods_between_train_test=3 *
7,
cv_max_splits=2)
silverkite_components = ModelComponentsParam(
seasonality={
"yearly_seasonality": False,
"weekly_seasonality": True
},
growth={"growth_term": "quadratic"},
events={
"holidays_to_model_separately":
SilverkiteHoliday.ALL_HOLIDAYS_IN_COUNTRIES,
"holiday_lookup_countries": ["UnitedStates"],
"holiday_pre_num_days": 3,
},
changepoints={
"changepoints_dict": {
"method": "uniform",
"n_changepoints": 20,
}
},
regressors={
"regressor_cols": ["regressor1", "regressor2", "regressor3"]
},
uncertainty={
"uncertainty_dict": "auto",
},
hyperparameter_override={"input__response__null__max_frac": 0.1},
custom={
"fit_algorithm_dict": {
"fit_algorithm": "ridge",
"fit_algorithm_params": {
"normalize": True
},
},
"feature_sets_enabled": False
})
prophet_components = ModelComponentsParam(
seasonality={
"seasonality_mode": ["additive"],
"yearly_seasonality": ["auto"],
"weekly_seasonality": [True],
"daily_seasonality": ["auto"],
},
growth={"growth_term": ["linear"]},
events={
"holiday_pre_num_days": [1],
"holiday_post_num_days": [1],
"holidays_prior_scale": [1.0]
},
changepoints={
"changepoint_prior_scale": [0.05],
"n_changepoints": [1],
"changepoint_range": [0.5],
},
regressors={
"add_regressor_dict": [{
"regressor1": {
"prior_scale": 10,
"standardize": True,
"mode": 'additive'
},
"regressor2": {
"prior_scale": 15,
"standardize": False,
"mode": 'additive'
},
"regressor3": {}
}]
},
uncertainty={"uncertainty_samples": [10]})
valid_prophet = ForecastConfig(
model_template=ModelTemplateEnum.PROPHET.name,
metadata_param=metadata,
computation_param=computation,
coverage=coverage,
evaluation_metric_param=evaluation_metric,
evaluation_period_param=evaluation_period,
forecast_horizon=forecast_horizon,
model_components_param=prophet_components)
valid_silverkite = ForecastConfig(
model_template=ModelTemplateEnum.SILVERKITE.name,
metadata_param=metadata,
computation_param=computation,
coverage=coverage,
evaluation_metric_param=evaluation_metric,
evaluation_period_param=evaluation_period,
forecast_horizon=forecast_horizon,
model_components_param=silverkite_components)
configs = {
"valid_prophet": valid_prophet,
"valid_silverkite": valid_silverkite
}
return configs
def test_run_forecast_config_with_single_simple_silverkite_template():
# The generic name of single simple silverkite templates are not added to `ModelTemplateEnum`,
# therefore we test if these are recognized.
data = generate_df_for_tests(freq="D", periods=365)
df = data["df"]
metric = EvaluationMetricEnum.MeanAbsoluteError
evaluation_metric = EvaluationMetricParam(cv_selection_metric=metric.name,
agg_periods=7,
agg_func=np.max,
null_model_params={
"strategy": "quantile",
"constant": None,
"quantile": 0.5
})
evaluation_period = EvaluationPeriodParam(test_horizon=10,
periods_between_train_test=5,
cv_horizon=4,
cv_min_train_periods=80,
cv_expanding_window=False,
cv_periods_between_splits=20,
cv_periods_between_train_test=3,
cv_max_splits=2)
model_components = ModelComponentsParam(
hyperparameter_override=[{
"estimator__yearly_seasonality": 1
}, {
"estimator__yearly_seasonality": 2
}])
computation = ComputationParam(verbose=2)
forecast_horizon = 27
coverage = 0.90
single_template_class = SimpleSilverkiteTemplateOptions(
freq=SILVERKITE_COMPONENT_KEYWORDS.FREQ.value.DAILY,
seas=SILVERKITE_COMPONENT_KEYWORDS.SEAS.value.NONE)
forecast_config = ForecastConfig(model_template=[
single_template_class, "DAILY_ALGO_SGD", "SILVERKITE_DAILY_90"
],
computation_param=computation,
coverage=coverage,
evaluation_metric_param=evaluation_metric,
evaluation_period_param=evaluation_period,
forecast_horizon=forecast_horizon,
model_components_param=model_components)
forecaster = Forecaster()
result = forecaster.run_forecast_config(df=df, config=forecast_config)
summary = summarize_grid_search_results(result.grid_search)
# single_template_class is 1 template,
# "DAILY_ALGO_SGD" is 1 template and "SILVERKITE_DAILY_90" has 4 templates.
# With 2 items in `hyperparameter_override, there should be a total of 12 cases.
assert summary.shape[0] == 12
# Tests functionality for single template class only.
forecast_config = ForecastConfig(model_template=single_template_class,
computation_param=computation,
coverage=coverage,
evaluation_metric_param=evaluation_metric,
evaluation_period_param=evaluation_period,
forecast_horizon=forecast_horizon)
forecaster = Forecaster()
pipeline_parameters = forecaster.apply_forecast_config(
df=df, config=forecast_config)
assert_equal(actual=pipeline_parameters["hyperparameter_grid"],
expected={
"estimator__time_properties": [None],
"estimator__origin_for_time_vars": [None],
"estimator__train_test_thresh": [None],
"estimator__training_fraction": [None],
"estimator__fit_algorithm_dict": [{
"fit_algorithm":
"linear",
"fit_algorithm_params":
None
}],
"estimator__holidays_to_model_separately": [[]],
"estimator__holiday_lookup_countries": [[]],
"estimator__holiday_pre_num_days": [0],
"estimator__holiday_post_num_days": [0],
"estimator__holiday_pre_post_num_dict": [None],
"estimator__daily_event_df_dict": [None],
"estimator__changepoints_dict": [None],
"estimator__seasonality_changepoints_dict": [None],
"estimator__yearly_seasonality": [0],
"estimator__quarterly_seasonality": [0],
"estimator__monthly_seasonality": [0],
"estimator__weekly_seasonality": [0],
"estimator__daily_seasonality": [0],
"estimator__max_daily_seas_interaction_order": [0],
"estimator__max_weekly_seas_interaction_order": [2],
"estimator__autoreg_dict": [None],
"estimator__min_admissible_value": [None],
"estimator__max_admissible_value": [None],
"estimator__uncertainty_dict": [None],
"estimator__growth_term": ["linear"],
"estimator__regressor_cols": [[]],
"estimator__feature_sets_enabled": [False],
"estimator__extra_pred_cols": [[]]
},
ignore_keys={"estimator__time_properties": None})
def test_run_forecast_config_custom():
"""Tests `run_forecast_config` on weekly data with custom config:
- numeric and categorical regressors
- coverage
- null model
"""
data = generate_df_with_reg_for_tests(freq="W-MON",
periods=140,
remove_extra_cols=True,
mask_test_actuals=True)
reg_cols = ["regressor1", "regressor2", "regressor_categ"]
keep_cols = [TIME_COL, VALUE_COL] + reg_cols
df = data["df"][keep_cols]
metric = EvaluationMetricEnum.MeanAbsoluteError
evaluation_metric = EvaluationMetricParam(cv_selection_metric=metric.name,
agg_periods=7,
agg_func=np.max,
null_model_params={
"strategy": "quantile",
"constant": None,
"quantile": 0.5
})
evaluation_period = EvaluationPeriodParam(test_horizon=10,
periods_between_train_test=5,
cv_horizon=4,
cv_min_train_periods=80,
cv_expanding_window=False,
cv_periods_between_splits=20,
cv_periods_between_train_test=3,
cv_max_splits=3)
model_components = ModelComponentsParam(
regressors={"regressor_cols": reg_cols},
custom={
"fit_algorithm_dict": {
"fit_algorithm": "ridge",
"fit_algorithm_params": {
"cv": 2
}
}
})
computation = ComputationParam(verbose=2)
forecast_horizon = 27
coverage = 0.90
forecast_config = ForecastConfig(
model_template=ModelTemplateEnum.SILVERKITE.name,
computation_param=computation,
coverage=coverage,
evaluation_metric_param=evaluation_metric,
evaluation_period_param=evaluation_period,
forecast_horizon=forecast_horizon,
model_components_param=model_components)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
forecaster = Forecaster()
result = forecaster.run_forecast_config(df=df, config=forecast_config)
mse = EvaluationMetricEnum.RootMeanSquaredError.get_metric_name()
q80 = EvaluationMetricEnum.Quantile80.get_metric_name()
assert result.backtest.test_evaluation[mse] == pytest.approx(2.976,
rel=1e-2)
assert result.backtest.test_evaluation[q80] == pytest.approx(1.360,
rel=1e-2)
assert result.forecast.train_evaluation[mse] == pytest.approx(2.224,
rel=1e-2)
assert result.forecast.train_evaluation[q80] == pytest.approx(0.941,
rel=1e-2)
check_forecast_pipeline_result(result,
coverage=coverage,
strategy=None,
score_func=metric.name,
greater_is_better=False)
with pytest.raises(KeyError, match="missing_regressor"):
model_components = ModelComponentsParam(
regressors={"regressor_cols": ["missing_regressor"]})
forecaster = Forecaster()
result = forecaster.run_forecast_config(
df=df,
config=ForecastConfig(
model_template=ModelTemplateEnum.SILVERKITE.name,
model_components_param=model_components))
check_forecast_pipeline_result(result,
coverage=None,
strategy=None,
score_func=metric.get_metric_func(),
greater_is_better=False)
def test_run_auto_arima_template_custom():
"""Tests running auto arima template through the pipeline"""
data = generate_df_with_reg_for_tests(freq="D",
periods=50,
train_frac=0.8,
conti_year_origin=2018,
remove_extra_cols=True,
mask_test_actuals=True)
# select relevant columns for testing
relevant_cols = [
cst.TIME_COL, cst.VALUE_COL, "regressor1", "regressor2", "regressor3"
]
df = data["df"][relevant_cols]
forecast_horizon = data["fut_time_num"]
# Model components - custom holidays; other params as defaults
model_components = ModelComponentsParam(
# Everything except `custom` and `hyperparameter_override` are ignored
seasonality={
"seasonality_mode": ["additive"],
"yearly_seasonality": ["auto"],
"weekly_seasonality": [True],
"daily_seasonality": ["auto"],
},
growth={"growth_term": ["linear"]},
events={
"holiday_pre_num_days": [1],
"holiday_post_num_days": [1],
"holidays_prior_scale": [1.0]
},
changepoints={
"changepoint_prior_scale": [0.05],
"n_changepoints": [1],
"changepoint_range": [0.5],
},
regressors={
"add_regressor_dict": [{
"regressor1": {
"prior_scale": 10,
"standardize": True,
"mode": "additive"
},
"regressor2": {
"prior_scale": 15,
"standardize": False,
"mode": "additive"
},
"regressor3": {}
}]
},
uncertainty={"uncertainty_samples": [10]},
custom={
"max_order": [10],
"information_criterion": ["bic"]
})
metadata = MetadataParam(
time_col=cst.TIME_COL,
value_col=cst.VALUE_COL,
freq="D",
)
evaluation_period = EvaluationPeriodParam(
test_horizon=5, # speeds up test case
periods_between_train_test=5,
cv_horizon=0, # speeds up test case
)
config = ForecastConfig(
model_template=ModelTemplateEnum.AUTO_ARIMA.name,
metadata_param=metadata,
forecast_horizon=forecast_horizon,
coverage=0.95,
model_components_param=model_components,
evaluation_period_param=evaluation_period,
)
result = Forecaster().run_forecast_config(
df=df,
config=config,
)
forecast_df = result.forecast.df_test.reset_index(drop=True)
expected_cols = [
"ts", "actual", "forecast", "forecast_lower", "forecast_upper"
]
assert list(forecast_df.columns) == expected_cols
assert result.backtest.coverage == 0.95, "coverage is not correct"
# NB: coverage is poor because of very small dataset size and low uncertainty_samples
assert result.backtest.train_evaluation[
cst.PREDICTION_BAND_COVERAGE] is not None
assert result.backtest.test_evaluation[
cst.PREDICTION_BAND_COVERAGE] is not None
assert result.backtest.train_evaluation["MSE"] is not None
assert result.backtest.test_evaluation["MSE"] is not None
assert result.forecast.train_evaluation[
cst.PREDICTION_BAND_COVERAGE] is not None
assert result.forecast.train_evaluation["MSE"] is not None
def benchmark_silverkite_template(
data_name,
df,
forecast_horizons,
fit_algorithms,
max_cvs,
metadata=None,
evaluation_metric=None):
"""Benchmarks silverkite template and returns the output as a list
:param data_name: str
Name of the dataset we are performing benchmarking on
For real datasets, the data_name matches the corresponding filename in the data/ folder
For simulated datasets, we follow the convention "<freq>_simulated" e.g. "daily_simulated"
:param df: pd.DataFrame
Dataframe containing the time and value columns
:param forecast_horizons: List[int]
One forecast is created for every given forecast_horizon
:param fit_algorithms: List[str]
Names of predictive models to fit.
Options are "linear", "lasso", "ridge", "rf" etc.
:param max_cvs: List[int] or None
Number of maximum CV folds to use.
:param metadata: :class:`~greykite.framework.templates.autogen.forecast_config.MetadataParam` or None, default None
Information about the input data. See
:class:`~greykite.framework.templates.autogen.forecast_config.MetadataParam`.
:param evaluation_metric: :class:`~greykite.framework.templates.autogen.forecast_config.EvaluationMetricParam` or None, default None
What metrics to evaluate. See
:class:`~greykite.framework.templates.autogen.forecast_config.EvaluationMetricParam`.
:return: .csv file
Each row of the .csv file records the following outputs from one run of the silverkite template:
- "data_name": Fixed string "<freq>_simulated", or name of the dataset in data/ folder
- "forecast_model_name": "silverkite_<fit_algorithm>" e.g. "silverkite_linear" or "prophet"
- "train_period": train_period
- "forecast_horizon": forecast_horizon
- "fit_algorithm": fit algorithm name
- "cv_folds": max_cv
- "runtime_sec": runtime in seconds
- "train_mae": Mean Absolute Error of training data in backtest
- "train_mape": Mean Absolute Percent Error of training data in backtest
- "test_mae": Mean Absolute Error of testing data in backtest
- "test_mape": Mean Absolute Percent Error of testing data in backtest
"""
benchmark_results = []
for forecast_horizon, fit_algorithm, max_cv in itertools.product(forecast_horizons, fit_algorithms, max_cvs):
model_components = ModelComponentsParam(
custom={
"fit_algorithm_dict": {
"fit_algorithm": fit_algorithm,
},
"feature_sets_enabled": True
}
)
evaluation_period = EvaluationPeriodParam(
cv_max_splits=max_cv
)
start_time = timeit.default_timer()
forecaster = Forecaster()
result = forecaster.run_forecast_config(
df=df,
config=ForecastConfig(
model_template=ModelTemplateEnum.SILVERKITE.name,
forecast_horizon=forecast_horizon,
metadata_param=metadata,
evaluation_metric_param=evaluation_metric,
model_components_param=model_components,
evaluation_period_param=evaluation_period,
)
)
runtime = timeit.default_timer() - start_time
output_dict = dict(
data_name=data_name,
forecast_model_name=f"silverkite_{fit_algorithm}",
train_period=df.shape[0],
forecast_horizon=forecast_horizon,
cv_folds=result.grid_search.n_splits_,
runtime_sec=round(runtime, 3),
train_mae=result.backtest.train_evaluation["MAE"].round(3),
train_mape=result.backtest.train_evaluation["MAPE"].round(3),
test_mae=result.backtest.test_evaluation["MAE"].round(3),
test_mape=result.backtest.test_evaluation["MAPE"].round(3)
)
benchmark_results.append(output_dict)
return benchmark_results
# Defines number of periods to forecast into the future
forecast_horizon = 7
# Specifies intended coverage of the prediction interval
coverage = 0.95
# Defines the metrics to evaluate the forecasts
# We use Mean Absolute Percent Error (MAPE) in this tutorial
evaluation_metric = EvaluationMetricParam(
cv_selection_metric=EvaluationMetricEnum.MeanAbsolutePercentError.name,
cv_report_metrics=None)
# Defines the cross-validation config within pipeline
evaluation_period = EvaluationPeriodParam(
cv_max_splits=
1, # Benchmarking n_splits is defined in tscv, here we don't need split to choose parameter sets
periods_between_train_test=0,
)
# Defines parameters related to grid-search computation
computation = ComputationParam(
hyperparameter_budget=None,
n_jobs=-1, # to debug, change to 1 for more informative error messages
verbose=3)
# Defines common components across all the configs
# ``model_template`` and ``model_components_param`` changes between configs
common_config = ForecastConfig(
metadata_param=metadata,
forecast_horizon=forecast_horizon,
coverage=coverage,