def test_check_cv():
cv = SlidingWindowForecastCV(h=12)
assert check_cv(cv) is cv
assert isinstance(check_cv(None), RollingForecastCV)
with pytest.raises(TypeError):
check_cv('something else')
def test_individual_model_cross_validate():
metrics = ["smape", "mean_squared_error", "mean_absolute_error"]
train = data_generator.generate_test_data(1, 1, 765, "2019-01-01")
model = AutoARIMA(max_order=5, out_of_sample_size=30).fit(train.df["y"])
cross_validator = SlidingWindowForecastCV(window_size=180, step=120, h=90)
cv_results = _cross_validate_single_model(
model,
train.df["y"],
metrics=metrics,
cross_validator=cross_validator,
error_score=np.nan,
exog=None,
verbosity=3,
)
expected_fields = [f"{met}_mean" for met in metrics] + [
f"{met}_stddev" for met in metrics
]
for key, value in cv_results.items():
assert key in expected_fields
assert value > 0
if "_stddev" in key:
assert value < cv_results.get(key.split("_stddev")[0] + "_mean") * 10.0
def test_issue_364_bad_splits():
endog = y[:100]
cv = SlidingWindowForecastCV(window_size=90, step=1, h=4)
gen = cv.split(endog)
expected = [
(np.arange(0, 90), np.array([90, 91, 92, 93])),
(np.arange(1, 91), np.array([91, 92, 93, 94])),
(np.arange(2, 92), np.array([92, 93, 94, 95])),
(np.arange(3, 93), np.array([93, 94, 95, 96])),
(np.arange(4, 94), np.array([94, 95, 96, 97])),
(np.arange(5, 95), np.array([95, 96, 97, 98])),
(np.arange(6, 96), np.array([96, 97, 98, 99])),
]
# should be 7
for i, (train, test) in enumerate(gen):
assert_array_equal(train, expected[i][0])
assert_array_equal(test, expected[i][1])
# assert no extra splits
with pytest.raises(StopIteration):
next(gen)
silence_warnings=True,
)
# Save to local directory
save_dir = "/tmp/group_pmdarima/arima.gpmd"
base_arima.save(save_dir)
# Load from saved model
loaded_model = GroupedPmdarima.load(save_dir)
print("\nARIMA results:\n", "-" * 40)
get_and_print_model_metrics_params(loaded_model)
prediction = loaded_model.predict(
n_periods=30, alpha=0.02, predict_col="forecast", return_conf_int=True
)
print("\nPredictions:\n", "-" * 40)
print(prediction.to_string())
print("\nCross validation metric results:\n", "-" * 40)
cross_validator = SlidingWindowForecastCV(h=90, step=365, window_size=730)
cv_results = loaded_model.cross_validate(
df=training_data,
metrics=["mean_squared_error", "smape", "mean_absolute_error"],
cross_validator=cross_validator,
error_score=np.nan,
verbosity=4,
)
print(cv_results.to_string())
def test_bad_window_size():
cv = SlidingWindowForecastCV(window_size=2, step=1, h=4)
with pytest.raises(ValueError) as ve:
list(cv.split(y))
assert "> 2" in pytest_error_str(ve)
def test_rolling_forecast_cv_passing(cv):
# get all splits
splits = list(cv.split(y))
last_train_step = None
for train, test in splits:
assert test.shape[0] == cv.h
assert test[-1] == train[-1] + cv.h
if last_train_step is not None:
assert train[-1] == last_train_step + cv.step
last_train_step = train[-1]
@pytest.mark.parametrize(
'cv', [
SlidingWindowForecastCV(),
SlidingWindowForecastCV(h=4),
SlidingWindowForecastCV(window_size=42, h=10),
SlidingWindowForecastCV(window_size=67, h=16, step=7),
]
)
def test_sliding_forecast_cv_passing(cv):
# get all splits
splits = list(cv.split(y))
last_train_step = None
last_window_size = None
for train, test in splits:
assert test.shape[0] == cv.h
assert test[-1] == train[-1] + cv.h
if last_train_step is not None:
datetime_col="ds",
ndiffs=ndiff,
silence_warnings=True,
)
# Save to local directory
save_dir = "/tmp/group_pmdarima/pipeline_override.gpmd"
grouped_model.save(save_dir)
# Load from saved model
loaded_model = GroupedPmdarima.load(save_dir)
print("\nAutoARIMA results:\n", "-" * 40)
get_and_print_model_metrics_params(loaded_model)
print("\nPredictions:\n", "-" * 40)
prediction = loaded_model.predict(
n_periods=30, alpha=0.1, predict_col="forecasted_values", return_conf_int=True
)
print(prediction.to_string())
cv_evaluator = SlidingWindowForecastCV(h=90, step=120, window_size=180)
cross_validation = loaded_model.cross_validate(
df=training_data,
metrics=["smape", "mean_squared_error", "mean_absolute_error"],
cross_validator=cv_evaluator,
)
print("\nCross validation metrics:\n", "-" * 40)
print(cross_validation.to_string())
y_col="y",
datetime_col="ds",
silence_warnings=True,
)
# Save to local directory
save_dir = "/tmp/group_pmdarima/autoarima.gpmd"
base_auto_arima.save(save_dir)
# Load from saved model
loaded_model = GroupedPmdarima.load(save_dir)
print("\nAutoARIMA results:\n", "-" * 40)
get_and_print_model_metrics_params(loaded_model)
print("\nPredictions:\n", "-" * 40)
prediction = loaded_model.predict(n_periods=30, alpha=0.1, return_conf_int=True)
print(prediction.to_string())
print("\nCross validation metric results:\n", "-" * 40)
cross_validator = SlidingWindowForecastCV(h=30, step=180, window_size=365)
cv_results = loaded_model.cross_validate(
df=training_data,
metrics=["mean_squared_error", "smape", "mean_absolute_error"],
cross_validator=cross_validator,
error_score=np.nan,
verbosity=3,
)
print(cv_results.to_string())
