def test_backtest_forecasting(self):
linear_series = lt(length=50)
linear_series_multi = linear_series.stack(linear_series)
# univariate model + univariate series
pred = NaiveDrift().backtest(linear_series, None,
pd.Timestamp('20000201'), 3)
self.assertEqual(r2_score(pred, linear_series), 1.0)
with self.assertRaises(ValueError):
NaiveDrift().backtest(linear_series,
None,
start=pd.Timestamp('20000217'),
forecast_horizon=3)
with self.assertRaises(ValueError):
NaiveDrift().backtest(linear_series,
None,
start=pd.Timestamp('20000217'),
forecast_horizon=3,
trim_to_series=True)
NaiveDrift().backtest(linear_series,
None,
start=pd.Timestamp('20000216'),
forecast_horizon=3)
NaiveDrift().backtest(linear_series,
None,
pd.Timestamp('20000217'),
forecast_horizon=3,
trim_to_series=False)
# Using forecast_horizon default value
NaiveDrift().backtest(linear_series,
None,
start=pd.Timestamp('20000216'))
NaiveDrift().backtest(linear_series,
None,
pd.Timestamp('20000217'),
trim_to_series=False)
# Using an int or float value for start
NaiveDrift().backtest(linear_series, None, start=30)
NaiveDrift().backtest(linear_series,
None,
start=0.7,
trim_to_series=False)
# Using invalid start and/or forecast_horizon values
with self.assertRaises(ValueError):
NaiveDrift().backtest(linear_series,
None,
start=0.7,
forecast_horizon=-1)
with self.assertRaises(ValueError):
NaiveDrift().backtest(linear_series, None, 0.7, -1)
with self.assertRaises(ValueError):
NaiveDrift().backtest(linear_series, None, start=100)
with self.assertRaises(ValueError):
NaiveDrift().backtest(linear_series, None, start=1.2)
with self.assertRaises(TypeError):
NaiveDrift().backtest(linear_series, None, start='wrong type')
with self.assertRaises(ValueError):
NaiveDrift().backtest(linear_series,
None,
start=49,
forecast_horizon=2,
trim_to_series=True)
# univariate model + multivariate series
with self.assertRaises(AssertionError):
NaiveDrift().backtest(linear_series_multi, None,
pd.Timestamp('20000201'), 3)
# multivariate model + univariate series
if TORCH_AVAILABLE:
tcn_model = TCNModel(batch_size=1, n_epochs=1)
pred = tcn_model.backtest(linear_series,
None,
pd.Timestamp('20000125'),
3,
verbose=False)
self.assertEqual(pred.width, 1)
# multivariate model + multivariate series
with self.assertRaises(ValueError):
tcn_model.backtest(linear_series_multi,
None,
pd.Timestamp('20000125'),
3,
verbose=False)
tcn_model = TCNModel(batch_size=1,
n_epochs=1,
input_size=2,
output_length=3)
with self.assertRaises(ValueError):
tcn_model.backtest(linear_series_multi,
None,
pd.Timestamp('20000125'),
3,
verbose=False,
use_full_output_length=False)
pred = tcn_model.backtest(linear_series_multi,
linear_series_multi[['0']],
pd.Timestamp('20000125'),
1,
verbose=False,
use_full_output_length=True)
self.assertEqual(pred.width, 1)
pred = tcn_model.backtest(linear_series_multi,
linear_series_multi[['1']],
pd.Timestamp('20000125'),
3,
verbose=False,
use_full_output_length=True)
self.assertEqual(pred.width, 1)
tcn_model = TCNModel(batch_size=1,
n_epochs=1,
input_size=2,
output_length=3,
output_size=2)
pred = tcn_model.backtest(linear_series_multi,
linear_series_multi,
pd.Timestamp('20000125'),
3,
verbose=False,
use_full_output_length=True)
self.assertEqual(pred.width, 2)
def test_backtest_regression(self):
gaussian_series = gt(mean=2, length=50)
sine_series = st(length=50)
features = [gaussian_series + sine_series, gaussian_series]
features_multivariate = [
(gaussian_series + sine_series).stack(gaussian_series),
gaussian_series
]
target = st(length=50)
# univariate feature test
score = StandardRegressionModel(15).backtest(features,
target,
pd.Timestamp('20000201'),
3,
metric=r2_score)
self.assertEqual(score, 1.0)
# Using an int or float value for start
score = StandardRegressionModel(15).backtest(features,
target,
start=30,
forecast_horizon=3,
metric=r2_score)
self.assertEqual(score, 1.0)
score = StandardRegressionModel(15).backtest(features,
target,
start=0.5,
forecast_horizon=3,
metric=r2_score)
self.assertEqual(score, 1.0)
# Using a too small start value
with self.assertRaises(ValueError):
StandardRegressionModel(15).backtest(features,
target,
start=0,
forecast_horizon=3)
with self.assertRaises(ValueError):
StandardRegressionModel(15).backtest(features,
target,
start=0.01,
forecast_horizon=3)
# Using StandardRegressionModel's start default value
score = StandardRegressionModel(15).backtest(features,
target,
forecast_horizon=3,
metric=r2_score)
self.assertEqual(score, 1.0)
# multivariate feature test
score = StandardRegressionModel(15).backtest(features_multivariate,
target,
pd.Timestamp('20000201'),
forecast_horizon=3,
metric=r2_score)
self.assertEqual(score, 1.0)
# multivariate target
score = StandardRegressionModel(15).backtest(features_multivariate,
target.stack(target),
pd.Timestamp('20000201'),
forecast_horizon=3,
metric=r2_score)
self.assertEqual(score, 1.0)
# multivariate target with stride
hist = StandardRegressionModel(15).historical_forecasts(
features_multivariate,
target.stack(target),
pd.Timestamp('20000201'),
forecast_horizon=3,
stride=3,
last_points_only=True)
self.assertEqual(r2_score(target.stack(target), hist), 1.0)
self.assertEqual((hist.time_index()[1] - hist.time_index()[0]).days, 3)
def test_backtest_regression(self):
gaussian_series = gt(mean=2, length=50)
sine_series = st(length=50)
features = [gaussian_series + sine_series, gaussian_series]
features_multivariate = [
(gaussian_series + sine_series).stack(gaussian_series),
gaussian_series
]
target = st(length=50)
# univariate feature test
pred = StandardRegressionModel(15).backtest(features, target,
pd.Timestamp('20000201'),
3)
self.assertEqual(r2_score(pred, target), 1.0)
# Using an int or float value for start
pred = StandardRegressionModel(15).backtest(features,
target,
start=30,
forecast_horizon=3)
self.assertEqual(r2_score(pred, target), 1.0)
pred = StandardRegressionModel(15).backtest(features,
target,
start=0.5,
forecast_horizon=3)
self.assertEqual(r2_score(pred, target), 1.0)
# Using a too small start value
with self.assertRaises(ValueError):
StandardRegressionModel(15).backtest(features,
target,
start=0,
forecast_horizon=3)
with self.assertRaises(ValueError):
StandardRegressionModel(15).backtest(features,
target,
start=0.01,
forecast_horizon=3)
# Using StandardRegressionModel's start default value
pred = StandardRegressionModel(15).backtest(features,
target,
forecast_horizon=3)
self.assertEqual(r2_score(pred, target), 1.0)
# multivariate feature test
pred = StandardRegressionModel(15).backtest(features_multivariate,
target,
pd.Timestamp('20000201'),
3)
self.assertEqual(r2_score(pred, target), 1.0)
# multivariate target
pred = StandardRegressionModel(15).backtest(features_multivariate,
target.stack(target),
pd.Timestamp('20000201'),
3)
self.assertEqual(r2_score(pred, target.stack(target)), 1.0)
# multivariate target with stride
pred = StandardRegressionModel(15).backtest(features_multivariate,
target.stack(target),
pd.Timestamp('20000201'),
3,
stride=3)
self.assertEqual(r2_score(pred, target.stack(target)), 1.0)
self.assertEqual((pred.time_index()[1] - pred.time_index()[0]).days, 3)