def test_multi_ts(self):
for model_cls, kwargs, err in models_cls_kwargs_errs:
model = model_cls(
input_chunk_length=IN_LEN,
output_chunk_length=OUT_LEN,
random_state=0,
**kwargs,
)
model.fit([self.ts_pass_train, self.ts_pass_train_1])
with self.assertRaises(ValueError):
# when model is fit from >1 series, one must provide a series in argument
model.predict(n=1)
pred = model.predict(n=36, series=self.ts_pass_train)
mape_err = mape(self.ts_pass_val, pred)
self.assertTrue(
mape_err < err,
"Model {} produces errors too high (several time "
"series). Error = {}".format(model_cls, mape_err),
)
# check prediction for several time series
pred_list = model.predict(
n=36, series=[self.ts_pass_train, self.ts_pass_train_1])
self.assertTrue(
len(pred_list) == 2,
f"Model {model_cls} did not return a list of prediction",
)
for pred in pred_list:
mape_err = mape(self.ts_pass_val, pred)
self.assertTrue(
mape_err < err,
"Model {} produces errors too high (several time series 2). "
"Error = {}".format(model_cls, mape_err),
)
def eval_model(model):
pred_series = model.predict(n=96)
plt.figure(figsize=(8, 5))
series_transformed.plot(label='actual')
pred_series.plot(label='forecast')
plt.title('MAPE: {:.2f}%'.format(mape(pred_series, val_transformed)))
plt.legend()
def compare_best_against_random(model_class, params, series, stride=1):
# instantiate best model in expanding window mode
np.random.seed(1)
best_model_1, _, _ = model_class.gridsearch(
params,
series,
forecast_horizon=10,
stride=stride,
metric=mape,
start=series.time_index[-21],
)
# instantiate best model in split mode
train, val = series.split_before(series.time_index[-10])
best_model_2, _, _ = model_class.gridsearch(
params, train, val_series=val, metric=mape
)
# intantiate model with random parameters from 'params'
random.seed(1)
random_param_choice = {}
for key in params.keys():
random_param_choice[key] = random.choice(params[key])
random_model = model_class(**random_param_choice)
# perform backtest forecasting on both models
best_score_1 = best_model_1.backtest(
series, start=series.time_index[-21], forecast_horizon=10
)
random_score_1 = random_model.backtest(
series, start=series.time_index[-21], forecast_horizon=10
)
# perform train/val evaluation on both models
best_model_2.fit(train)
best_score_2 = mape(best_model_2.predict(len(val)), series)
random_model = model_class(**random_param_choice)
random_model.fit(train)
random_score_2 = mape(random_model.predict(len(val)), series)
# check whether best models are at least as good as random models
expanding_window_ok = best_score_1 <= random_score_1
split_ok = best_score_2 <= random_score_2
return expanding_window_ok and split_ok
def test_future_covariates(self):
# models with future covariates should produce better predictions over a long forecasting horizon
# than a model trained with no covariates
model = TCNModel(
input_chunk_length=50,
output_chunk_length=5,
n_epochs=20,
random_state=0,
)
model.fit(series=self.target_past)
long_pred_no_cov = model.predict(n=160)
model = TCNModel(
input_chunk_length=50,
output_chunk_length=5,
n_epochs=20,
random_state=0,
)
model.fit(series=self.target_past,
past_covariates=self.covariates_past)
long_pred_with_cov = model.predict(n=160,
past_covariates=self.covariates)
self.assertTrue(
mape(self.target_future, long_pred_no_cov) > mape(
self.target_future, long_pred_with_cov),
"Models with future covariates should produce better predictions.",
)
# block models can predict up to self.output_chunk_length points beyond the last future covariate...
model.predict(n=165, past_covariates=self.covariates)
# ... not more
with self.assertRaises(ValueError):
model.predict(n=166, series=self.ts_pass_train)
# recurrent models can only predict data points for time steps where future covariates are available
model = RNNModel(12, n_epochs=1)
model.fit(series=self.target_past,
future_covariates=self.covariates_past)
model.predict(n=160, future_covariates=self.covariates)
with self.assertRaises(ValueError):
model.predict(n=161, future_covariates=self.covariates)
def test_multivariate_models_performance(self):
# for every model, check whether its errors do not exceed the given bounds
for model, max_mape in multivariate_models:
np.random.seed(1)
model.fit(self.ts_ice_heater_train)
prediction = model.predict(len(self.ts_ice_heater_val))
current_mape = mape(prediction, self.ts_ice_heater_val)
self.assertTrue(
current_mape < max_mape,
"{} model exceeded the maximum MAPE of {}. "
"with a MAPE of {}".format(str(model), max_mape, current_mape),
)
def test_single_ts(self):
for model_cls, kwargs, err in models_cls_kwargs_errs:
model = model_cls(
input_chunk_length=IN_LEN,
output_chunk_length=OUT_LEN,
random_state=0,
**kwargs,
)
model.fit(self.ts_pass_train)
pred = model.predict(n=36)
mape_err = mape(self.ts_pass_val, pred)
self.assertTrue(
mape_err < err,
"Model {} produces errors too high (one time "
"series). Error = {}".format(model_cls, mape_err),
)
self.assertTrue(
pred.static_covariates.equals(
self.ts_passengers.static_covariates))
def test_covariates(self):
for model_cls, kwargs, err in models_cls_kwargs_errs:
model = model_cls(
input_chunk_length=IN_LEN,
output_chunk_length=OUT_LEN,
random_state=0,
**kwargs,
)
# Here we rely on the fact that all non-Dual models currently are Past models
cov_name = ("future_covariates" if isinstance(
model, DualCovariatesTorchModel) else "past_covariates")
cov_kwargs = {
cov_name:
[self.time_covariates_train, self.time_covariates_train]
}
model.fit(series=[self.ts_pass_train, self.ts_pass_train_1],
**cov_kwargs)
with self.assertRaises(ValueError):
# when model is fit from >1 series, one must provide a series in argument
model.predict(n=1)
with self.assertRaises(ValueError):
# when model is fit using multiple covariates, covariates are required at prediction time
model.predict(n=1, series=self.ts_pass_train)
cov_kwargs_train = {cov_name: self.time_covariates_train}
cov_kwargs_notrain = {cov_name: self.time_covariates}
with self.assertRaises(ValueError):
# when model is fit using covariates, n cannot be greater than output_chunk_length...
model.predict(n=13,
series=self.ts_pass_train,
**cov_kwargs_train)
# ... unless future covariates are provided
pred = model.predict(n=13,
series=self.ts_pass_train,
**cov_kwargs_notrain)
pred = model.predict(n=12,
series=self.ts_pass_train,
**cov_kwargs_notrain)
mape_err = mape(self.ts_pass_val, pred)
self.assertTrue(
mape_err < err,
"Model {} produces errors too high (several time "
"series with covariates). Error = {}".format(
model_cls, mape_err),
)
# when model is fit using 1 training and 1 covariate series, time series args are optional
if model._is_probabilistic:
continue
model = model_cls(input_chunk_length=IN_LEN,
output_chunk_length=OUT_LEN,
**kwargs)
model.fit(series=self.ts_pass_train, **cov_kwargs_train)
pred1 = model.predict(1)
pred2 = model.predict(1, series=self.ts_pass_train)
pred3 = model.predict(1, **cov_kwargs_train)
pred4 = model.predict(1,
**cov_kwargs_train,
series=self.ts_pass_train)
self.assertEqual(pred1, pred2)
self.assertEqual(pred1, pred3)
self.assertEqual(pred1, pred4)
from darts.models import AutoARIMA
model_aarima = AutoARIMA()
model_aarima.fit(train)
prediction_aarima = model_aarima.predict(len(val))
series.plot(label='actual', lw=3)
prediction_aarima.plot(label='forecast', lw=3)
plt.legend()
plt.xlabel('Year')
"""Backtesting for comparing two models"""
from darts.backtesting import backtest_forecasting
from darts.models import Prophet
models = [ExponentialSmoothing(), Prophet()]
backtests = [backtest_forecasting(series,
model,
pd.Timestamp('19550101'),
fcast_horizon_n=3)
for model in models]
"""To compute error metrics — mean absolute percentage error(mape):"""
from darts.metrics import mape
series.plot(label='data')
for i, m in enumerate(models):
err = mape(backtests[i], series)
backtests[i].plot(lw=3, label='{}, MAPE={:.2f}%'.format(m, err))
plt.title('Backtests with 3-months forecast horizon')
plt.legend()
def eval_model(model):
pred_series = model.predict(n=96)
plt.figure(figsize=(8, 5))
series_transformed.plot(label='actual')
pred_series.plot(label='forecast')
plt.title('MAPE: {:.2f}%'.format(mape(pred_series, val_transformed)))
plt.legend()
eval_model(my_model)
best_model = RNNModel.load_from_checkpoint(model_name='Eq_RNN', best=True)
eval_model(best_model)
backtest_series = my_model.historical_forecasts(series_transformed,
start=pd.Timestamp('20021231'),
forecast_horizon=12,
retrain=False,
verbose=True)
plt.figure(figsize=(8, 5))
series_transformed.plot(label='actual')
backtest_series.plot(label='backtest')
plt.legend()
plt.title('Backtest, starting Jan 2003, 12-months horizon')
print('MAPE: {:.2f}%'.format(
mape(transformer.inverse_transform(series_transformed),
transformer.inverse_transform(backtest_series))))
ts_var = 'CPU_Used'
time_df = df.filter(['date', ts_var])
#convert variable from object dtype to numeric dtype
time_df[ts_var] = pd.to_numeric(time_df[ts_var], errors='coerce')
#remove duplicates
time_df.sort_values("date", inplace=True)
time_df.drop_duplicates(subset="date", keep=False, inplace=True)
#generate time series using darts
series = TimeSeries.from_dataframe(time_df, 'date', ts_var, freq='T')
#treat missing values
filler = MissingValuesFiller()
series = filler.transform(series)
#training and testing dataset
train, val = series.split_after(pd.Timestamp('2019-10-23 19:41:50'))
#FFT model
model = FFT(required_matches=set(), nr_freqs_to_keep=None)
model.fit(train)
pred_val = model.predict(len(val))
#Evaluation metrics
series.plot(label='actual')
pred_val.plot(label='forecast', lw=3)
plt.legend()
print("MAPE:", mape(pred_val, val))
ts_var = 'Memory_Used'
time_df = df.filter(['date', ts_var])
#convert variable from object dtype to numeric dtype
time_df[ts_var] = pd.to_numeric(time_df[ts_var], errors='coerce')
#generate time series using darts
series = TimeSeries.from_dataframe(time_df, 'date', ts_var, freq='S')
#treat missing values
filler = MissingValuesFiller()
series = filler.transform(series)
#scale the values
scaler = Scaler()
rescaled = scaler.fit_transform(series)
#training and testing dataset
train, val = rescaled.split_after(pd.Timestamp('2020-01-23 19:41:50'))
#Exponential smoothing model
model = ExponentialSmoothing()
model.fit(train)
prediction = model.predict(len(val))
#Evaluation metrics
rescaled.plot(label='actual')
prediction.plot(label='forecast', lw=3)
plt.legend()
print("MAPE:", mape(prediction, val))
