def test_multi_ts(self): for model_cls, kwargs, err in models_cls_kwargs_errs: model = model_cls( input_chunk_length=IN_LEN, output_chunk_length=OUT_LEN, random_state=0, **kwargs, ) model.fit([self.ts_pass_train, self.ts_pass_train_1]) with self.assertRaises(ValueError): # when model is fit from >1 series, one must provide a series in argument model.predict(n=1) pred = model.predict(n=36, series=self.ts_pass_train) mape_err = mape(self.ts_pass_val, pred) self.assertTrue( mape_err < err, "Model {} produces errors too high (several time " "series). Error = {}".format(model_cls, mape_err), ) # check prediction for several time series pred_list = model.predict( n=36, series=[self.ts_pass_train, self.ts_pass_train_1]) self.assertTrue( len(pred_list) == 2, f"Model {model_cls} did not return a list of prediction", ) for pred in pred_list: mape_err = mape(self.ts_pass_val, pred) self.assertTrue( mape_err < err, "Model {} produces errors too high (several time series 2). " "Error = {}".format(model_cls, mape_err), )
def eval_model(model): pred_series = model.predict(n=96) plt.figure(figsize=(8, 5)) series_transformed.plot(label='actual') pred_series.plot(label='forecast') plt.title('MAPE: {:.2f}%'.format(mape(pred_series, val_transformed))) plt.legend()
def compare_best_against_random(model_class, params, series, stride=1): # instantiate best model in expanding window mode np.random.seed(1) best_model_1, _, _ = model_class.gridsearch( params, series, forecast_horizon=10, stride=stride, metric=mape, start=series.time_index[-21], ) # instantiate best model in split mode train, val = series.split_before(series.time_index[-10]) best_model_2, _, _ = model_class.gridsearch( params, train, val_series=val, metric=mape ) # intantiate model with random parameters from 'params' random.seed(1) random_param_choice = {} for key in params.keys(): random_param_choice[key] = random.choice(params[key]) random_model = model_class(**random_param_choice) # perform backtest forecasting on both models best_score_1 = best_model_1.backtest( series, start=series.time_index[-21], forecast_horizon=10 ) random_score_1 = random_model.backtest( series, start=series.time_index[-21], forecast_horizon=10 ) # perform train/val evaluation on both models best_model_2.fit(train) best_score_2 = mape(best_model_2.predict(len(val)), series) random_model = model_class(**random_param_choice) random_model.fit(train) random_score_2 = mape(random_model.predict(len(val)), series) # check whether best models are at least as good as random models expanding_window_ok = best_score_1 <= random_score_1 split_ok = best_score_2 <= random_score_2 return expanding_window_ok and split_ok
def test_future_covariates(self): # models with future covariates should produce better predictions over a long forecasting horizon # than a model trained with no covariates model = TCNModel( input_chunk_length=50, output_chunk_length=5, n_epochs=20, random_state=0, ) model.fit(series=self.target_past) long_pred_no_cov = model.predict(n=160) model = TCNModel( input_chunk_length=50, output_chunk_length=5, n_epochs=20, random_state=0, ) model.fit(series=self.target_past, past_covariates=self.covariates_past) long_pred_with_cov = model.predict(n=160, past_covariates=self.covariates) self.assertTrue( mape(self.target_future, long_pred_no_cov) > mape( self.target_future, long_pred_with_cov), "Models with future covariates should produce better predictions.", ) # block models can predict up to self.output_chunk_length points beyond the last future covariate... model.predict(n=165, past_covariates=self.covariates) # ... not more with self.assertRaises(ValueError): model.predict(n=166, series=self.ts_pass_train) # recurrent models can only predict data points for time steps where future covariates are available model = RNNModel(12, n_epochs=1) model.fit(series=self.target_past, future_covariates=self.covariates_past) model.predict(n=160, future_covariates=self.covariates) with self.assertRaises(ValueError): model.predict(n=161, future_covariates=self.covariates)
def test_multivariate_models_performance(self): # for every model, check whether its errors do not exceed the given bounds for model, max_mape in multivariate_models: np.random.seed(1) model.fit(self.ts_ice_heater_train) prediction = model.predict(len(self.ts_ice_heater_val)) current_mape = mape(prediction, self.ts_ice_heater_val) self.assertTrue( current_mape < max_mape, "{} model exceeded the maximum MAPE of {}. " "with a MAPE of {}".format(str(model), max_mape, current_mape), )
def test_single_ts(self): for model_cls, kwargs, err in models_cls_kwargs_errs: model = model_cls( input_chunk_length=IN_LEN, output_chunk_length=OUT_LEN, random_state=0, **kwargs, ) model.fit(self.ts_pass_train) pred = model.predict(n=36) mape_err = mape(self.ts_pass_val, pred) self.assertTrue( mape_err < err, "Model {} produces errors too high (one time " "series). Error = {}".format(model_cls, mape_err), ) self.assertTrue( pred.static_covariates.equals( self.ts_passengers.static_covariates))
def test_covariates(self): for model_cls, kwargs, err in models_cls_kwargs_errs: model = model_cls( input_chunk_length=IN_LEN, output_chunk_length=OUT_LEN, random_state=0, **kwargs, ) # Here we rely on the fact that all non-Dual models currently are Past models cov_name = ("future_covariates" if isinstance( model, DualCovariatesTorchModel) else "past_covariates") cov_kwargs = { cov_name: [self.time_covariates_train, self.time_covariates_train] } model.fit(series=[self.ts_pass_train, self.ts_pass_train_1], **cov_kwargs) with self.assertRaises(ValueError): # when model is fit from >1 series, one must provide a series in argument model.predict(n=1) with self.assertRaises(ValueError): # when model is fit using multiple covariates, covariates are required at prediction time model.predict(n=1, series=self.ts_pass_train) cov_kwargs_train = {cov_name: self.time_covariates_train} cov_kwargs_notrain = {cov_name: self.time_covariates} with self.assertRaises(ValueError): # when model is fit using covariates, n cannot be greater than output_chunk_length... model.predict(n=13, series=self.ts_pass_train, **cov_kwargs_train) # ... unless future covariates are provided pred = model.predict(n=13, series=self.ts_pass_train, **cov_kwargs_notrain) pred = model.predict(n=12, series=self.ts_pass_train, **cov_kwargs_notrain) mape_err = mape(self.ts_pass_val, pred) self.assertTrue( mape_err < err, "Model {} produces errors too high (several time " "series with covariates). Error = {}".format( model_cls, mape_err), ) # when model is fit using 1 training and 1 covariate series, time series args are optional if model._is_probabilistic: continue model = model_cls(input_chunk_length=IN_LEN, output_chunk_length=OUT_LEN, **kwargs) model.fit(series=self.ts_pass_train, **cov_kwargs_train) pred1 = model.predict(1) pred2 = model.predict(1, series=self.ts_pass_train) pred3 = model.predict(1, **cov_kwargs_train) pred4 = model.predict(1, **cov_kwargs_train, series=self.ts_pass_train) self.assertEqual(pred1, pred2) self.assertEqual(pred1, pred3) self.assertEqual(pred1, pred4)
from darts.models import AutoARIMA model_aarima = AutoARIMA() model_aarima.fit(train) prediction_aarima = model_aarima.predict(len(val)) series.plot(label='actual', lw=3) prediction_aarima.plot(label='forecast', lw=3) plt.legend() plt.xlabel('Year') """Backtesting for comparing two models""" from darts.backtesting import backtest_forecasting from darts.models import Prophet models = [ExponentialSmoothing(), Prophet()] backtests = [backtest_forecasting(series, model, pd.Timestamp('19550101'), fcast_horizon_n=3) for model in models] """To compute error metrics — mean absolute percentage error(mape):""" from darts.metrics import mape series.plot(label='data') for i, m in enumerate(models): err = mape(backtests[i], series) backtests[i].plot(lw=3, label='{}, MAPE={:.2f}%'.format(m, err)) plt.title('Backtests with 3-months forecast horizon') plt.legend()
def eval_model(model): pred_series = model.predict(n=96) plt.figure(figsize=(8, 5)) series_transformed.plot(label='actual') pred_series.plot(label='forecast') plt.title('MAPE: {:.2f}%'.format(mape(pred_series, val_transformed))) plt.legend() eval_model(my_model) best_model = RNNModel.load_from_checkpoint(model_name='Eq_RNN', best=True) eval_model(best_model) backtest_series = my_model.historical_forecasts(series_transformed, start=pd.Timestamp('20021231'), forecast_horizon=12, retrain=False, verbose=True) plt.figure(figsize=(8, 5)) series_transformed.plot(label='actual') backtest_series.plot(label='backtest') plt.legend() plt.title('Backtest, starting Jan 2003, 12-months horizon') print('MAPE: {:.2f}%'.format( mape(transformer.inverse_transform(series_transformed), transformer.inverse_transform(backtest_series))))
ts_var = 'CPU_Used' time_df = df.filter(['date', ts_var]) #convert variable from object dtype to numeric dtype time_df[ts_var] = pd.to_numeric(time_df[ts_var], errors='coerce') #remove duplicates time_df.sort_values("date", inplace=True) time_df.drop_duplicates(subset="date", keep=False, inplace=True) #generate time series using darts series = TimeSeries.from_dataframe(time_df, 'date', ts_var, freq='T') #treat missing values filler = MissingValuesFiller() series = filler.transform(series) #training and testing dataset train, val = series.split_after(pd.Timestamp('2019-10-23 19:41:50')) #FFT model model = FFT(required_matches=set(), nr_freqs_to_keep=None) model.fit(train) pred_val = model.predict(len(val)) #Evaluation metrics series.plot(label='actual') pred_val.plot(label='forecast', lw=3) plt.legend() print("MAPE:", mape(pred_val, val))
ts_var = 'Memory_Used' time_df = df.filter(['date', ts_var]) #convert variable from object dtype to numeric dtype time_df[ts_var] = pd.to_numeric(time_df[ts_var], errors='coerce') #generate time series using darts series = TimeSeries.from_dataframe(time_df, 'date', ts_var, freq='S') #treat missing values filler = MissingValuesFiller() series = filler.transform(series) #scale the values scaler = Scaler() rescaled = scaler.fit_transform(series) #training and testing dataset train, val = rescaled.split_after(pd.Timestamp('2020-01-23 19:41:50')) #Exponential smoothing model model = ExponentialSmoothing() model.fit(train) prediction = model.predict(len(val)) #Evaluation metrics rescaled.plot(label='actual') prediction.plot(label='forecast', lw=3) plt.legend() print("MAPE:", mape(prediction, val))