def helper_relevant_attributes(self, freq, length, period_attributes_tuples): # test random walk random_walk_ts = tg.random_walk_timeseries(freq=freq, length=length) self.assertEqual(_find_relevant_timestamp_attributes(random_walk_ts), set()) for period, relevant_attributes in period_attributes_tuples: # test seasonal period with no noise seasonal_ts = tg.sine_timeseries(freq=freq, value_frequency=1 / period, length=length) self.assertEqual( _find_relevant_timestamp_attributes(seasonal_ts), relevant_attributes, "failed to recognize season in non-noisy timeseries", ) # test seasonal period with no noise seasonal_noisy_ts = seasonal_ts + tg.gaussian_timeseries( freq=freq, length=length) self.assertEqual( _find_relevant_timestamp_attributes(seasonal_noisy_ts), relevant_attributes, "failed to recognize season in noisy timeseries", )
def test_stationarity_tests(self): series_1 = constant_timeseries(start=0, end=9999).stack( constant_timeseries(start=0, end=9999)) series_2 = TimeSeries.from_values( np.random.uniform(0, 1, (1000, 2, 1000))) series_3 = gaussian_timeseries(start=0, end=9999) # Test univariate with self.assertRaises(AssertionError): stationarity_tests(series_1) with self.assertRaises(AssertionError): stationarity_test_adf(series_1) with self.assertRaises(AssertionError): stationarity_test_kpss(series_1) # Test deterministic with self.assertRaises(AssertionError): stationarity_tests(series_2) with self.assertRaises(AssertionError): stationarity_test_adf(series_2) with self.assertRaises(AssertionError): stationarity_test_kpss(series_2) # Test basics self.assertTrue(stationarity_test_kpss(series_3)[1] > 0.05) self.assertTrue(stationarity_test_adf(series_3)[1] < 0.05) self.assertTrue(stationarity_tests)
def test_gaussian_process_multivariate(self): gpf = GaussianProcessFilter() sine_ts = tg.sine_timeseries(length=30, value_frequency=0.1) noise_ts = tg.gaussian_timeseries(length=30) * 0.1 ts = sine_ts.stack(noise_ts) prediction = gpf.filter(ts) self.assertEqual(prediction.width, 2)
def test_granger_causality(self): series_cause_1 = constant_timeseries(start=0, end=9999).stack( constant_timeseries(start=0, end=9999)) series_cause_2 = gaussian_timeseries(start=0, end=9999) series_effect_1 = constant_timeseries(start=0, end=999) series_effect_2 = TimeSeries.from_values(np.random.uniform( 0, 1, 10000)) series_effect_3 = TimeSeries.from_values( np.random.uniform(0, 1, (1000, 2, 1000))) series_effect_4 = constant_timeseries(start=pd.Timestamp("2000-01-01"), length=10000) # Test univariate with self.assertRaises(AssertionError): granger_causality_tests(series_cause_1, series_effect_1, 10, verbose=False) with self.assertRaises(AssertionError): granger_causality_tests(series_effect_1, series_cause_1, 10, verbose=False) # Test deterministic with self.assertRaises(AssertionError): granger_causality_tests(series_cause_1, series_effect_3, 10, verbose=False) with self.assertRaises(AssertionError): granger_causality_tests(series_effect_3, series_cause_1, 10, verbose=False) # Test Frequency with self.assertRaises(ValueError): granger_causality_tests(series_cause_2, series_effect_4, 10, verbose=False) # Test granger basics tests = granger_causality_tests(series_effect_2, series_effect_2, 10, verbose=False) self.assertTrue(tests[1][0]["ssr_ftest"][1] > 0.99) tests = granger_causality_tests(series_cause_2, series_effect_2, 10, verbose=False) self.assertTrue(tests[1][0]["ssr_ftest"][1] > 0.01)
def test_kalman_multivariate(self): kf = KalmanFilter(dim_x=3) sine_ts = tg.sine_timeseries(length=30, value_frequency=0.1) noise_ts = tg.gaussian_timeseries(length=30) * 0.1 series = sine_ts.stack(noise_ts) kf.fit(series) prediction = kf.filter(series) self.assertEqual(prediction.width, 2) self.assertEqual(prediction.n_samples, 1)
def test_exogenous_variables_support(self): # test case with pd.DatetimeIndex target_dt_idx = self.ts_gaussian fc_dt_idx = self.ts_gaussian_long # test case with numerical pd.RangeIndex target_num_idx = TimeSeries.from_times_and_values( times=tg._generate_index(start=0, length=len(self.ts_gaussian)), values=self.ts_gaussian.all_values(copy=False), ) fc_num_idx = TimeSeries.from_times_and_values( times=tg._generate_index(start=0, length=len(self.ts_gaussian_long)), values=self.ts_gaussian_long.all_values(copy=False), ) for target, future_covariates in zip([target_dt_idx, target_num_idx], [fc_dt_idx, fc_num_idx]): for model in dual_models: # skip models which do not support RangeIndex if isinstance(target.time_index, pd.RangeIndex): try: # _supports_range_index raises a ValueError if model does not support RangeIndex model._supports_range_index() except ValueError: continue # Test models runnability - proper future covariates slicing model.fit(target, future_covariates=future_covariates) prediction = model.predict(self.forecasting_horizon, future_covariates=future_covariates) self.assertTrue(len(prediction) == self.forecasting_horizon) # Test mismatch in length between exogenous variables and forecasting horizon with self.assertRaises(ValueError): model.predict( self.forecasting_horizon, future_covariates=tg.gaussian_timeseries( start=future_covariates.start_time(), length=self.forecasting_horizon - 1, ), ) # Test mismatch in time-index/length between series and exogenous variables with self.assertRaises(ValueError): model.fit(target, future_covariates=target[:-1]) with self.assertRaises(ValueError): model.fit(target[1:], future_covariates=target[:-1])
def test_moving_average_multivariate(self): ma = MovingAverage(window=3) sine_ts = tg.sine_timeseries(length=30, value_frequency=0.1) noise_ts = tg.gaussian_timeseries(length=30) * 0.1 ts = sine_ts.stack(noise_ts) ts_filtered = ma.filter(ts) self.assertGreater( np.mean(np.abs(ts.values()[:, 0])), np.mean(np.abs(ts_filtered.values()[:, 0])), ) self.assertGreater( np.mean(np.abs(ts.values()[:, 1])), np.mean(np.abs(ts_filtered.values()[:, 1])), )
def denoising_input(self): np.random.seed(self.RANDOM_SEED) ts_periodic = tg.sine_timeseries(length=500) ts_gaussian = tg.gaussian_timeseries(length=500) ts_random_walk = tg.random_walk_timeseries(length=500) ts_cov1 = ts_periodic.stack(ts_gaussian) ts_cov1 = ts_cov1.pd_dataframe() ts_cov1.columns = ["Periodic", "Gaussian"] ts_cov1 = TimeSeries.from_dataframe(ts_cov1) ts_sum1 = ts_periodic + ts_gaussian ts_cov2 = ts_sum1.stack(ts_random_walk) ts_sum2 = ts_sum1 + ts_random_walk return ts_sum1, ts_cov1, ts_sum2, ts_cov2
def test_kalman_missing_values(self): sine = tg.sine_timeseries( length=100, value_frequency=0.05) + 0.1 * tg.gaussian_timeseries(length=100) values = sine.values() values[20:22] = np.nan values[28:40] = np.nan sine_holes = TimeSeries.from_values(values) sine = TimeSeries.from_values(sine.values()) kf = KalmanFilter(dim_x=2) kf.fit(sine_holes[-50:]) # fit on the part with no holes # reconstructruction should succeed filtered_series = kf.filter(sine_holes, num_samples=100) # reconstruction error should be sufficiently small self.assertLess(rmse(filtered_series, sine), 0.1)
class ProbabilisticRegressionModelsTestCase(DartsBaseTestClass): models_cls_kwargs_errs = [ ( LightGBMModel, { "lags": 2, "likelihood": "quantile", "random_state": 42 }, 0.4, ), ( LightGBMModel, { "lags": 2, "likelihood": "quantile", "quantiles": [0.1, 0.3, 0.5, 0.7, 0.9], "random_state": 42, }, 0.4, ), ( LightGBMModel, { "lags": 2, "likelihood": "poisson", "random_state": 42 }, 0.6, ), ( LinearRegressionModel, { "lags": 2, "likelihood": "quantile", "random_state": 42 }, 0.6, ), ( LinearRegressionModel, { "lags": 2, "likelihood": "poisson", "random_state": 42 }, 0.6, ), ] constant_ts = tg.constant_timeseries(length=200, value=0.5) constant_noisy_ts = constant_ts + tg.gaussian_timeseries(length=200, std=0.1) constant_multivar_ts = constant_ts.stack(constant_ts) constant_noisy_multivar_ts = constant_noisy_ts.stack(constant_noisy_ts) num_samples = 5 def test_fit_predict_determinism(self): for model_cls, model_kwargs, _ in self.models_cls_kwargs_errs: # whether the first predictions of two models initiated with the same random state are the same model = model_cls(**model_kwargs) model.fit(self.constant_noisy_multivar_ts) pred1 = model.predict(n=10, num_samples=2).values() model = model_cls(**model_kwargs) model.fit(self.constant_noisy_multivar_ts) pred2 = model.predict(n=10, num_samples=2).values() self.assertTrue((pred1 == pred2).all()) # test whether the next prediction of the same model is different pred3 = model.predict(n=10, num_samples=2).values() self.assertTrue((pred2 != pred3).any()) def test_probabilistic_forecast_accuracy(self): for model_cls, model_kwargs, err in self.models_cls_kwargs_errs: self.helper_test_probabilistic_forecast_accuracy( model_cls, model_kwargs, err, self.constant_ts, self.constant_noisy_ts, ) if issubclass(model_cls, GlobalForecastingModel): self.helper_test_probabilistic_forecast_accuracy( model_cls, model_kwargs, err, self.constant_multivar_ts, self.constant_noisy_multivar_ts, ) def helper_test_probabilistic_forecast_accuracy( self, model_cls, model_kwargs, err, ts, noisy_ts): model = model_cls(**model_kwargs) model.fit(noisy_ts[:100]) pred = model.predict(n=100, num_samples=100) # test accuracy of the median prediction compared to the noiseless ts mae_err_median = mae(ts[100:], pred) self.assertLess(mae_err_median, err) # test accuracy for increasing quantiles between 0.7 and 1 (it should ~decrease, mae should ~increase) tested_quantiles = [0.7, 0.8, 0.9, 0.99] mae_err = mae_err_median for quantile in tested_quantiles: new_mae = mae(ts[100:], pred.quantile_timeseries(quantile=quantile)) self.assertLess(mae_err, new_mae + 0.1) mae_err = new_mae # test accuracy for decreasing quantiles between 0.3 and 0 (it should ~decrease, mae should ~increase) tested_quantiles = [0.3, 0.2, 0.1, 0.01] mae_err = mae_err_median for quantile in tested_quantiles: new_mae = mae(ts[100:], pred.quantile_timeseries(quantile=quantile)) self.assertLess(mae_err, new_mae + 0.1) mae_err = new_mae
def test_coverage(self): torch.manual_seed(0) input_chunk_lengths = range(20, 50) kernel_sizes = range(2, 5) dilation_bases = range(2, 5) for kernel_size in kernel_sizes: for dilation_base in dilation_bases: if dilation_base > kernel_size: continue for input_chunk_length in input_chunk_lengths: # create model with all weights set to one model = TCNModel( input_chunk_length=input_chunk_length, output_chunk_length=1, kernel_size=kernel_size, dilation_base=dilation_base, weight_norm=False, n_epochs=1, ) # we have to fit the model on a dummy series in order to create the internal nn.Module model.fit(tg.gaussian_timeseries(length=100)) for res_block in model.model.res_blocks: res_block.conv1.weight = torch.nn.Parameter( torch.ones( res_block.conv1.weight.shape, dtype=torch.float64 ) ) res_block.conv2.weight = torch.nn.Parameter( torch.ones( res_block.conv2.weight.shape, dtype=torch.float64 ) ) model.model.eval() # also disable MC Dropout: model.model.set_mc_dropout(False) input_tensor = torch.zeros( [1, input_chunk_length, 1], dtype=torch.float64 ) zero_output = model.model.forward((input_tensor, None))[ 0, -1, 0 ] # test for full coverage for i in range(input_chunk_length): input_tensor[0, i, 0] = 1 curr_output = model.model.forward((input_tensor, None))[ 0, -1, 0 ] self.assertNotEqual(zero_output, curr_output) input_tensor[0, i, 0] = 0 # create model with all weights set to one and one layer less than is automatically detected model_2 = TCNModel( input_chunk_length=input_chunk_length, output_chunk_length=1, kernel_size=kernel_size, dilation_base=dilation_base, weight_norm=False, num_layers=model.model.num_layers - 1, n_epochs=1, ) # we have to fit the model on a dummy series in order to create the internal nn.Module model_2.fit(tg.gaussian_timeseries(length=100)) for res_block in model_2.model.res_blocks: res_block.conv1.weight = torch.nn.Parameter( torch.ones( res_block.conv1.weight.shape, dtype=torch.float64 ) ) res_block.conv2.weight = torch.nn.Parameter( torch.ones( res_block.conv2.weight.shape, dtype=torch.float64 ) ) model_2.model.eval() # also disable MC Dropout: model_2.model.set_mc_dropout(False) input_tensor = torch.zeros( [1, input_chunk_length, 1], dtype=torch.float64 ) zero_output = model_2.model.forward((input_tensor, None))[ 0, -1, 0 ] # test for incomplete coverage uncovered_input_found = False if model_2.model.num_layers == 1: continue for i in range(input_chunk_length): input_tensor[0, i, 0] = 1 curr_output = model_2.model.forward((input_tensor, None))[ 0, -1, 0 ] if zero_output == curr_output: uncovered_input_found = True break input_tensor[0, i, 0] = 0 self.assertTrue(uncovered_input_found)
class LocalForecastingModelsTestCase(DartsBaseTestClass): # forecasting horizon used in runnability tests forecasting_horizon = 5 # dummy timeseries for runnability tests np.random.seed(1) ts_gaussian = tg.gaussian_timeseries(length=100, mean=50) # for testing covariate slicing ts_gaussian_long = tg.gaussian_timeseries( length=len(ts_gaussian) + 2 * forecasting_horizon, start=ts_gaussian.start_time() - forecasting_horizon * ts_gaussian.freq, mean=50, ) # real timeseries for functionality tests ts_passengers = AirPassengersDataset().load() ts_pass_train, ts_pass_val = ts_passengers.split_after( pd.Timestamp("19570101")) # real multivariate timeseries for functionality tests ts_ice_heater = IceCreamHeaterDataset().load() ts_ice_heater_train, ts_ice_heater_val = ts_ice_heater.split_after( split_point=0.7) def test_save_model_parameters(self): # model creation parameters were saved before. check if re-created model has same params as original for model, _ in models: self.assertTrue( model._model_params == model.untrained_model()._model_params) def test_models_runnability(self): for model, _ in models: prediction = model.fit(self.ts_gaussian).predict( self.forecasting_horizon) self.assertTrue(len(prediction) == self.forecasting_horizon) def test_models_performance(self): # for every model, check whether its errors do not exceed the given bounds for model, max_mape in models: np.random.seed(1) # some models are probabilist... model.fit(self.ts_pass_train) prediction = model.predict(len(self.ts_pass_val)) current_mape = mape(prediction, self.ts_pass_val) self.assertTrue( current_mape < max_mape, "{} model exceeded the maximum MAPE of {}. " "with a MAPE of {}".format(str(model), max_mape, current_mape), ) def test_multivariate_models_performance(self): # for every model, check whether its errors do not exceed the given bounds for model, max_mape in multivariate_models: np.random.seed(1) model.fit(self.ts_ice_heater_train) prediction = model.predict(len(self.ts_ice_heater_val)) current_mape = mape(prediction, self.ts_ice_heater_val) self.assertTrue( current_mape < max_mape, "{} model exceeded the maximum MAPE of {}. " "with a MAPE of {}".format(str(model), max_mape, current_mape), ) def test_multivariate_input(self): es_model = ExponentialSmoothing() ts_passengers_enhanced = self.ts_passengers.add_datetime_attribute( "month") with self.assertRaises(AssertionError): es_model.fit(ts_passengers_enhanced) es_model.fit(ts_passengers_enhanced["#Passengers"]) with self.assertRaises(KeyError): es_model.fit(ts_passengers_enhanced["2"]) def test_exogenous_variables_support(self): # test case with pd.DatetimeIndex target_dt_idx = self.ts_gaussian fc_dt_idx = self.ts_gaussian_long # test case with numerical pd.RangeIndex target_num_idx = TimeSeries.from_times_and_values( times=tg._generate_index(start=0, length=len(self.ts_gaussian)), values=self.ts_gaussian.all_values(copy=False), ) fc_num_idx = TimeSeries.from_times_and_values( times=tg._generate_index(start=0, length=len(self.ts_gaussian_long)), values=self.ts_gaussian_long.all_values(copy=False), ) for target, future_covariates in zip([target_dt_idx, target_num_idx], [fc_dt_idx, fc_num_idx]): for model in dual_models: # skip models which do not support RangeIndex if isinstance(target.time_index, pd.RangeIndex): try: # _supports_range_index raises a ValueError if model does not support RangeIndex model._supports_range_index() except ValueError: continue # Test models runnability - proper future covariates slicing model.fit(target, future_covariates=future_covariates) prediction = model.predict(self.forecasting_horizon, future_covariates=future_covariates) self.assertTrue(len(prediction) == self.forecasting_horizon) # Test mismatch in length between exogenous variables and forecasting horizon with self.assertRaises(ValueError): model.predict( self.forecasting_horizon, future_covariates=tg.gaussian_timeseries( start=future_covariates.start_time(), length=self.forecasting_horizon - 1, ), ) # Test mismatch in time-index/length between series and exogenous variables with self.assertRaises(ValueError): model.fit(target, future_covariates=target[:-1]) with self.assertRaises(ValueError): model.fit(target[1:], future_covariates=target[:-1]) def test_dummy_series(self): values = np.random.uniform(low=-10, high=10, size=100) ts = TimeSeries.from_dataframe(pd.DataFrame({"V1": values})) varima = VARIMA(trend="t") with self.assertRaises(ValueError): varima.fit(series=ts) if PMDARIMA_AVAILABLE: autoarima = AutoARIMA(trend="t") with self.assertRaises(ValueError): autoarima.fit(series=ts)
class RegressionEnsembleModelsTestCase(DartsBaseTestClass): RANDOM_SEED = 111 sine_series = tg.sine_timeseries(value_frequency=(1 / 5), value_y_offset=10, length=50) lin_series = tg.linear_timeseries(length=50) combined = sine_series + lin_series seq1 = [_make_ts(0), _make_ts(10), _make_ts(20)] cov1 = [_make_ts(5), _make_ts(15), _make_ts(25)] seq2 = [_make_ts(0, 20), _make_ts(10, 20), _make_ts(20, 20)] cov2 = [_make_ts(5, 30), _make_ts(15, 30), _make_ts(25, 30)] # dummy feature and target TimeSeries instances ts_periodic = tg.sine_timeseries(length=500) ts_gaussian = tg.gaussian_timeseries(length=500) ts_random_walk = tg.random_walk_timeseries(length=500) ts_cov1 = ts_periodic.stack(ts_gaussian) ts_cov1 = ts_cov1.pd_dataframe() ts_cov1.columns = ["Periodic", "Gaussian"] ts_cov1 = TimeSeries.from_dataframe(ts_cov1) ts_sum1 = ts_periodic + ts_gaussian ts_cov2 = ts_sum1.stack(ts_random_walk) ts_sum2 = ts_sum1 + ts_random_walk def get_local_models(self): return [NaiveDrift(), NaiveSeasonal(5), NaiveSeasonal(10)] @unittest.skipUnless(TORCH_AVAILABLE, "requires torch") def get_global_models(self, output_chunk_length=5): return [ RNNModel( input_chunk_length=20, output_chunk_length=output_chunk_length, n_epochs=1, random_state=42, ), BlockRNNModel( input_chunk_length=20, output_chunk_length=output_chunk_length, n_epochs=1, random_state=42, ), ] @unittest.skipUnless(TORCH_AVAILABLE, "requires torch") def test_accepts_different_regression_models(self): regr1 = LinearRegression() regr2 = RandomForestRegressor() regr3 = RandomForest(lags_future_covariates=[0]) model0 = RegressionEnsembleModel(self.get_local_models(), 10) model1 = RegressionEnsembleModel(self.get_local_models(), 10, regr1) model2 = RegressionEnsembleModel(self.get_local_models(), 10, regr2) model3 = RegressionEnsembleModel(self.get_local_models(), 10, regr3) models = [model0, model1, model2, model3] for model in models: model.fit(series=self.combined) model.predict(10) @unittest.skipUnless(TORCH_AVAILABLE, "requires torch") def test_accepts_one_model(self): regr1 = LinearRegression() regr2 = RandomForest(lags_future_covariates=[0]) model0 = RegressionEnsembleModel([self.get_local_models()[0]], 10) model1 = RegressionEnsembleModel([self.get_local_models()[0]], 10, regr1) model2 = RegressionEnsembleModel([self.get_local_models()[0]], 10, regr2) models = [model0, model1, model2] for model in models: model.fit(series=self.combined) model.predict(10) @unittest.skipUnless(TORCH_AVAILABLE, "requires torch") def test_train_n_points(self): regr = LinearRegressionModel(lags_future_covariates=[0]) # same values ensemble = RegressionEnsembleModel(self.get_local_models(), 5, regr) # too big value to perform the split ensemble = RegressionEnsembleModel(self.get_local_models(), 100) with self.assertRaises(ValueError): ensemble.fit(self.combined) ensemble = RegressionEnsembleModel(self.get_local_models(), 50) with self.assertRaises(ValueError): ensemble.fit(self.combined) # too big value considering min_train_series_length ensemble = RegressionEnsembleModel(self.get_local_models(), 45) with self.assertRaises(ValueError): ensemble.fit(self.combined) @unittest.skipUnless(TORCH_AVAILABLE, "requires torch") def test_torch_models_retrain(self): model1 = BlockRNNModel(input_chunk_length=12, output_chunk_length=1, random_state=0, n_epochs=2) model2 = BlockRNNModel(input_chunk_length=12, output_chunk_length=1, random_state=0, n_epochs=2) ensemble = RegressionEnsembleModel([model1], 5) ensemble.fit(self.combined) model1_fitted = ensemble.models[0] forecast1 = model1_fitted.predict(10) model2.fit(self.combined) forecast2 = model2.predict(10) self.assertAlmostEqual(sum(forecast1.values() - forecast2.values())[0], 0.0, places=2) @unittest.skipUnless(TORCH_AVAILABLE, "requires torch") def test_train_predict_global_models_univar(self): ensemble_models = self.get_global_models(output_chunk_length=10) ensemble_models.append(RegressionModel(lags=1)) ensemble = RegressionEnsembleModel(ensemble_models, 10) ensemble.fit(series=self.combined) ensemble.predict(10) @unittest.skipUnless(TORCH_AVAILABLE, "requires torch") def test_train_predict_global_models_multivar_no_covariates(self): ensemble_models = self.get_global_models(output_chunk_length=10) ensemble_models.append(RegressionModel(lags=1)) ensemble = RegressionEnsembleModel(ensemble_models, 10) ensemble.fit(self.seq1) ensemble.predict(10, self.seq1) @unittest.skipUnless(TORCH_AVAILABLE, "requires torch") def test_train_predict_global_models_multivar_with_covariates(self): ensemble_models = self.get_global_models(output_chunk_length=10) ensemble_models.append( RegressionModel(lags=1, lags_past_covariates=[-1])) ensemble = RegressionEnsembleModel(ensemble_models, 10) ensemble.fit(self.seq1, self.cov1) ensemble.predict(10, self.seq2, self.cov2) @unittest.skipUnless(TORCH_AVAILABLE, "requires torch") def helper_test_models_accuracy(self, model_instance, n, series, past_covariates, min_rmse): # for every model, test whether it predicts the target with a minimum r2 score of `min_rmse` train_series, test_series = train_test_split(series, pd.Timestamp("20010101")) train_past_covariates, _ = train_test_split(past_covariates, pd.Timestamp("20010101")) model_instance.fit(series=train_series, past_covariates=train_past_covariates) prediction = model_instance.predict(n=n, past_covariates=past_covariates) current_rmse = rmse(test_series, prediction) self.assertTrue( current_rmse <= min_rmse, f"Model was not able to denoise data. A rmse score of {current_rmse} was recorded.", ) @unittest.skipUnless(TORCH_AVAILABLE, "requires torch") def denoising_input(self): np.random.seed(self.RANDOM_SEED) ts_periodic = tg.sine_timeseries(length=500) ts_gaussian = tg.gaussian_timeseries(length=500) ts_random_walk = tg.random_walk_timeseries(length=500) ts_cov1 = ts_periodic.stack(ts_gaussian) ts_cov1 = ts_cov1.pd_dataframe() ts_cov1.columns = ["Periodic", "Gaussian"] ts_cov1 = TimeSeries.from_dataframe(ts_cov1) ts_sum1 = ts_periodic + ts_gaussian ts_cov2 = ts_sum1.stack(ts_random_walk) ts_sum2 = ts_sum1 + ts_random_walk return ts_sum1, ts_cov1, ts_sum2, ts_cov2 @unittest.skipUnless(TORCH_AVAILABLE, "requires torch") def test_ensemble_models_denoising(self): # for every model, test whether it correctly denoises ts_sum using ts_gaussian and ts_sum as inputs # WARNING: this test isn't numerically stable, changing self.RANDOM_SEED can lead to exploding coefficients horizon = 10 ts_sum1, ts_cov1, _, _ = self.denoising_input() torch.manual_seed(self.RANDOM_SEED) ensemble_models = [ RNNModel( input_chunk_length=20, output_chunk_length=horizon, n_epochs=1, random_state=self.RANDOM_SEED, ), BlockRNNModel( input_chunk_length=20, output_chunk_length=horizon, n_epochs=1, random_state=self.RANDOM_SEED, ), RegressionModel(lags_past_covariates=[-1]), ] ensemble = RegressionEnsembleModel(ensemble_models, horizon) self.helper_test_models_accuracy(ensemble, horizon, ts_sum1, ts_cov1, 3) @unittest.skipUnless(TORCH_AVAILABLE, "requires torch") def test_ensemble_models_denoising_multi_input(self): # for every model, test whether it correctly denoises ts_sum_2 using ts_random_multi and ts_sum_2 as inputs # WARNING: this test isn't numerically stable, changing self.RANDOM_SEED can lead to exploding coefficients horizon = 10 _, _, ts_sum2, ts_cov2 = self.denoising_input() torch.manual_seed(self.RANDOM_SEED) ensemble_models = [ RNNModel( input_chunk_length=20, output_chunk_length=horizon, n_epochs=1, random_state=self.RANDOM_SEED, ), BlockRNNModel( input_chunk_length=20, output_chunk_length=horizon, n_epochs=1, random_state=self.RANDOM_SEED, ), RegressionModel(lags_past_covariates=[-1]), RegressionModel(lags_past_covariates=[-1]), ] ensemble = RegressionEnsembleModel(ensemble_models, horizon) self.helper_test_models_accuracy(ensemble, horizon, ts_sum2, ts_cov2, 3)
class ReconciliationTestCase(unittest.TestCase): __test__ = True @classmethod def setUpClass(cls): logging.disable(logging.CRITICAL) np.random.seed(42) """ test case with a more intricate hierarchy """ LENGTH = 200 total_series = (tg.sine_timeseries(value_frequency=0.03, length=LENGTH) + 1 + tg.gaussian_timeseries(length=LENGTH) * 0.2) bottom_1 = total_series / 3 + tg.gaussian_timeseries(length=LENGTH) * 0.01 bottom_2 = 2 * total_series / 3 + tg.gaussian_timeseries( length=LENGTH) * 0.01 series = concatenate([total_series, bottom_1, bottom_2], axis=1) hierarchy = {"sine_1": ["sine"], "sine_2": ["sine"]} series = series.with_hierarchy(hierarchy) # get a single forecast model = LinearRegressionModel(lags=30, output_chunk_length=10) model.fit(series) pred = model.predict(n=20) # get a backtest forecast to get residuals pred_back = model.historical_forecasts(series, start=0.75, forecast_horizon=10) intersection = series.slice_intersect(pred_back) residuals = intersection - pred_back """ test case with a more intricate hierarchy """ components_complex = ["total", "a", "b", "x", "y", "ax", "ay", "bx", "by"] hierarchy_complex = { "ax": ["a", "x"], "ay": ["a", "y"], "bx": ["b", "x"], "by": ["b", "y"], "a": ["total"], "b": ["total"], "x": ["total"], "y": ["total"], } series_complex = TimeSeries.from_values( values=np.random.rand(50, len(components_complex), 5), columns=components_complex, hierarchy=hierarchy_complex, ) def _assert_reconciliation(self, fitted_recon): pred_r = fitted_recon.transform(self.pred) np.testing.assert_almost_equal( pred_r["sine"].values(copy=False), (pred_r["sine_1"] + pred_r["sine_2"]).values(copy=False), ) def _assert_reconciliation_complex(self, fitted_recon): reconciled = fitted_recon.transform(self.series_complex) def _assert_comps(comp, comps): np.testing.assert_almost_equal( reconciled[comp].values(copy=False), sum(reconciled[c] for c in comps).values(copy=False), ) _assert_comps("a", ["ax", "ay"]) _assert_comps("b", ["bx", "by"]) _assert_comps("x", ["ax", "bx"]) _assert_comps("y", ["ay", "by"]) _assert_comps("total", ["ax", "ay", "bx", "by"]) _assert_comps("total", ["a", "b"]) _assert_comps("total", ["x", "y"]) def test_bottom_up(self): recon = BottomUpReconciliator() self._assert_reconciliation(recon) def test_top_down(self): # should work when fitting on training series recon = TopDownReconciliator() recon.fit(self.series) self._assert_reconciliation(recon) # or when fitting on forecasts recon = TopDownReconciliator() recon.fit(self.pred) self._assert_reconciliation(recon) def test_mint(self): # ols recon = MinTReconciliator("ols") recon.fit(self.series) self._assert_reconciliation(recon) # wls_struct recon = MinTReconciliator("wls_struct") recon.fit(self.series) self._assert_reconciliation(recon) # wls_var recon = MinTReconciliator("wls_var") recon.fit(self.residuals) self._assert_reconciliation(recon) # mint_cov recon = MinTReconciliator("mint_cov") recon.fit(self.residuals) self._assert_reconciliation(recon) # wls_val recon = MinTReconciliator("wls_val") recon.fit(self.series) self._assert_reconciliation(recon) def test_summation_matrix(self): np.testing.assert_equal( _get_summation_matrix(self.series_complex), np.array([ [1, 1, 1, 1], [1, 1, 0, 0], [0, 0, 1, 1], [1, 0, 1, 0], [0, 1, 0, 1], [1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1], ]), ) def test_hierarchy_preserved_after_predict(self): self.assertEqual(self.pred.hierarchy, self.series.hierarchy) def test_more_intricate_hierarchy(self): recon = BottomUpReconciliator() self._assert_reconciliation_complex(recon) recon = TopDownReconciliator() recon.fit(self.series_complex) self._assert_reconciliation_complex(recon) recon = MinTReconciliator("ols") recon.fit(self.series_complex) self._assert_reconciliation_complex(recon) recon = MinTReconciliator("wls_struct") recon.fit(self.series_complex) self._assert_reconciliation_complex(recon) recon = MinTReconciliator("wls_val") recon.fit(self.series_complex) self._assert_reconciliation_complex(recon)
class ProbabilisticTorchModelsTestCase(DartsBaseTestClass): np.random.seed(0) constant_ts = tg.constant_timeseries(length=200, value=0.5) constant_noisy_ts = constant_ts + tg.gaussian_timeseries(length=200, std=0.1) constant_multivar_ts = constant_ts.stack(constant_ts) constant_noisy_multivar_ts = constant_noisy_ts.stack(constant_noisy_ts) num_samples = 5 def test_fit_predict_determinism(self): for model_cls, model_kwargs, _ in models_cls_kwargs_errs: # whether the first predictions of two models initiated with the same random state are the same model = model_cls(**model_kwargs) model.fit(self.constant_noisy_ts) pred1 = model.predict(n=10, num_samples=2).values() model = model_cls(**model_kwargs) model.fit(self.constant_noisy_ts) pred2 = model.predict(n=10, num_samples=2).values() self.assertTrue((pred1 == pred2).all()) # test whether the next prediction of the same model is different pred3 = model.predict(n=10, num_samples=2).values() self.assertTrue((pred2 != pred3).any()) def test_probabilistic_forecast_accuracy(self): for model_cls, model_kwargs, err in models_cls_kwargs_errs: self.helper_test_probabilistic_forecast_accuracy( model_cls, model_kwargs, err, self.constant_ts, self.constant_noisy_ts) if issubclass(model_cls, GlobalForecastingModel): self.helper_test_probabilistic_forecast_accuracy( model_cls, model_kwargs, err, self.constant_multivar_ts, self.constant_noisy_multivar_ts, ) def helper_test_probabilistic_forecast_accuracy(self, model_cls, model_kwargs, err, ts, noisy_ts): model = model_cls(**model_kwargs) model.fit(noisy_ts[:100]) pred = model.predict(n=100, num_samples=100) # test accuracy of the median prediction compared to the noiseless ts mae_err_median = mae(ts[100:], pred) self.assertLess(mae_err_median, err) # test accuracy for increasing quantiles between 0.7 and 1 (it should ~decrease, mae should ~increase) tested_quantiles = [0.7, 0.8, 0.9, 0.99] mae_err = mae_err_median for quantile in tested_quantiles: new_mae = mae(ts[100:], pred.quantile_timeseries(quantile=quantile)) self.assertLess(mae_err, new_mae + 0.1) mae_err = new_mae # test accuracy for decreasing quantiles between 0.3 and 0 (it should ~decrease, mae should ~increase) tested_quantiles = [0.3, 0.2, 0.1, 0.01] mae_err = mae_err_median for quantile in tested_quantiles: new_mae = mae(ts[100:], pred.quantile_timeseries(quantile=quantile)) self.assertLess(mae_err, new_mae + 0.1) mae_err = new_mae """ More likelihood tests """ if TORCH_AVAILABLE: np.random.seed(42) torch.manual_seed(42) real_series = TimeSeries.from_values(np.random.randn(100, 2) + [0, 5]) vals = real_series.all_values() real_pos_series = TimeSeries.from_values( np.where(vals > 0, vals, -vals)) discrete_pos_series = TimeSeries.from_values( np.random.randint(low=0, high=11, size=(100, 2))) binary_series = TimeSeries.from_values( np.random.randint(low=0, high=2, size=(100, 2))) bounded_series = TimeSeries.from_values( np.random.beta(2, 5, size=(100, 2))) simplex_series = bounded_series["0"].stack(1.0 - bounded_series["0"]) lkl_series = ( (GaussianLikelihood(), real_series, 0.1, 3), (PoissonLikelihood(), discrete_pos_series, 2, 2), (NegativeBinomialLikelihood(), discrete_pos_series, 0.5, 0.5), (BernoulliLikelihood(), binary_series, 0.15, 0.15), (GammaLikelihood(), real_pos_series, 0.3, 0.3), (GumbelLikelihood(), real_series, 0.2, 3), (LaplaceLikelihood(), real_series, 0.3, 4), (BetaLikelihood(), bounded_series, 0.1, 0.1), (ExponentialLikelihood(), real_pos_series, 0.3, 2), (DirichletLikelihood(), simplex_series, 0.3, 0.3), (GeometricLikelihood(), discrete_pos_series, 1, 1), (CauchyLikelihood(), real_series, 3, 11), (ContinuousBernoulliLikelihood(), bounded_series, 0.1, 0.1), (HalfNormalLikelihood(), real_pos_series, 0.3, 8), (LogNormalLikelihood(), real_pos_series, 0.3, 1), (WeibullLikelihood(), real_pos_series, 0.2, 2.5), (QuantileRegression(), real_series, 0.2, 1), ) def test_likelihoods_and_resulting_mean_forecasts(self): def _get_avgs(series): return np.mean(series.all_values()[:, 0, :]), np.mean( series.all_values()[:, 1, :]) for lkl, series, diff1, diff2 in self.lkl_series: model = RNNModel(input_chunk_length=5, likelihood=lkl) model.fit(series, epochs=50) pred = model.predict(n=50, num_samples=50) avgs_orig, avgs_pred = _get_avgs(series), _get_avgs(pred) self.assertLess( abs(avgs_orig[0] - avgs_pred[0]), diff1, "The difference between the mean forecast and the mean series is larger " "than expected on component 0 for distribution {}".format( lkl), ) self.assertLess( abs(avgs_orig[1] - avgs_pred[1]), diff2, "The difference between the mean forecast and the mean series is larger " "than expected on component 1 for distribution {}".format( lkl), ) def test_stochastic_inputs(self): model = RNNModel(input_chunk_length=5) model.fit(self.constant_ts, epochs=2) # build a stochastic series target_vals = self.constant_ts.values() stochastic_vals = np.random.normal(loc=target_vals, scale=1.0, size=(len(self.constant_ts), 100)) stochastic_vals = np.expand_dims(stochastic_vals, axis=1) stochastic_series = TimeSeries.from_times_and_values( self.constant_ts.time_index, stochastic_vals) # A deterministic model forecasting a stochastic series # should return stochastic samples preds = [ model.predict(series=stochastic_series, n=10) for _ in range(2) ] # random samples should differ self.assertFalse( np.alltrue(preds[0].values() == preds[1].values()))
class GlobalForecastingModelsTestCase(DartsBaseTestClass): # forecasting horizon used in runnability tests forecasting_horizon = 12 np.random.seed(42) torch.manual_seed(42) # some arbitrary static covariates static_covariates = pd.DataFrame([[0.0, 1.0]], columns=["st1", "st2"]) # real timeseries for functionality tests ts_passengers = (AirPassengersDataset().load().with_static_covariates( static_covariates)) scaler = Scaler() ts_passengers = scaler.fit_transform(ts_passengers) ts_pass_train, ts_pass_val = ts_passengers[:-36], ts_passengers[-36:] # an additional noisy series ts_pass_train_1 = ts_pass_train + 0.01 * tg.gaussian_timeseries( length=len(ts_pass_train), freq=ts_pass_train.freq_str, start=ts_pass_train.start_time(), ) # an additional time series serving as covariates year_series = tg.datetime_attribute_timeseries(ts_passengers, attribute="year") month_series = tg.datetime_attribute_timeseries(ts_passengers, attribute="month") scaler_dt = Scaler() time_covariates = scaler_dt.fit_transform( year_series.stack(month_series)) time_covariates_train, time_covariates_val = ( time_covariates[:-36], time_covariates[-36:], ) # an artificial time series that is highly dependent on covariates ts_length = 400 split_ratio = 0.6 sine_1_ts = tg.sine_timeseries(length=ts_length) sine_2_ts = tg.sine_timeseries(length=ts_length, value_frequency=0.05) sine_3_ts = tg.sine_timeseries(length=ts_length, value_frequency=0.003, value_amplitude=5) linear_ts = tg.linear_timeseries(length=ts_length, start_value=3, end_value=8) covariates = sine_3_ts.stack(sine_2_ts).stack(linear_ts) covariates_past, _ = covariates.split_after(split_ratio) target = sine_1_ts + sine_2_ts + linear_ts + sine_3_ts target_past, target_future = target.split_after(split_ratio) def test_save_model_parameters(self): # model creation parameters were saved before. check if re-created model has same params as original for model_cls, kwargs, err in models_cls_kwargs_errs: model = model_cls(input_chunk_length=IN_LEN, output_chunk_length=OUT_LEN, **kwargs) self.assertTrue(model._model_params, model.untrained_model()._model_params) def test_single_ts(self): for model_cls, kwargs, err in models_cls_kwargs_errs: model = model_cls( input_chunk_length=IN_LEN, output_chunk_length=OUT_LEN, random_state=0, **kwargs, ) model.fit(self.ts_pass_train) pred = model.predict(n=36) mape_err = mape(self.ts_pass_val, pred) self.assertTrue( mape_err < err, "Model {} produces errors too high (one time " "series). Error = {}".format(model_cls, mape_err), ) self.assertTrue( pred.static_covariates.equals( self.ts_passengers.static_covariates)) def test_multi_ts(self): for model_cls, kwargs, err in models_cls_kwargs_errs: model = model_cls( input_chunk_length=IN_LEN, output_chunk_length=OUT_LEN, random_state=0, **kwargs, ) model.fit([self.ts_pass_train, self.ts_pass_train_1]) with self.assertRaises(ValueError): # when model is fit from >1 series, one must provide a series in argument model.predict(n=1) pred = model.predict(n=36, series=self.ts_pass_train) mape_err = mape(self.ts_pass_val, pred) self.assertTrue( mape_err < err, "Model {} produces errors too high (several time " "series). Error = {}".format(model_cls, mape_err), ) # check prediction for several time series pred_list = model.predict( n=36, series=[self.ts_pass_train, self.ts_pass_train_1]) self.assertTrue( len(pred_list) == 2, f"Model {model_cls} did not return a list of prediction", ) for pred in pred_list: mape_err = mape(self.ts_pass_val, pred) self.assertTrue( mape_err < err, "Model {} produces errors too high (several time series 2). " "Error = {}".format(model_cls, mape_err), ) def test_covariates(self): for model_cls, kwargs, err in models_cls_kwargs_errs: model = model_cls( input_chunk_length=IN_LEN, output_chunk_length=OUT_LEN, random_state=0, **kwargs, ) # Here we rely on the fact that all non-Dual models currently are Past models cov_name = ("future_covariates" if isinstance( model, DualCovariatesTorchModel) else "past_covariates") cov_kwargs = { cov_name: [self.time_covariates_train, self.time_covariates_train] } model.fit(series=[self.ts_pass_train, self.ts_pass_train_1], **cov_kwargs) with self.assertRaises(ValueError): # when model is fit from >1 series, one must provide a series in argument model.predict(n=1) with self.assertRaises(ValueError): # when model is fit using multiple covariates, covariates are required at prediction time model.predict(n=1, series=self.ts_pass_train) cov_kwargs_train = {cov_name: self.time_covariates_train} cov_kwargs_notrain = {cov_name: self.time_covariates} with self.assertRaises(ValueError): # when model is fit using covariates, n cannot be greater than output_chunk_length... model.predict(n=13, series=self.ts_pass_train, **cov_kwargs_train) # ... unless future covariates are provided pred = model.predict(n=13, series=self.ts_pass_train, **cov_kwargs_notrain) pred = model.predict(n=12, series=self.ts_pass_train, **cov_kwargs_notrain) mape_err = mape(self.ts_pass_val, pred) self.assertTrue( mape_err < err, "Model {} produces errors too high (several time " "series with covariates). Error = {}".format( model_cls, mape_err), ) # when model is fit using 1 training and 1 covariate series, time series args are optional if model._is_probabilistic: continue model = model_cls(input_chunk_length=IN_LEN, output_chunk_length=OUT_LEN, **kwargs) model.fit(series=self.ts_pass_train, **cov_kwargs_train) pred1 = model.predict(1) pred2 = model.predict(1, series=self.ts_pass_train) pred3 = model.predict(1, **cov_kwargs_train) pred4 = model.predict(1, **cov_kwargs_train, series=self.ts_pass_train) self.assertEqual(pred1, pred2) self.assertEqual(pred1, pred3) self.assertEqual(pred1, pred4) def test_future_covariates(self): # models with future covariates should produce better predictions over a long forecasting horizon # than a model trained with no covariates model = TCNModel( input_chunk_length=50, output_chunk_length=5, n_epochs=20, random_state=0, ) model.fit(series=self.target_past) long_pred_no_cov = model.predict(n=160) model = TCNModel( input_chunk_length=50, output_chunk_length=5, n_epochs=20, random_state=0, ) model.fit(series=self.target_past, past_covariates=self.covariates_past) long_pred_with_cov = model.predict(n=160, past_covariates=self.covariates) self.assertTrue( mape(self.target_future, long_pred_no_cov) > mape( self.target_future, long_pred_with_cov), "Models with future covariates should produce better predictions.", ) # block models can predict up to self.output_chunk_length points beyond the last future covariate... model.predict(n=165, past_covariates=self.covariates) # ... not more with self.assertRaises(ValueError): model.predict(n=166, series=self.ts_pass_train) # recurrent models can only predict data points for time steps where future covariates are available model = RNNModel(12, n_epochs=1) model.fit(series=self.target_past, future_covariates=self.covariates_past) model.predict(n=160, future_covariates=self.covariates) with self.assertRaises(ValueError): model.predict(n=161, future_covariates=self.covariates) def test_batch_predictions(self): # predicting multiple time series at once needs to work for arbitrary batch sizes # univariate case targets_univar = [ self.target_past, self.target_past[:60], self.target_past[:80], ] self._batch_prediction_test_helper_function(targets_univar) # multivariate case targets_multivar = [tgt.stack(tgt) for tgt in targets_univar] self._batch_prediction_test_helper_function(targets_multivar) def _batch_prediction_test_helper_function(self, targets): epsilon = 1e-4 model = TCNModel( input_chunk_length=50, output_chunk_length=10, n_epochs=10, random_state=0, ) model.fit(series=targets[0], past_covariates=self.covariates_past) preds_default = model.predict( n=160, series=targets, past_covariates=[self.covariates] * len(targets), batch_size=None, ) # make batch size large enough to test stacking samples for batch_size in range(1, 4 * len(targets)): preds = model.predict( n=160, series=targets, past_covariates=[self.covariates] * len(targets), batch_size=batch_size, ) for i in range(len(targets)): self.assertLess( sum(sum((preds[i] - preds_default[i]).values())), epsilon) def test_predict_from_dataset_unsupported_input(self): # an exception should be thrown if an unsupported type is passed unsupported_type = "unsupported_type" # just need to test this with one model model_cls, kwargs, err = models_cls_kwargs_errs[0] model = model_cls(input_chunk_length=IN_LEN, output_chunk_length=OUT_LEN, **kwargs) model.fit([self.ts_pass_train, self.ts_pass_train_1]) with self.assertRaises(ValueError): model.predict_from_dataset( n=1, input_series_dataset=unsupported_type) def test_prediction_with_different_n(self): # test model predictions for n < out_len, n == out_len and n > out_len for model_cls, kwargs, err in models_cls_kwargs_errs: model = model_cls(input_chunk_length=IN_LEN, output_chunk_length=OUT_LEN, **kwargs) self.assertTrue( isinstance( model, ( PastCovariatesTorchModel, DualCovariatesTorchModel, MixedCovariatesTorchModel, ), ), "unit test not yet defined for the given {X}CovariatesTorchModel.", ) if isinstance(model, PastCovariatesTorchModel): past_covs, future_covs = self.covariates, None elif isinstance(model, DualCovariatesTorchModel): past_covs, future_covs = None, self.covariates else: past_covs, future_covs = self.covariates, self.covariates model.fit( self.target_past, past_covariates=past_covs, future_covariates=future_covs, epochs=1, ) # test prediction for n < out_len, n == out_len and n > out_len for n in [OUT_LEN - 1, OUT_LEN, 2 * OUT_LEN - 1]: pred = model.predict(n=n, past_covariates=past_covs, future_covariates=future_covs) self.assertEqual(len(pred), n) def test_same_result_with_different_n_jobs(self): for model_cls, kwargs, err in models_cls_kwargs_errs: model = model_cls(input_chunk_length=IN_LEN, output_chunk_length=OUT_LEN, **kwargs) multiple_ts = [self.ts_pass_train] * 10 model.fit(multiple_ts) # safe random state for two successive identical predictions if model._is_probabilistic(): random_state = deepcopy(model._random_instance) else: random_state = None pred1 = model.predict(n=36, series=multiple_ts, n_jobs=1) if random_state is not None: model._random_instance = random_state pred2 = model.predict( n=36, series=multiple_ts, n_jobs=-1) # assuming > 1 core available in the machine self.assertEqual( pred1, pred2, "Model {} produces different predictions with different number of jobs", ) @patch( "darts.models.forecasting.torch_forecasting_model.TorchForecastingModel._init_trainer" ) def test_fit_with_constr_epochs(self, init_trainer): for model_cls, kwargs, err in models_cls_kwargs_errs: model = model_cls(input_chunk_length=IN_LEN, output_chunk_length=OUT_LEN, **kwargs) multiple_ts = [self.ts_pass_train] * 10 model.fit(multiple_ts) init_trainer.assert_called_with(max_epochs=kwargs["n_epochs"], trainer_params=ANY) @patch( "darts.models.forecasting.torch_forecasting_model.TorchForecastingModel._init_trainer" ) def test_fit_with_fit_epochs(self, init_trainer): for model_cls, kwargs, err in models_cls_kwargs_errs: model = model_cls(input_chunk_length=IN_LEN, output_chunk_length=OUT_LEN, **kwargs) multiple_ts = [self.ts_pass_train] * 10 epochs = 3 model.fit(multiple_ts, epochs=epochs) init_trainer.assert_called_with(max_epochs=epochs, trainer_params=ANY) model.total_epochs = epochs # continue training model.fit(multiple_ts, epochs=epochs) init_trainer.assert_called_with(max_epochs=epochs, trainer_params=ANY) @patch( "darts.models.forecasting.torch_forecasting_model.TorchForecastingModel._init_trainer" ) def test_fit_from_dataset_with_epochs(self, init_trainer): for model_cls, kwargs, err in models_cls_kwargs_errs: model = model_cls(input_chunk_length=IN_LEN, output_chunk_length=OUT_LEN, **kwargs) multiple_ts = [self.ts_pass_train] * 10 train_dataset = model._build_train_dataset( multiple_ts, past_covariates=None, future_covariates=None, max_samples_per_ts=None, ) epochs = 3 model.fit_from_dataset(train_dataset, epochs=epochs) init_trainer.assert_called_with(max_epochs=epochs, trainer_params=ANY) # continue training model.fit_from_dataset(train_dataset, epochs=epochs) init_trainer.assert_called_with(max_epochs=epochs, trainer_params=ANY) def test_predit_after_fit_from_dataset(self): model_cls, kwargs, _ = models_cls_kwargs_errs[0] model = model_cls(input_chunk_length=IN_LEN, output_chunk_length=OUT_LEN, **kwargs) multiple_ts = [self.ts_pass_train] * 10 train_dataset = model._build_train_dataset( multiple_ts, past_covariates=None, future_covariates=None, max_samples_per_ts=None, ) model.fit_from_dataset(train_dataset, epochs=3) # test predict() works after fit_from_dataset() model.predict(n=1, series=multiple_ts[0]) def test_sample_smaller_than_batch_size(self): """ Checking that the TorchForecastingModels do not crash even if the number of available samples for training is strictly lower than the selected batch_size """ # TS with 50 timestamps. TorchForecastingModels will use the SequentialDataset for producing training # samples, which means we will have 50 - 22 - 2 + 1 = 27 samples, which is < 32 (batch_size). The model # should still train on those samples and not crash in any way ts = linear_timeseries(start_value=0, end_value=1, length=50) model = RNNModel(input_chunk_length=20, output_chunk_length=2, n_epochs=2, batch_size=32) model.fit(ts) def test_max_samples_per_ts(self): """ Checking that we can fit TorchForecastingModels with max_samples_per_ts, without crash """ ts = linear_timeseries(start_value=0, end_value=1, length=50) model = RNNModel(input_chunk_length=20, output_chunk_length=2, n_epochs=2, batch_size=32) model.fit(ts, max_samples_per_ts=5) def test_residuals(self): """ Torch models should not fail when computing residuals on a series long enough to accomodate at least one training sample. """ ts = linear_timeseries(start_value=0, end_value=1, length=38) model = NBEATSModel( input_chunk_length=24, output_chunk_length=12, num_stacks=2, num_blocks=1, num_layers=1, layer_widths=2, n_epochs=2, ) model.residuals(ts)
def test_routine(start, end=None, length=None): gaussian_ts = gaussian_timeseries(start=start, end=end, length=length) self.assertEqual(len(gaussian_ts), length_assert)
def test_multiple_ts(self): lags = 4 lags_past_covariates = 3 model = RegressionModel(lags=lags, lags_past_covariates=lags_past_covariates) target_series = tg.linear_timeseries(start_value=0, end_value=49, length=50) past_covariates = tg.linear_timeseries(start_value=100, end_value=149, length=50) past_covariates = past_covariates.stack( tg.linear_timeseries(start_value=400, end_value=449, length=50)) target_train, target_test = target_series.split_after(0.7) past_covariates_train, past_covariates_test = past_covariates.split_after( 0.7) model.fit( series=[target_train, target_train + 0.5], past_covariates=[ past_covariates_train, past_covariates_train + 0.5 ], ) predictions = model.predict( 10, series=[target_train, target_train + 0.5], past_covariates=[past_covariates, past_covariates + 0.5], ) self.assertEqual(len(predictions[0]), 10, f"Found {len(predictions)} instead") # multiple TS, both future and past covariates, checking that both covariates lead to better results than # using a single one (target series = past_cov + future_cov + noise) np.random.seed(42) linear_ts_1 = tg.linear_timeseries(start_value=10, end_value=59, length=50) linear_ts_2 = tg.linear_timeseries(start_value=40, end_value=89, length=50) past_covariates = tg.sine_timeseries(length=50) * 10 future_covariates = ( tg.sine_timeseries(length=50, value_frequency=0.015) * 50) target_series_1 = linear_ts_1 + 4 * past_covariates + 2 * future_covariates target_series_2 = linear_ts_2 + 4 * past_covariates + 2 * future_covariates target_series_1_noise = (linear_ts_1 + 4 * past_covariates + 2 * future_covariates + tg.gaussian_timeseries(std=7, length=50)) target_series_2_noise = (linear_ts_2 + 4 * past_covariates + 2 * future_covariates + tg.gaussian_timeseries(std=7, length=50)) target_train_1, target_test_1 = target_series_1.split_after(0.7) target_train_2, target_test_2 = target_series_2.split_after(0.7) ( target_train_1_noise, target_test_1_noise, ) = target_series_1_noise.split_after(0.7) ( target_train_2_noise, target_test_2_noise, ) = target_series_2_noise.split_after(0.7) # testing improved denoise with multiple TS # test 1: with single TS, 2 covariates should be better than one model = RegressionModel(lags=3, lags_past_covariates=5) model.fit([target_train_1_noise], [past_covariates]) prediction_past_only = model.predict( n=len(target_test_1), series=[target_train_1_noise, target_train_2_noise], past_covariates=[past_covariates] * 2, ) model = RegressionModel(lags=3, lags_past_covariates=5, lags_future_covariates=(5, 0)) model.fit([target_train_1_noise], [past_covariates], [future_covariates]) prediction_past_and_future = model.predict( n=len(target_test_1), series=[target_train_1_noise, target_train_2_noise], past_covariates=[past_covariates] * 2, future_covariates=[future_covariates] * 2, ) error_past_only = rmse( [target_test_1, target_test_2], prediction_past_only, inter_reduction=np.mean, ) error_both = rmse( [target_test_1, target_test_2], prediction_past_and_future, inter_reduction=np.mean, ) self.assertTrue(error_past_only > error_both) # test 2: with both covariates, 2 TS should learn more than one (with little noise) model = RegressionModel(lags=3, lags_past_covariates=5, lags_future_covariates=(5, 0)) model.fit( [target_train_1_noise, target_train_2_noise], [past_covariates] * 2, [future_covariates] * 2, ) prediction_past_and_future_multi_ts = model.predict( n=len(target_test_1), series=[target_train_1_noise, target_train_2_noise], past_covariates=[past_covariates] * 2, future_covariates=[future_covariates] * 2, ) error_both_multi_ts = rmse( [target_test_1, target_test_2], prediction_past_and_future_multi_ts, inter_reduction=np.mean, ) self.assertTrue(error_both > error_both_multi_ts)