def evaluate(dataset_name, estimator): dataset = get_dataset(dataset_name) estimator = estimator( prediction_length=dataset.metadata.prediction_length, freq=dataset.metadata.time_granularity, ) print(f"evaluating {estimator} on {dataset}") predictor = estimator.train(dataset.train) forecast_it, ts_it = make_evaluation_predictions(dataset.test, predictor=predictor, num_eval_samples=100) agg_metrics, item_metrics = Evaluator()(ts_it, forecast_it, num_series=len(dataset.test)) pprint.pprint(agg_metrics) eval_dict = agg_metrics eval_dict["dataset"] = dataset_name eval_dict["estimator"] = type(estimator).__name__ return eval_dict
def run_example(): dataset = get_dataset("electricity") serialize_path = Path("GluonTSTabularPredictor") estimator = TabularEstimator( freq="H", prediction_length=24, time_limit=600, # ten minutes for training disable_auto_regression= True, # makes prediction faster, but potentially less accurate last_k_for_val= 24, # split the last 24 targets from each time series to be the validation data quantiles_to_predict=[0.1, 0.5, 0.9], ) n_train = 5 training_data = list(islice(dataset.train, n_train)) predictor = estimator.train(training_data=training_data) os.makedirs(serialize_path, exist_ok=True) predictor.serialize(serialize_path) predictor = None # the quantiles_to_predict parameters should be List[str] type predictor = Predictor.deserialize(serialize_path) forecasts = list(predictor.predict(training_data)) print(forecasts)
def run_example(): dataset = get_dataset("electricity") estimator = TabularEstimator( freq="H", prediction_length=24, time_limits=2 * 60, # two minutes for training disable_auto_regression= True, # makes prediction faster, but potentially less accurate ) n_train = 5 training_data = list(islice(dataset.train, n_train)) predictor = estimator.train(training_data=training_data, ) forecasts = list(predictor.predict(training_data)) for entry, forecast in zip(training_data, forecasts): ts = to_pandas(entry) plt.figure() plt.plot(ts[-7 * predictor.prediction_length:], label="target") forecast.plot() plt.show()
def test_forecasts(method_name): if method_name == "mlp": # https://stackoverflow.com/questions/56254321/error-in-ifncol-matrix-rep-argument-is-of-length-zero # https://cran.r-project.org/web/packages/neuralnet/index.html # published before the bug fix: https://github.com/bips-hb/neuralnet/pull/21 # The issue is still open on nnfor package: https://github.com/trnnick/nnfor/issues/8 # TODO: look for a workaround. pytest.xfail( "MLP currently does not work because " "the `neuralnet` package is not yet updated with a known bug fix in ` bips-hb/neuralnet`" ) dataset = datasets.get_dataset("constant") (train_dataset, test_dataset, metadata) = ( dataset.train, dataset.test, dataset.metadata, ) freq = metadata.freq prediction_length = metadata.prediction_length params = dict( freq=freq, prediction_length=prediction_length, method_name=method_name ) predictor = RForecastPredictor(**params) predictions = list(predictor.predict(train_dataset)) forecast_type = ( QuantileForecast if method_name in QUANTILE_FORECAST_METHODS else SampleForecast ) assert all( isinstance(prediction, forecast_type) for prediction in predictions ) assert all(prediction.freq == freq for prediction in predictions) assert all( prediction.prediction_length == prediction_length for prediction in predictions ) assert all( prediction.start_date == forecast_start(data) for data, prediction in zip(train_dataset, predictions) ) evaluator = Evaluator() agg_metrics, item_metrics = backtest_metrics( test_dataset=test_dataset, predictor=predictor, evaluator=evaluator, ) assert agg_metrics["mean_wQuantileLoss"] < TOLERANCE assert agg_metrics["NRMSE"] < TOLERANCE assert agg_metrics["RMSE"] < TOLERANCE
def run_example(): dataset = get_dataset("electricity") serialize_path = Path("GluonTSTabularPredictor") estimator = TabularEstimator( freq="H", prediction_length=24, time_limit=10, # two minutes for training disable_auto_regression=True, # makes prediction faster, but potentially less accurate last_k_for_val=24, # split the last 24 targets from each time series to be the validation data quantiles_to_predict=None, ) n_train = 5 training_data = list(islice(dataset.train, n_train)) predictor = estimator.train(training_data=training_data) os.makedirs(serialize_path, exist_ok=True) predictor.serialize(serialize_path) predictor = None predictor = Predictor.deserialize(serialize_path) forecasts = list(predictor.predict(training_data)) for entry, forecast in zip(training_data, forecasts): ts = to_pandas(entry) plt.figure() plt.plot(ts[-7 * predictor.prediction_length :], label="target") forecast.plot() plt.show()
def test_train_script(dataset_name, custom_dataset): # we need to write some data for this test, so we use a temporary directory with tempfile.TemporaryDirectory() as temp_dir: temp_dir_path = Path(temp_dir) dataset = get_dataset( dataset_name, path=temp_dir_path, regenerate=True ) # exchange_rate, m4_yearly # either use provided dataset, in which case it must be present in the directory, or a built in one # for testing we will provide a built in dataset as a custom one too if custom_dataset: args = create_arguments( str(temp_dir_path), dataset_name, s3_dataset_path=str(temp_dir_path / dataset_name), ) else: args = create_arguments(str(temp_dir_path), dataset_name) # the test requires using a deserialized estimator, which we first need to create estimator_cls, hyperparameters = simple_feedforward_estimator() estimator = estimator_cls.from_hyperparameters( prediction_length=dataset.metadata.prediction_length, freq=dataset.metadata.freq, **hyperparameters ) serialized = serde.dump_json(estimator) with open(temp_dir_path / "estimator.json", "w") as estimator_file: estimator_file.write(serialized) # No assert necessary, the idea is just that the code needs to run through train(args)
def simple_main(): import mxnet as mx from pprint import pprint dataset = get_dataset("electricity", regenerate=False) trainer = Trainer( ctx=mx.cpu(0), epochs=10, num_batches_per_epoch=200, learning_rate=1e-3, hybridize=False, ) cardinality = int(dataset.metadata.feat_static_cat[0].cardinality) estimator = DeepFactorEstimator( trainer=trainer, context_length=168, cardinality=[cardinality], prediction_length=dataset.metadata.prediction_length, freq=dataset.metadata.freq, ) predictor = estimator.train(dataset.train) forecast_it, ts_it = make_evaluation_predictions(dataset.test, predictor=predictor, num_eval_samples=100) agg_metrics, item_metrics = Evaluator()(ts_it, forecast_it, num_series=len(dataset.test)) pprint(agg_metrics)
def test_callbacks(): n_epochs = 4 history = TrainingHistory() iter_avg = ModelIterationAveraging(avg_strategy=NTA(epochs=2 * n_epochs)) dataset = "m4_hourly" dataset = get_dataset(dataset) prediction_length = dataset.metadata.prediction_length freq = dataset.metadata.freq estimator = SimpleFeedForwardEstimator( prediction_length=prediction_length, freq=freq, trainer=Trainer(epochs=n_epochs, callbacks=[history, iter_avg]), ) predictor = estimator.train(dataset.train, num_workers=None) assert len(history.loss_history) == n_epochs ws = WarmStart(predictor=predictor) estimator = SimpleFeedForwardEstimator( prediction_length=prediction_length, freq=freq, trainer=Trainer(epochs=n_epochs, callbacks=[history, iter_avg, ws]), ) predictor = estimator.train(dataset.train, num_workers=None) assert len(history.loss_history) == n_epochs * 2
def r_forecast_package(): import ast from gluonts.model.r_forecast import RForecastPredictor dataset = get_dataset("exchange_rate", regenerate=False) prediction_length = dataset.metadata.prediction_length freq = dataset.metadata.freq cardinality = ast.literal_eval(dataset.metadata.feat_static_cat[0].cardinality) train_ds = dataset.train test_ds = dataset.test #-------------------- # ETS. ets_predictor = RForecastPredictor( freq=freq, prediction_length=prediction_length, method_name="ets", ) ets_forecast = list(ets_predictor.predict(train_ds)) # ARIMA. arima_predictor = RForecastPredictor( freq=freq, prediction_length=prediction_length, method_name="arima", ) arima_forecast = list(arima_predictor.predict(train_ds))
def evaluate(dataset_name, estimator): dataset = get_dataset(dataset_name) estimator = estimator( prediction_length=dataset.metadata.prediction_length, freq=dataset.metadata.freq, use_feat_static_cat=True, cardinality=[ feat_static_cat.cardinality for feat_static_cat in dataset.metadata.feat_static_cat ], ) print(f"evaluating {estimator} on {dataset}") predictor = estimator.train(dataset.train) forecast_it, ts_it = make_evaluation_predictions(dataset.test, predictor=predictor, num_samples=100) agg_metrics, item_metrics = Evaluator()(ts_it, forecast_it, num_series=len(dataset.test)) pprint.pprint(agg_metrics) eval_dict = agg_metrics eval_dict["dataset"] = dataset_name eval_dict["estimator"] = type(estimator).__name__ return eval_dict
def test_data_leakage(name): try: dataset = get_dataset(name) except RuntimeError: print(f"WARN dataset '{name}' could not be obtained") check_train_test_split(dataset)
def test_torch_deepar(): constant = get_dataset("constant") estimator = DeepAREstimator( freq=constant.metadata.freq, prediction_length=constant.metadata.prediction_length, batch_size=4, num_batches_per_epoch=3, trainer_kwargs=dict(max_epochs=2), ) predictor = estimator.train( training_data=constant.train, validation_data=constant.train, shuffle_buffer_length=5, ) with tempfile.TemporaryDirectory() as td: predictor.serialize(Path(td)) predictor_copy = Predictor.deserialize(Path(td)) forecasts = predictor_copy.predict(constant.test) for f in islice(forecasts, 5): f.mean
def test_training_with_implicit_quantile_output(): dataset = get_dataset("constant") metadata = dataset.metadata deepar_estimator = DeepAREstimator( distr_output=ImplicitQuantileOutput(output_domain="Real"), freq=metadata.freq, prediction_length=metadata.prediction_length, trainer=Trainer( device="cpu", epochs=5, learning_rate=1e-3, num_batches_per_epoch=3, batch_size=256, ), input_size=15, ) deepar_predictor = deepar_estimator.train(dataset.train, num_workers=1) forecast_it, ts_it = make_evaluation_predictions( dataset=dataset.test, # test dataset predictor=deepar_predictor, # predictor num_samples=100, # number of sample paths we want for evaluation ) forecasts = list(forecast_it) tss = list(ts_it) evaluator = Evaluator(num_workers=0) agg_metrics, item_metrics = evaluator(iter(tss), iter(forecasts), num_series=len(dataset.test)) assert agg_metrics["MSE"] > 0
def load_dataset(dataset_name: str, path: Path) -> TrainDatasets: dataset = get_dataset(dataset_name, path, regenerate=False) target_dim = dataset.metadata.feat_static_cat[0].cardinality grouper_train = MultivariateGrouper(max_target_dim=target_dim) grouper_test = MultivariateGrouper(max_target_dim=target_dim) return TrainDatasets( metadata=dataset.metadata, train=grouper_train(dataset.train), test=grouper_test(dataset.test), )
def train(arguments): """ Generic train method that trains a specified estimator on a specified dataset. """ logger.info("Downloading estimator config.") estimator_config = Path(arguments.estimator) / "estimator.json" with estimator_config.open() as config_file: estimator = serde.load_json(config_file.read()) logger.info("Downloading dataset.") if arguments.s3_dataset is None: # load built in dataset dataset = datasets.get_dataset(arguments.dataset) else: # load custom dataset s3_dataset_dir = Path(arguments.s3_dataset) dataset = common.load_datasets( metadata=s3_dataset_dir, train=s3_dataset_dir / "train", test=s3_dataset_dir / "test", ) logger.info("Starting model training.") predictor = estimator.train(dataset.train) forecast_it, ts_it = backtest.make_evaluation_predictions( dataset=dataset.test, predictor=predictor, num_samples=int(arguments.num_samples), ) logger.info("Starting model evaluation.") evaluator = Evaluator(quantiles=eval(arguments.quantiles)) agg_metrics, item_metrics = evaluator(ts_it, forecast_it, num_series=len(list(dataset.test))) # required for metric tracking. for name, value in agg_metrics.items(): logger.info(f"gluonts[metric-{name}]: {value}") # save the evaluation results metrics_output_dir = Path(arguments.output_data_dir) with open(metrics_output_dir / "agg_metrics.json", "w") as f: json.dump(agg_metrics, f) with open(metrics_output_dir / "item_metrics.csv", "w") as f: item_metrics.to_csv(f, index=False) # save the model model_output_dir = Path(arguments.model_dir) predictor.serialize(model_output_dir)
def load(self, frequency: str, subset_filter: str, training: bool) -> ListDataset: """ Load M4 subset :param frequency: :param subset_filter: :param training: :return: """ dataset = get_dataset(f'm4_{subset_filter}') return ListDataset(list(dataset.train if training else dataset.test), freq=frequency)
def load_dataset(args: Namespace) -> TrainDatasets: """Load data from channel or fallback to named public dataset.""" if args.s3_dataset is None: # load built in dataset logger.info("Downloading dataset %s", args.dataset) dataset = datasets.get_dataset(args.dataset) else: # load custom dataset logger.info("Loading dataset from %s", args.s3_dataset) s3_dataset_dir = Path(args.s3_dataset) dataset = load_datasets( metadata=s3_dataset_dir / "metadata", train=s3_dataset_dir / "train", test=s3_dataset_dir / "test", ) return dataset
def build_deepar_model(): # get the financial data "exchange_rate" gluon_data = get_dataset("exchange_rate", regenerate=True) train_data = next(iter(gluon_data.train)) test_data = next(iter(gluon_data.test)) meta_data = gluon_data.metadata # data set visualisation fig, ax = plt.subplots(1, 1, figsize=(11, 8)) to_pandas(train_data).plot(ax=ax) ax.grid(which="both") ax.legend(["train data"], loc="upper left") plt.savefig("dataset.png") # visualize various members of the 'gluon_data.*' print(train_data.keys()) print(test_data.keys()) print(meta_data) # convert dataset into an object recognised by GluonTS training_data = common.ListDataset(gluon_data.train, freq=meta_data.freq) testing_data = common.ListDataset(gluon_data.test, freq=meta_data.freq) # create an Estimator with DeepAR # an object of Trainer() class is used to customize Estimator estimator = deepar.DeepAREstimator( freq=meta_data.freq, prediction_length=meta_data.prediction_length, trainer=Trainer(ctx="cpu", epochs=100, learning_rate=1e-4)) # create a Predictor by training the Estimator with training dataset predictor = estimator.train(training_data=training_data) # make predictions forecasts, test_series = make_evaluation_predictions(dataset=testing_data, predictor=predictor, num_samples=10) # visualise forecasts prediction_intervals = (50.0, 90.0) legend = ["actual data", "median forecast" ] + [f"{k}% forecast interval" for k in prediction_intervals][::-1] fig, ax = plt.subplots(1, 1, figsize=(11, 8)) list(test_series)[0][-150:].plot(ax=ax) # plot the time series list(forecasts)[0].plot(prediction_intervals=prediction_intervals, color='r') plt.grid(which="both") plt.legend(legend, loc="upper left") plt.savefig("deepar-model.png")
def npts_test(): import ast from gluonts.model.npts import NPTSPredictor dataset = get_dataset("exchange_rate", regenerate=False) prediction_length = dataset.metadata.prediction_length freq = dataset.metadata.freq cardinality = ast.literal_eval(dataset.metadata.feat_static_cat[0].cardinality) train_ds = dataset.train test_ds = dataset.test npts_predictor = NPTSPredictor(freq=freq, prediction_length=prediction_length, context_length=300, kernel_type="uniform", use_seasonal_model=False) npts_forecast = list(npts_predictor.predict(train_ds))
def data_loader(estimator, dataset, batch): dataset = get_dataset(dataset) data = dataset.train epochs = 5 batch_size = batch num_batches_per_epoch = 10 bin_edges = np.array([-1e20, -1e10, 1, 1e20]) transform = estimator.create_transformation(bin_edges=bin_edges, pred_length=dataset.metadata.prediction_length)\ if estimator.__class__.__name__ == 'WaveNetEstimator' else estimator.create_transformation() loader = TrainDataLoader( data, transform=transform, batch_size=batch_size, ctx=mx.cpu(), num_batches_per_epoch=num_batches_per_epoch, )
def test_estimator_constant_dataset(estimator_constructor): constant = get_dataset("constant") estimator = estimator_constructor(constant) predictor = estimator.train( training_data=constant.train, validation_data=constant.train, shuffle_buffer_length=5, ) with tempfile.TemporaryDirectory() as td: predictor.serialize(Path(td)) predictor_copy = Predictor.deserialize(Path(td)) forecasts = predictor_copy.predict(constant.test) for f in islice(forecasts, 5): if isinstance(f, DistributionForecast): f = f.to_sample_forecast() f.mean
def test_splitter(): dataset = get_dataset("m4_hourly") prediction_length = dataset.metadata.prediction_length splitter = DateSplitter( prediction_length=prediction_length, split_date=pd.Timestamp("1750-01-05 04:00:00", freq="h"), ) train, validation = splitter.split(dataset.train) assert len(train[1][0][FieldName.TARGET]) + prediction_length == len( validation[1][0][FieldName.TARGET] ) max_history = 2 * prediction_length splitter = OffsetSplitter( prediction_length=prediction_length, split_offset=4 * prediction_length, max_history=max_history, ) train, validation = splitter.split(dataset.train) assert len(validation[1][0][FieldName.TARGET]) == max_history assert len(train[1][0][FieldName.TARGET]) == 4 * prediction_length train, validation = splitter.rolling_split(dataset.train, windows=3) for i in range(3): assert len(validation[1][i][FieldName.TARGET]) == max_history assert len(train[1][i][FieldName.TARGET]) == 4 * prediction_length max_history = 2 * prediction_length splitter = DateSplitter( prediction_length=prediction_length, split_date=pd.Timestamp("1750-01-05 04:00:00", freq="h"), max_history=max_history, ) train, validation = splitter.split(dataset.train) assert len(validation[1][0][FieldName.TARGET]) == max_history train, validation = splitter.rolling_split(dataset.train, windows=3) for i in range(3): assert len(validation[1][i][FieldName.TARGET]) == max_history
from gluonts.model.simple_feedforward import SimpleFeedForwardEstimator from gluonts.trainer import Trainer from gluonts.dataset.repository.datasets import get_dataset import matplotlib.pyplot as plt from gluonts.evaluation import Evaluator import json dataset = get_dataset("m4_hourly", regenerate=True) # model estimator = SimpleFeedForwardEstimator( num_hidden_dimensions=[10], prediction_length=dataset.metadata.prediction_length, context_length=100, freq=dataset.metadata.freq, trainer=Trainer(ctx="cpu", epochs=5, learning_rate=1e-3, num_batches_per_epoch=100), ) # train predictor = estimator.train(dataset.train) # test from gluonts.evaluation.backtest import make_evaluation_predictions forecast_it, ts_it = make_evaluation_predictions( dataset=dataset.test, # test dataset
This example shows how to serialize and deserialize a model """ import os import pprint from gluonts.dataset.repository.datasets import get_dataset from gluonts.evaluation import Evaluator from gluonts.evaluation.backtest import make_evaluation_predictions from gluonts.model.simple_feedforward import SimpleFeedForwardEstimator from gluonts.support.util import get_download_path from gluonts.trainer import Trainer from gluonts.model.predictor import Predictor if __name__ == "__main__": dataset = get_dataset("exchange_rate") estimator = SimpleFeedForwardEstimator( prediction_length=dataset.metadata.prediction_length, freq=dataset.metadata.freq, trainer=Trainer(epochs=5, num_batches_per_epoch=10), ) predictor = estimator.train(dataset.train) # save the trained model in a path ~/.mxnet/gluon-ts/feedforward/ # or $MXNET_HOME/feedforward if MXNET_HOME is defined model_path = get_download_path() / "feedforward" os.makedirs(model_path, exist_ok=True) predictor.serialize(model_path)
""" import numpy as np from itertools import islice import mxnet as mx import matplotlib.pyplot as plt import pandas as pd from gluonts.dataset.loader import TrainDataLoader from gluonts.model.deepar import DeepAREstimator from gluonts.support.util import get_hybrid_forward_input_names from gluonts.trainer import Trainer from gluonts.dataset.repository.datasets import get_dataset if __name__ == '__main__': dataset = get_dataset(dataset_name="electricity") estimator = DeepAREstimator( prediction_length=dataset.metadata.prediction_length, freq=dataset.metadata.time_granularity, trainer=Trainer(learning_rate=1e-3, epochs=50, num_batches_per_epoch=100), ) # instead of calling `train` method, we call `train_model` that returns more things including the training model train_output = estimator.train_model(dataset.train) # we construct a data_entry that contains 500 random windows batch_size = 500 num_samples = 100
This example shows how to fit a model and evaluate its predictions. """ import pprint from gluonts.dataset.repository.datasets import get_dataset, dataset_recipes from gluonts.evaluation import Evaluator from gluonts.evaluation.backtest import make_evaluation_predictions from gluonts.model.simple_feedforward import SimpleFeedForwardEstimator from gluonts.trainer import Trainer if __name__ == "__main__": print(f"datasets available: {dataset_recipes.keys()}") # we pick m4_hourly as it only contains a few hundred time series dataset = get_dataset("m4_hourly", regenerate=False) estimator = SimpleFeedForwardEstimator( prediction_length=dataset.metadata.prediction_length, freq=dataset.metadata.freq, trainer=Trainer(epochs=5, num_batches_per_epoch=10), ) predictor = estimator.train(dataset.train) forecast_it, ts_it = make_evaluation_predictions(dataset.test, predictor=predictor, num_eval_samples=100) agg_metrics, item_metrics = Evaluator()(ts_it, forecast_it,
import io from data_store import * # import pandas as pd from gluonts.dataset.repository.datasets import get_dataset from gluonts.model.deepar import DeepAREstimator from gluonts.model.seq2seq import MQCNNEstimator from gluonts.model.simple_feedforward import SimpleFeedForwardEstimator from gluonts.model.wavenet import WaveNetEstimator from run_data_loader import data_loader import numpy as np datasets = [ 'm4_daily', 'm4_hourly', 'm4_weekly', 'solar-energy', 'electricity' ] for ds in datasets: dataset = get_dataset(ds) freq = dataset.metadata.freq prediction_length = dataset.metadata.prediction_length deepar = DeepAREstimator( freq=freq, prediction_length=prediction_length, ) mqcnn = MQCNNEstimator( freq=freq, prediction_length=prediction_length, ) sff = SimpleFeedForwardEstimator( num_hidden_dimensions=[10], context_length=100,
freq="1D", hidden_dim=hidden_dims, prediction_length=prediction_length, context_length=context_length, input_size=1 + exog_dim, ) output = net(input_tensor, past_exog_features, future_exog_features) assert output.shape == (batch_size, 1, prediction_length) net.eval() output = net(input_tensor, past_exog_features, future_exog_features) assert output.shape == (batch_size, 1, prediction_length) dataset = get_dataset("electricity") freq = "1H" context_length = 2 * 7 * 24 prediction_length = dataset.metadata.prediction_length hidden_dimensions = [64] batch_size = 16 num_batches_per_epoch = 100 ASSETS_PATH = pathlib.Path(__file__).parent / "assets" model = RNNNetEstimator( freq=freq, prediction_length=prediction_length, context_length=context_length,
for m, v in evaluation.items() if m in metrics_persisted } evaluation["dataset"] = dataset evaluation["estimator"] = estimator_name with open(path, 'w') as f: f.write(json.dumps(evaluation, indent=4, sort_keys=True)) if __name__ == "__main__": for dataset_name in datasets: for Estimator in Estimators: dataset = get_dataset( dataset_name=dataset_name, regenerate=False, path="../datasets/", ) estimator = Estimator( prediction_length=dataset.metadata.prediction_length, freq=dataset.metadata.time_granularity, ) estimator_name = type(estimator).__name__ print(f"evaluating {estimator_name} on {dataset_name}") agg_metrics, item_metrics = backtest_metrics( train_dataset=dataset.train, test_dataset=dataset.test,
def test_instanciation_of_args_proj(): class MockedImplicitQuantileOutput(ImplicitQuantileOutput): method_calls = 0 @classmethod def set_args_proj(cls): super().set_args_proj() cls.method_calls += 1 dataset = get_dataset("constant") metadata = dataset.metadata distr_output = MockedImplicitQuantileOutput(output_domain="Real") deepar_estimator = DeepAREstimator( distr_output=distr_output, freq=metadata.freq, prediction_length=metadata.prediction_length, trainer=Trainer( device="cpu", epochs=3, learning_rate=1e-3, num_batches_per_epoch=1, batch_size=256, ), input_size=15, ) assert distr_output.method_calls == 1 deepar_predictor = deepar_estimator.train(dataset.train, num_workers=1) # Method should be called when the MockedImplicitQuantileOutput is instanciated, # and one more time because in_features is different from 1 assert distr_output.method_calls == 2 forecast_it, ts_it = make_evaluation_predictions( dataset=dataset.test, # test dataset predictor=deepar_predictor, # predictor num_samples=100, # number of sample paths we want for evaluation ) forecasts = list(forecast_it) tss = list(ts_it) evaluator = Evaluator(num_workers=0) agg_metrics, item_metrics = evaluator(iter(tss), iter(forecasts), num_series=len(dataset.test)) assert distr_output.method_calls == 2 # Test that the implicit output module is proper reset new_estimator = DeepAREstimator( distr_output=MockedImplicitQuantileOutput(output_domain="Real"), freq=metadata.freq, prediction_length=metadata.prediction_length, trainer=Trainer( device="cpu", epochs=3, learning_rate=1e-3, num_batches_per_epoch=1, batch_size=256, ), input_size=15, ) assert distr_output.method_calls == 3 new_estimator.train(dataset.train, num_workers=1) assert ( distr_output.method_calls == 3 ) # Since in_feature is the same as before, there should be no additional call