def evaluate(dataset_name, estimator):
dataset = get_dataset(dataset_name)
estimator = estimator(
prediction_length=dataset.metadata.prediction_length,
freq=dataset.metadata.time_granularity,
)
print(f"evaluating {estimator} on {dataset}")
predictor = estimator.train(dataset.train)
forecast_it, ts_it = make_evaluation_predictions(dataset.test,
predictor=predictor,
num_eval_samples=100)
agg_metrics, item_metrics = Evaluator()(ts_it,
forecast_it,
num_series=len(dataset.test))
pprint.pprint(agg_metrics)
eval_dict = agg_metrics
eval_dict["dataset"] = dataset_name
eval_dict["estimator"] = type(estimator).__name__
return eval_dict
def run_example():
dataset = get_dataset("electricity")
serialize_path = Path("GluonTSTabularPredictor")
estimator = TabularEstimator(
freq="H",
prediction_length=24,
time_limit=600, # ten minutes for training
disable_auto_regression=
True, # makes prediction faster, but potentially less accurate
last_k_for_val=
24, # split the last 24 targets from each time series to be the validation data
quantiles_to_predict=[0.1, 0.5, 0.9],
)
n_train = 5
training_data = list(islice(dataset.train, n_train))
predictor = estimator.train(training_data=training_data)
os.makedirs(serialize_path, exist_ok=True)
predictor.serialize(serialize_path)
predictor = None
# the quantiles_to_predict parameters should be List[str] type
predictor = Predictor.deserialize(serialize_path)
forecasts = list(predictor.predict(training_data))
print(forecasts)
def run_example():
dataset = get_dataset("electricity")
estimator = TabularEstimator(
freq="H",
prediction_length=24,
time_limits=2 * 60, # two minutes for training
disable_auto_regression=
True, # makes prediction faster, but potentially less accurate
)
n_train = 5
training_data = list(islice(dataset.train, n_train))
predictor = estimator.train(training_data=training_data, )
forecasts = list(predictor.predict(training_data))
for entry, forecast in zip(training_data, forecasts):
ts = to_pandas(entry)
plt.figure()
plt.plot(ts[-7 * predictor.prediction_length:], label="target")
forecast.plot()
plt.show()
def test_forecasts(method_name):
if method_name == "mlp":
# https://stackoverflow.com/questions/56254321/error-in-ifncol-matrix-rep-argument-is-of-length-zero
# https://cran.r-project.org/web/packages/neuralnet/index.html
# published before the bug fix: https://github.com/bips-hb/neuralnet/pull/21
# The issue is still open on nnfor package: https://github.com/trnnick/nnfor/issues/8
# TODO: look for a workaround.
pytest.xfail(
"MLP currently does not work because "
"the `neuralnet` package is not yet updated with a known bug fix in ` bips-hb/neuralnet`"
)
dataset = datasets.get_dataset("constant")
(train_dataset, test_dataset, metadata) = (
dataset.train,
dataset.test,
dataset.metadata,
)
freq = metadata.freq
prediction_length = metadata.prediction_length
params = dict(
freq=freq, prediction_length=prediction_length, method_name=method_name
)
predictor = RForecastPredictor(**params)
predictions = list(predictor.predict(train_dataset))
forecast_type = (
QuantileForecast
if method_name in QUANTILE_FORECAST_METHODS
else SampleForecast
)
assert all(
isinstance(prediction, forecast_type) for prediction in predictions
)
assert all(prediction.freq == freq for prediction in predictions)
assert all(
prediction.prediction_length == prediction_length
for prediction in predictions
)
assert all(
prediction.start_date == forecast_start(data)
for data, prediction in zip(train_dataset, predictions)
)
evaluator = Evaluator()
agg_metrics, item_metrics = backtest_metrics(
test_dataset=test_dataset,
predictor=predictor,
evaluator=evaluator,
)
assert agg_metrics["mean_wQuantileLoss"] < TOLERANCE
assert agg_metrics["NRMSE"] < TOLERANCE
assert agg_metrics["RMSE"] < TOLERANCE
def run_example():
dataset = get_dataset("electricity")
serialize_path = Path("GluonTSTabularPredictor")
estimator = TabularEstimator(
freq="H",
prediction_length=24,
time_limit=10, # two minutes for training
disable_auto_regression=True, # makes prediction faster, but potentially less accurate
last_k_for_val=24, # split the last 24 targets from each time series to be the validation data
quantiles_to_predict=None,
)
n_train = 5
training_data = list(islice(dataset.train, n_train))
predictor = estimator.train(training_data=training_data)
os.makedirs(serialize_path, exist_ok=True)
predictor.serialize(serialize_path)
predictor = None
predictor = Predictor.deserialize(serialize_path)
forecasts = list(predictor.predict(training_data))
for entry, forecast in zip(training_data, forecasts):
ts = to_pandas(entry)
plt.figure()
plt.plot(ts[-7 * predictor.prediction_length :], label="target")
forecast.plot()
plt.show()
def test_train_script(dataset_name, custom_dataset):
# we need to write some data for this test, so we use a temporary directory
with tempfile.TemporaryDirectory() as temp_dir:
temp_dir_path = Path(temp_dir)
dataset = get_dataset(
dataset_name, path=temp_dir_path, regenerate=True
) # exchange_rate, m4_yearly
# either use provided dataset, in which case it must be present in the directory, or a built in one
# for testing we will provide a built in dataset as a custom one too
if custom_dataset:
args = create_arguments(
str(temp_dir_path),
dataset_name,
s3_dataset_path=str(temp_dir_path / dataset_name),
)
else:
args = create_arguments(str(temp_dir_path), dataset_name)
# the test requires using a deserialized estimator, which we first need to create
estimator_cls, hyperparameters = simple_feedforward_estimator()
estimator = estimator_cls.from_hyperparameters(
prediction_length=dataset.metadata.prediction_length,
freq=dataset.metadata.freq,
**hyperparameters
)
serialized = serde.dump_json(estimator)
with open(temp_dir_path / "estimator.json", "w") as estimator_file:
estimator_file.write(serialized)
# No assert necessary, the idea is just that the code needs to run through
train(args)
def simple_main():
import mxnet as mx
from pprint import pprint
dataset = get_dataset("electricity", regenerate=False)
trainer = Trainer(
ctx=mx.cpu(0),
epochs=10,
num_batches_per_epoch=200,
learning_rate=1e-3,
hybridize=False,
)
cardinality = int(dataset.metadata.feat_static_cat[0].cardinality)
estimator = DeepFactorEstimator(
trainer=trainer,
context_length=168,
cardinality=[cardinality],
prediction_length=dataset.metadata.prediction_length,
freq=dataset.metadata.freq,
)
predictor = estimator.train(dataset.train)
forecast_it, ts_it = make_evaluation_predictions(dataset.test,
predictor=predictor,
num_eval_samples=100)
agg_metrics, item_metrics = Evaluator()(ts_it,
forecast_it,
num_series=len(dataset.test))
pprint(agg_metrics)
def test_callbacks():
n_epochs = 4
history = TrainingHistory()
iter_avg = ModelIterationAveraging(avg_strategy=NTA(epochs=2 * n_epochs))
dataset = "m4_hourly"
dataset = get_dataset(dataset)
prediction_length = dataset.metadata.prediction_length
freq = dataset.metadata.freq
estimator = SimpleFeedForwardEstimator(
prediction_length=prediction_length,
freq=freq,
trainer=Trainer(epochs=n_epochs, callbacks=[history, iter_avg]),
)
predictor = estimator.train(dataset.train, num_workers=None)
assert len(history.loss_history) == n_epochs
ws = WarmStart(predictor=predictor)
estimator = SimpleFeedForwardEstimator(
prediction_length=prediction_length,
freq=freq,
trainer=Trainer(epochs=n_epochs, callbacks=[history, iter_avg, ws]),
)
predictor = estimator.train(dataset.train, num_workers=None)
assert len(history.loss_history) == n_epochs * 2
def r_forecast_package():
import ast
from gluonts.model.r_forecast import RForecastPredictor
dataset = get_dataset("exchange_rate", regenerate=False)
prediction_length = dataset.metadata.prediction_length
freq = dataset.metadata.freq
cardinality = ast.literal_eval(dataset.metadata.feat_static_cat[0].cardinality)
train_ds = dataset.train
test_ds = dataset.test
#--------------------
# ETS.
ets_predictor = RForecastPredictor(
freq=freq,
prediction_length=prediction_length,
method_name="ets",
)
ets_forecast = list(ets_predictor.predict(train_ds))
# ARIMA.
arima_predictor = RForecastPredictor(
freq=freq,
prediction_length=prediction_length,
method_name="arima",
)
arima_forecast = list(arima_predictor.predict(train_ds))
def evaluate(dataset_name, estimator):
dataset = get_dataset(dataset_name)
estimator = estimator(
prediction_length=dataset.metadata.prediction_length,
freq=dataset.metadata.freq,
use_feat_static_cat=True,
cardinality=[
feat_static_cat.cardinality
for feat_static_cat in dataset.metadata.feat_static_cat
],
)
print(f"evaluating {estimator} on {dataset}")
predictor = estimator.train(dataset.train)
forecast_it, ts_it = make_evaluation_predictions(dataset.test,
predictor=predictor,
num_samples=100)
agg_metrics, item_metrics = Evaluator()(ts_it,
forecast_it,
num_series=len(dataset.test))
pprint.pprint(agg_metrics)
eval_dict = agg_metrics
eval_dict["dataset"] = dataset_name
eval_dict["estimator"] = type(estimator).__name__
return eval_dict
def test_data_leakage(name):
try:
dataset = get_dataset(name)
except RuntimeError:
print(f"WARN dataset '{name}' could not be obtained")
check_train_test_split(dataset)
def test_torch_deepar():
constant = get_dataset("constant")
estimator = DeepAREstimator(
freq=constant.metadata.freq,
prediction_length=constant.metadata.prediction_length,
batch_size=4,
num_batches_per_epoch=3,
trainer_kwargs=dict(max_epochs=2),
)
predictor = estimator.train(
training_data=constant.train,
validation_data=constant.train,
shuffle_buffer_length=5,
)
with tempfile.TemporaryDirectory() as td:
predictor.serialize(Path(td))
predictor_copy = Predictor.deserialize(Path(td))
forecasts = predictor_copy.predict(constant.test)
for f in islice(forecasts, 5):
f.mean
def test_training_with_implicit_quantile_output():
dataset = get_dataset("constant")
metadata = dataset.metadata
deepar_estimator = DeepAREstimator(
distr_output=ImplicitQuantileOutput(output_domain="Real"),
freq=metadata.freq,
prediction_length=metadata.prediction_length,
trainer=Trainer(
device="cpu",
epochs=5,
learning_rate=1e-3,
num_batches_per_epoch=3,
batch_size=256,
),
input_size=15,
)
deepar_predictor = deepar_estimator.train(dataset.train, num_workers=1)
forecast_it, ts_it = make_evaluation_predictions(
dataset=dataset.test, # test dataset
predictor=deepar_predictor, # predictor
num_samples=100, # number of sample paths we want for evaluation
)
forecasts = list(forecast_it)
tss = list(ts_it)
evaluator = Evaluator(num_workers=0)
agg_metrics, item_metrics = evaluator(iter(tss),
iter(forecasts),
num_series=len(dataset.test))
assert agg_metrics["MSE"] > 0
def load_dataset(dataset_name: str, path: Path) -> TrainDatasets:
dataset = get_dataset(dataset_name, path, regenerate=False)
target_dim = dataset.metadata.feat_static_cat[0].cardinality
grouper_train = MultivariateGrouper(max_target_dim=target_dim)
grouper_test = MultivariateGrouper(max_target_dim=target_dim)
return TrainDatasets(
metadata=dataset.metadata,
train=grouper_train(dataset.train),
test=grouper_test(dataset.test),
)
def train(arguments):
"""
Generic train method that trains a specified estimator on a specified
dataset.
"""
logger.info("Downloading estimator config.")
estimator_config = Path(arguments.estimator) / "estimator.json"
with estimator_config.open() as config_file:
estimator = serde.load_json(config_file.read())
logger.info("Downloading dataset.")
if arguments.s3_dataset is None:
# load built in dataset
dataset = datasets.get_dataset(arguments.dataset)
else:
# load custom dataset
s3_dataset_dir = Path(arguments.s3_dataset)
dataset = common.load_datasets(
metadata=s3_dataset_dir,
train=s3_dataset_dir / "train",
test=s3_dataset_dir / "test",
)
logger.info("Starting model training.")
predictor = estimator.train(dataset.train)
forecast_it, ts_it = backtest.make_evaluation_predictions(
dataset=dataset.test,
predictor=predictor,
num_samples=int(arguments.num_samples),
)
logger.info("Starting model evaluation.")
evaluator = Evaluator(quantiles=eval(arguments.quantiles))
agg_metrics, item_metrics = evaluator(ts_it,
forecast_it,
num_series=len(list(dataset.test)))
# required for metric tracking.
for name, value in agg_metrics.items():
logger.info(f"gluonts[metric-{name}]: {value}")
# save the evaluation results
metrics_output_dir = Path(arguments.output_data_dir)
with open(metrics_output_dir / "agg_metrics.json", "w") as f:
json.dump(agg_metrics, f)
with open(metrics_output_dir / "item_metrics.csv", "w") as f:
item_metrics.to_csv(f, index=False)
# save the model
model_output_dir = Path(arguments.model_dir)
predictor.serialize(model_output_dir)
def load(self, frequency: str, subset_filter: str,
training: bool) -> ListDataset:
"""
Load M4 subset
:param frequency:
:param subset_filter:
:param training:
:return:
"""
dataset = get_dataset(f'm4_{subset_filter}')
return ListDataset(list(dataset.train if training else dataset.test),
freq=frequency)
def load_dataset(args: Namespace) -> TrainDatasets:
"""Load data from channel or fallback to named public dataset."""
if args.s3_dataset is None:
# load built in dataset
logger.info("Downloading dataset %s", args.dataset)
dataset = datasets.get_dataset(args.dataset)
else:
# load custom dataset
logger.info("Loading dataset from %s", args.s3_dataset)
s3_dataset_dir = Path(args.s3_dataset)
dataset = load_datasets(
metadata=s3_dataset_dir / "metadata", train=s3_dataset_dir / "train", test=s3_dataset_dir / "test",
)
return dataset
def build_deepar_model():
# get the financial data "exchange_rate"
gluon_data = get_dataset("exchange_rate", regenerate=True)
train_data = next(iter(gluon_data.train))
test_data = next(iter(gluon_data.test))
meta_data = gluon_data.metadata
# data set visualisation
fig, ax = plt.subplots(1, 1, figsize=(11, 8))
to_pandas(train_data).plot(ax=ax)
ax.grid(which="both")
ax.legend(["train data"], loc="upper left")
plt.savefig("dataset.png")
# visualize various members of the 'gluon_data.*'
print(train_data.keys())
print(test_data.keys())
print(meta_data)
# convert dataset into an object recognised by GluonTS
training_data = common.ListDataset(gluon_data.train, freq=meta_data.freq)
testing_data = common.ListDataset(gluon_data.test, freq=meta_data.freq)
# create an Estimator with DeepAR
# an object of Trainer() class is used to customize Estimator
estimator = deepar.DeepAREstimator(
freq=meta_data.freq,
prediction_length=meta_data.prediction_length,
trainer=Trainer(ctx="cpu", epochs=100, learning_rate=1e-4))
# create a Predictor by training the Estimator with training dataset
predictor = estimator.train(training_data=training_data)
# make predictions
forecasts, test_series = make_evaluation_predictions(dataset=testing_data,
predictor=predictor,
num_samples=10)
# visualise forecasts
prediction_intervals = (50.0, 90.0)
legend = ["actual data", "median forecast"
] + [f"{k}% forecast interval"
for k in prediction_intervals][::-1]
fig, ax = plt.subplots(1, 1, figsize=(11, 8))
list(test_series)[0][-150:].plot(ax=ax) # plot the time series
list(forecasts)[0].plot(prediction_intervals=prediction_intervals,
color='r')
plt.grid(which="both")
plt.legend(legend, loc="upper left")
plt.savefig("deepar-model.png")
def npts_test():
import ast
from gluonts.model.npts import NPTSPredictor
dataset = get_dataset("exchange_rate", regenerate=False)
prediction_length = dataset.metadata.prediction_length
freq = dataset.metadata.freq
cardinality = ast.literal_eval(dataset.metadata.feat_static_cat[0].cardinality)
train_ds = dataset.train
test_ds = dataset.test
npts_predictor = NPTSPredictor(freq=freq, prediction_length=prediction_length, context_length=300, kernel_type="uniform", use_seasonal_model=False)
npts_forecast = list(npts_predictor.predict(train_ds))
def data_loader(estimator, dataset, batch):
dataset = get_dataset(dataset)
data = dataset.train
epochs = 5
batch_size = batch
num_batches_per_epoch = 10
bin_edges = np.array([-1e20, -1e10, 1, 1e20])
transform = estimator.create_transformation(bin_edges=bin_edges, pred_length=dataset.metadata.prediction_length)\
if estimator.__class__.__name__ == 'WaveNetEstimator' else estimator.create_transformation()
loader = TrainDataLoader(
data,
transform=transform,
batch_size=batch_size,
ctx=mx.cpu(),
num_batches_per_epoch=num_batches_per_epoch,
)
def test_estimator_constant_dataset(estimator_constructor):
constant = get_dataset("constant")
estimator = estimator_constructor(constant)
predictor = estimator.train(
training_data=constant.train,
validation_data=constant.train,
shuffle_buffer_length=5,
)
with tempfile.TemporaryDirectory() as td:
predictor.serialize(Path(td))
predictor_copy = Predictor.deserialize(Path(td))
forecasts = predictor_copy.predict(constant.test)
for f in islice(forecasts, 5):
if isinstance(f, DistributionForecast):
f = f.to_sample_forecast()
f.mean
def test_splitter():
dataset = get_dataset("m4_hourly")
prediction_length = dataset.metadata.prediction_length
splitter = DateSplitter(
prediction_length=prediction_length,
split_date=pd.Timestamp("1750-01-05 04:00:00", freq="h"),
)
train, validation = splitter.split(dataset.train)
assert len(train[1][0][FieldName.TARGET]) + prediction_length == len(
validation[1][0][FieldName.TARGET]
)
max_history = 2 * prediction_length
splitter = OffsetSplitter(
prediction_length=prediction_length,
split_offset=4 * prediction_length,
max_history=max_history,
)
train, validation = splitter.split(dataset.train)
assert len(validation[1][0][FieldName.TARGET]) == max_history
assert len(train[1][0][FieldName.TARGET]) == 4 * prediction_length
train, validation = splitter.rolling_split(dataset.train, windows=3)
for i in range(3):
assert len(validation[1][i][FieldName.TARGET]) == max_history
assert len(train[1][i][FieldName.TARGET]) == 4 * prediction_length
max_history = 2 * prediction_length
splitter = DateSplitter(
prediction_length=prediction_length,
split_date=pd.Timestamp("1750-01-05 04:00:00", freq="h"),
max_history=max_history,
)
train, validation = splitter.split(dataset.train)
assert len(validation[1][0][FieldName.TARGET]) == max_history
train, validation = splitter.rolling_split(dataset.train, windows=3)
for i in range(3):
assert len(validation[1][i][FieldName.TARGET]) == max_history
from gluonts.model.simple_feedforward import SimpleFeedForwardEstimator
from gluonts.trainer import Trainer
from gluonts.dataset.repository.datasets import get_dataset
import matplotlib.pyplot as plt
from gluonts.evaluation import Evaluator
import json
dataset = get_dataset("m4_hourly", regenerate=True)
# model
estimator = SimpleFeedForwardEstimator(
num_hidden_dimensions=[10],
prediction_length=dataset.metadata.prediction_length,
context_length=100,
freq=dataset.metadata.freq,
trainer=Trainer(ctx="cpu",
epochs=5,
learning_rate=1e-3,
num_batches_per_epoch=100),
)
# train
predictor = estimator.train(dataset.train)
# test
from gluonts.evaluation.backtest import make_evaluation_predictions
forecast_it, ts_it = make_evaluation_predictions(
dataset=dataset.test, # test dataset
This example shows how to serialize and deserialize a model
"""
import os
import pprint
from gluonts.dataset.repository.datasets import get_dataset
from gluonts.evaluation import Evaluator
from gluonts.evaluation.backtest import make_evaluation_predictions
from gluonts.model.simple_feedforward import SimpleFeedForwardEstimator
from gluonts.support.util import get_download_path
from gluonts.trainer import Trainer
from gluonts.model.predictor import Predictor
if __name__ == "__main__":
dataset = get_dataset("exchange_rate")
estimator = SimpleFeedForwardEstimator(
prediction_length=dataset.metadata.prediction_length,
freq=dataset.metadata.freq,
trainer=Trainer(epochs=5, num_batches_per_epoch=10),
)
predictor = estimator.train(dataset.train)
# save the trained model in a path ~/.mxnet/gluon-ts/feedforward/
# or $MXNET_HOME/feedforward if MXNET_HOME is defined
model_path = get_download_path() / "feedforward"
os.makedirs(model_path, exist_ok=True)
predictor.serialize(model_path)
"""
import numpy as np
from itertools import islice
import mxnet as mx
import matplotlib.pyplot as plt
import pandas as pd
from gluonts.dataset.loader import TrainDataLoader
from gluonts.model.deepar import DeepAREstimator
from gluonts.support.util import get_hybrid_forward_input_names
from gluonts.trainer import Trainer
from gluonts.dataset.repository.datasets import get_dataset
if __name__ == '__main__':
dataset = get_dataset(dataset_name="electricity")
estimator = DeepAREstimator(
prediction_length=dataset.metadata.prediction_length,
freq=dataset.metadata.time_granularity,
trainer=Trainer(learning_rate=1e-3,
epochs=50,
num_batches_per_epoch=100),
)
# instead of calling `train` method, we call `train_model` that returns more things including the training model
train_output = estimator.train_model(dataset.train)
# we construct a data_entry that contains 500 random windows
batch_size = 500
num_samples = 100
This example shows how to fit a model and evaluate its predictions.
"""
import pprint
from gluonts.dataset.repository.datasets import get_dataset, dataset_recipes
from gluonts.evaluation import Evaluator
from gluonts.evaluation.backtest import make_evaluation_predictions
from gluonts.model.simple_feedforward import SimpleFeedForwardEstimator
from gluonts.trainer import Trainer
if __name__ == "__main__":
print(f"datasets available: {dataset_recipes.keys()}")
# we pick m4_hourly as it only contains a few hundred time series
dataset = get_dataset("m4_hourly", regenerate=False)
estimator = SimpleFeedForwardEstimator(
prediction_length=dataset.metadata.prediction_length,
freq=dataset.metadata.freq,
trainer=Trainer(epochs=5, num_batches_per_epoch=10),
)
predictor = estimator.train(dataset.train)
forecast_it, ts_it = make_evaluation_predictions(dataset.test,
predictor=predictor,
num_eval_samples=100)
agg_metrics, item_metrics = Evaluator()(ts_it,
forecast_it,
import io
from data_store import *
# import pandas as pd
from gluonts.dataset.repository.datasets import get_dataset
from gluonts.model.deepar import DeepAREstimator
from gluonts.model.seq2seq import MQCNNEstimator
from gluonts.model.simple_feedforward import SimpleFeedForwardEstimator
from gluonts.model.wavenet import WaveNetEstimator
from run_data_loader import data_loader
import numpy as np
datasets = [
'm4_daily', 'm4_hourly', 'm4_weekly', 'solar-energy', 'electricity'
]
for ds in datasets:
dataset = get_dataset(ds)
freq = dataset.metadata.freq
prediction_length = dataset.metadata.prediction_length
deepar = DeepAREstimator(
freq=freq,
prediction_length=prediction_length,
)
mqcnn = MQCNNEstimator(
freq=freq,
prediction_length=prediction_length,
)
sff = SimpleFeedForwardEstimator(
num_hidden_dimensions=[10],
context_length=100,
freq="1D",
hidden_dim=hidden_dims,
prediction_length=prediction_length,
context_length=context_length,
input_size=1 + exog_dim,
)
output = net(input_tensor, past_exog_features, future_exog_features)
assert output.shape == (batch_size, 1, prediction_length)
net.eval()
output = net(input_tensor, past_exog_features, future_exog_features)
assert output.shape == (batch_size, 1, prediction_length)
dataset = get_dataset("electricity")
freq = "1H"
context_length = 2 * 7 * 24
prediction_length = dataset.metadata.prediction_length
hidden_dimensions = [64]
batch_size = 16
num_batches_per_epoch = 100
ASSETS_PATH = pathlib.Path(__file__).parent / "assets"
model = RNNNetEstimator(
freq=freq,
prediction_length=prediction_length,
context_length=context_length,
for m, v in evaluation.items() if m in metrics_persisted
}
evaluation["dataset"] = dataset
evaluation["estimator"] = estimator_name
with open(path, 'w') as f:
f.write(json.dumps(evaluation, indent=4, sort_keys=True))
if __name__ == "__main__":
for dataset_name in datasets:
for Estimator in Estimators:
dataset = get_dataset(
dataset_name=dataset_name,
regenerate=False,
path="../datasets/",
)
estimator = Estimator(
prediction_length=dataset.metadata.prediction_length,
freq=dataset.metadata.time_granularity,
)
estimator_name = type(estimator).__name__
print(f"evaluating {estimator_name} on {dataset_name}")
agg_metrics, item_metrics = backtest_metrics(
train_dataset=dataset.train,
test_dataset=dataset.test,
def test_instanciation_of_args_proj():
class MockedImplicitQuantileOutput(ImplicitQuantileOutput):
method_calls = 0
@classmethod
def set_args_proj(cls):
super().set_args_proj()
cls.method_calls += 1
dataset = get_dataset("constant")
metadata = dataset.metadata
distr_output = MockedImplicitQuantileOutput(output_domain="Real")
deepar_estimator = DeepAREstimator(
distr_output=distr_output,
freq=metadata.freq,
prediction_length=metadata.prediction_length,
trainer=Trainer(
device="cpu",
epochs=3,
learning_rate=1e-3,
num_batches_per_epoch=1,
batch_size=256,
),
input_size=15,
)
assert distr_output.method_calls == 1
deepar_predictor = deepar_estimator.train(dataset.train, num_workers=1)
# Method should be called when the MockedImplicitQuantileOutput is instanciated,
# and one more time because in_features is different from 1
assert distr_output.method_calls == 2
forecast_it, ts_it = make_evaluation_predictions(
dataset=dataset.test, # test dataset
predictor=deepar_predictor, # predictor
num_samples=100, # number of sample paths we want for evaluation
)
forecasts = list(forecast_it)
tss = list(ts_it)
evaluator = Evaluator(num_workers=0)
agg_metrics, item_metrics = evaluator(iter(tss),
iter(forecasts),
num_series=len(dataset.test))
assert distr_output.method_calls == 2
# Test that the implicit output module is proper reset
new_estimator = DeepAREstimator(
distr_output=MockedImplicitQuantileOutput(output_domain="Real"),
freq=metadata.freq,
prediction_length=metadata.prediction_length,
trainer=Trainer(
device="cpu",
epochs=3,
learning_rate=1e-3,
num_batches_per_epoch=1,
batch_size=256,
),
input_size=15,
)
assert distr_output.method_calls == 3
new_estimator.train(dataset.train, num_workers=1)
assert (
distr_output.method_calls == 3
) # Since in_feature is the same as before, there should be no additional call
