if __name__ == "__main__":
dataset = get_dataset("exchange_rate")
estimator = SimpleFeedForwardEstimator(
prediction_length=dataset.metadata.prediction_length,
freq=dataset.metadata.freq,
trainer=Trainer(epochs=5, num_batches_per_epoch=10),
)
predictor = estimator.train(dataset.train)
# save the trained model in a path ~/.mxnet/gluon-ts/feedforward/
# or $MXNET_HOME/feedforward if MXNET_HOME is defined
model_path = get_download_path() / "feedforward"
os.makedirs(model_path, exist_ok=True)
predictor.serialize(model_path)
# loads it back and evaluate predictions accuracy with the deserialized model
predictor_deserialized = Predictor.deserialize(model_path)
forecast_it, ts_it = make_evaluation_predictions(
dataset.test, predictor=predictor_deserialized, num_eval_samples=100)
agg_metrics, item_metrics = Evaluator()(ts_it,
forecast_it,
num_series=len(dataset.test))
pprint.pprint(agg_metrics)
def backtest_metrics(
test_dataset: Dataset,
predictor: Predictor,
evaluator=Evaluator(
quantiles=(0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9)
),
num_samples: int = 100,
logging_file: Optional[str] = None,
):
"""
Parameters
----------
test_dataset
Dataset to use for testing.
predictor
The predictor to test.
evaluator
Evaluator to use.
num_samples
Number of samples to use when generating sample-based forecasts.
logging_file
If specified, information of the backtest is redirected to this file.
Returns
-------
tuple
A tuple of aggregate metrics and per-time-series metrics obtained by
training `forecaster` on `train_dataset` and evaluating the resulting
`evaluator` provided on the `test_dataset`.
"""
if logging_file is not None:
log_formatter = logging.Formatter(
"[%(asctime)s %(levelname)s %(thread)d] %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
)
logger = logging.getLogger(__name__)
handler = logging.FileHandler(logging_file)
handler.setFormatter(log_formatter)
logger.addHandler(handler)
else:
logger = logging.getLogger(__name__)
test_statistics = calculate_dataset_statistics(test_dataset)
serialize_message(logger, test_dataset_stats_key, test_statistics)
forecast_it, ts_it = make_evaluation_predictions(
test_dataset, predictor=predictor, num_samples=num_samples
)
agg_metrics, item_metrics = evaluator(
ts_it, forecast_it, num_series=maybe_len(test_dataset)
)
# we only log aggregate metrics for now as item metrics may be very large
for name, value in agg_metrics.items():
serialize_message(logger, f"metric-{name}", value)
if logging_file is not None:
# Close the file handler to avoid letting the file open.
# https://stackoverflow.com/questions/24816456/python-logging-wont-shutdown
logger.removeHandler(handler)
del logger, handler
return agg_metrics, item_metrics
def train(bucket, seq, algo, freq, prediction_length, epochs, learning_rate,
hybridize, num_batches_per_epoch):
#create train dataset
df = pd.read_csv(filepath_or_buffer=os.environ['SM_CHANNEL_TRAIN'] +
"/train.csv",
header=0,
index_col=0)
training_data = ListDataset([{
"start": df.index[0],
"target": df.usage[:],
"item_id": df.client[:]
}],
freq=freq)
#create test dataset
df = pd.read_csv(filepath_or_buffer=os.environ['SM_CHANNEL_TEST'] +
"/test.csv",
header=0,
index_col=0)
test_data = ListDataset([{
"start": df.index[0],
"target": df.usage[:],
"item_id": 'client_12'
}],
freq=freq)
hook = Hook.create_from_json_file()
#determine estimators##################################
if algo == "DeepAR":
estimator = DeepAREstimator(
freq=freq,
prediction_length=prediction_length,
context_length=1,
trainer=Trainer(ctx="cpu",
epochs=epochs,
learning_rate=learning_rate,
hybridize=hybridize,
num_batches_per_epoch=num_batches_per_epoch))
#train the model
predictor = estimator.train(training_data=training_data)
print("DeepAR training is complete SUCCESS")
elif algo == "SFeedFwd":
estimator = SimpleFeedForwardEstimator(
freq=freq,
prediction_length=prediction_length,
trainer=Trainer(ctx="cpu",
epochs=epochs,
learning_rate=learning_rate,
hybridize=hybridize,
num_batches_per_epoch=num_batches_per_epoch))
#train the model
predictor = estimator.train(training_data=training_data)
print("training is complete SUCCESS")
elif algo == "lstnet":
# Needed for LSTNet ONLY
grouper = MultivariateGrouper(max_target_dim=6)
training_data = grouper(training_data)
test_data = grouper(test_data)
context_length = prediction_length
num_series = 1
skip_size = 1
ar_window = 1
channels = 4
estimator = LSTNetEstimator(
freq=freq,
prediction_length=prediction_length,
context_length=context_length,
num_series=num_series,
skip_size=skip_size,
ar_window=ar_window,
channels=channels,
trainer=Trainer(ctx="cpu",
epochs=epochs,
learning_rate=learning_rate,
hybridize=hybridize,
num_batches_per_epoch=num_batches_per_epoch))
#train the model
predictor = estimator.train(training_data=training_data)
print("training is complete SUCCESS")
elif algo == "seq2seq":
estimator = MQCNNEstimator(
freq=freq,
prediction_length=prediction_length,
trainer=Trainer(ctx="cpu",
epochs=epochs,
learning_rate=learning_rate,
hybridize=hybridize,
num_batches_per_epoch=num_batches_per_epoch))
#train the model
predictor = estimator.train(training_data=training_data)
print("training is complete SUCCESS")
else:
estimator = TransformerEstimator(
freq=freq,
prediction_length=prediction_length,
trainer=Trainer(ctx="cpu",
epochs=epochs,
learning_rate=learning_rate,
hybridize=hybridize,
num_batches_per_epoch=num_batches_per_epoch))
#train the model
predictor = estimator.train(training_data=training_data)
print("training is complete SUCCESS")
###################################################
#evaluate trained model on test data
forecast_it, ts_it = make_evaluation_predictions(test_data,
predictor,
num_samples=100)
print("EVALUATION is complete SUCCESS")
forecasts = list(forecast_it)
tss = list(ts_it)
evaluator = Evaluator(quantiles=[0.1, 0.5, 0.9])
agg_metrics, item_metrics = evaluator(iter(tss),
iter(forecasts),
num_series=len(test_data))
print("METRICS retrieved SUCCESS")
#bucket = "bwp-sandbox"
mainpref = "gluonts/blog-models/"
prefix = mainpref + str(seq) + "/"
agg_df = pd.DataFrame(agg_metrics, index=[0])
file = "metrics" + str(seq) + ".csv"
os.system('mkdir metrics')
cspath = os.path.join('metrics', file)
agg_df.to_csv(cspath)
s3.upload_file(cspath, bucket, mainpref + "metrics/" + file)
hook.save_scalar("MAPE", agg_metrics["MAPE"], sm_metric=True)
hook.save_scalar("RMSE", agg_metrics["RMSE"], sm_metric=True)
hook.save_scalar("MASE", agg_metrics["MASE"], sm_metric=True)
hook.save_scalar("MSE", agg_metrics["MSE"], sm_metric=True)
print("MAPE:", agg_metrics["MAPE"])
#save the model
predictor.serialize(pathlib.Path(os.environ['SM_MODEL_DIR']))
uploadDirectory(os.environ['SM_MODEL_DIR'], prefix, bucket)
return predictor
def train(args):
# Parse arguments
epochs = args.epochs
pred_length = args.pred_length
num_layers = args.num_layers
num_cells = args.num_cells
dropout_rate = args.dropout_rate
batch_size = args.batch_size
lr = args.lr
model_dir = args.model_dir
data_dir = args.data_dir
num_gpus = args.num_gpus
output_dir = args.output_dir
device = "gpu" if num_gpus > 0 else "cpu"
FREQ = 'D'
target_col = 'Weekly_Sales_sum'
related_cols = ['Temperature', 'Fuel_Price', 'CPI', 'Unemployment']
# Get training data
target_train_df = pd.read_csv(os.path.join(data_dir, 'target_train.csv'), index_col=0, header=[0,1])
related_train_df = pd.read_csv(os.path.join(data_dir, 'related_train.csv'), index_col=0, header=[0,1])
store_df = pd.read_csv(os.path.join(data_dir, 'item.csv'), index_col=0)
num_steps, num_series = target_train_df.shape
target = target_train_df.values
start_train_dt = target_train_df.index[0]
custom_ds_metadata = {'num_series': num_series,
'num_steps': num_steps,
'prediction_length': pred_length,
'freq': FREQ,
'start': [start_train_dt for _ in range(num_series)]
}
# Prepare GlounTS Dataset
related_list = [related_train_df[c].values for c in related_cols]
train_lst = []
for i in range(0, num_series):
target_vec = target[:-pred_length, i]
related_vecs = [related[:-pred_length, i] for related in related_list]
item = store_df.loc[i+1]
dic = {FieldName.TARGET: target_vec,
FieldName.START: start_train_dt,
FieldName.FEAT_DYNAMIC_REAL: related_vecs,
FieldName.FEAT_STATIC_CAT: [item[0]],
FieldName.FEAT_STATIC_REAL: [item[1]]
}
train_lst.append(dic)
test_lst = []
for i in range(0, num_series):
target_vec = target[:, i]
related_vecs = [related[:, i] for related in related_list]
item = store_df.loc[i+1]
dic = {FieldName.TARGET: target_vec,
FieldName.START: start_train_dt,
FieldName.FEAT_DYNAMIC_REAL: related_vecs,
FieldName.FEAT_STATIC_CAT: [item[0]],
FieldName.FEAT_STATIC_REAL: [item[1]]
}
test_lst.append(dic)
train_ds = ListDataset(train_lst, freq=FREQ)
test_ds = ListDataset(test_lst, freq=FREQ)
# Define Estimator
trainer = Trainer(
ctx=device,
epochs=epochs,
learning_rate=lr,
batch_size=batch_size
)
deepar_estimator = DeepAREstimator(freq=FREQ,
prediction_length=pred_length,
use_feat_dynamic_real=True,
use_feat_static_cat=True,
use_feat_static_real=True,
cardinality=[3],
num_cells=30,
distr_output=StudentTOutput(),
trainer=trainer)
# Train the model
deepar_predictor = deepar_estimator.train(train_ds)
# Evaluate trained model on test data
forecast_it, ts_it = make_evaluation_predictions(test_ds, deepar_predictor, num_samples=100)
forecasts = list(forecast_it)
tss = list(ts_it)
evaluator = Evaluator(quantiles=[0.1, 0.5, 0.9])
agg_metrics, item_metrics = evaluator(iter(tss), iter(forecasts), num_series=len(test_ds))
metrics = ['RMSE', 'MAPE', 'wQuantileLoss[0.1]', 'wQuantileLoss[0.5]', 'wQuantileLoss[0.9]', 'mean_wQuantileLoss']
metrics_dic = dict((key,value) for key, value in agg_metrics.items() if key in metrics)
print(json.dumps(metrics_dic, indent=2))
# Save the model
deepar_predictor.serialize(pathlib.Path(model_dir))
return deepar_predictor
def train(args):
# Parse arguments
epochs = args.epochs
pred_length = args.pred_length
batch_size = args.batch_size
lr = args.lr
model_dir = args.model_dir
data_dir = args.data_dir
num_gpus = args.num_gpus
output_dir = args.output_dir
device = "gpu" if num_gpus > 0 else "cpu"
FREQ = 'H'
target_col = 'traffic_volume'
related_cols = [
'holiday', 'temp', 'rain_1h', 'snow_1h', 'clouds_all', 'weather_main',
'weather_description'
]
# Get training data
target_train_df = pd.read_csv(os.path.join(data_dir, 'target_train.csv'),
index_col=0)
related_train_df = pd.read_csv(os.path.join(data_dir, 'related_train.csv'),
index_col=0)
num_steps, num_series = target_train_df.shape
target = target_train_df.values
start_train_dt = '2017-01-01 00:00:00'
custom_ds_metadata = {
'num_series': num_series,
'num_steps': num_steps,
'prediction_length': pred_length,
'freq': FREQ,
'start': start_train_dt
}
# Prepare GlounTS Dataset
related_list = [related_train_df[c].values for c in related_cols]
train_lst = []
target_vec = target[:-pred_length].squeeze()
related_vecs = [
related[:-pred_length].squeeze() for related in related_list
]
dic = {
FieldName.TARGET: target_vec,
FieldName.START: start_train_dt,
FieldName.FEAT_DYNAMIC_REAL: related_vecs
}
train_lst.append(dic)
test_lst = []
target_vec = target.squeeze()
related_vecs = [related.squeeze() for related in related_list]
dic = {
FieldName.TARGET: target_vec,
FieldName.START: start_train_dt,
FieldName.FEAT_DYNAMIC_REAL: related_vecs
}
test_lst.append(dic)
train_ds = ListDataset(train_lst, freq=FREQ)
test_ds = ListDataset(test_lst, freq=FREQ)
# Define Estimator
trainer = Trainer(ctx=device,
epochs=epochs,
learning_rate=lr,
batch_size=batch_size)
mlp_estimator = SimpleFeedForwardEstimator(num_hidden_dimensions=[50],
prediction_length=pred_length,
context_length=2 * pred_length,
freq=FREQ,
trainer=trainer)
# Train the model
mlp_predictor = mlp_estimator.train(train_ds)
# Evaluate trained model on test data
forecast_it, ts_it = make_evaluation_predictions(test_ds,
mlp_predictor,
num_samples=100)
forecasts = list(forecast_it)
tss = list(ts_it)
evaluator = Evaluator(quantiles=[0.1, 0.5, 0.9])
agg_metrics, item_metrics = evaluator(iter(tss),
iter(forecasts),
num_series=len(test_ds))
metrics = [
'RMSE', 'MAPE', 'wQuantileLoss[0.1]', 'wQuantileLoss[0.5]',
'wQuantileLoss[0.9]', 'mean_wQuantileLoss'
]
metrics_dic = dict(
(key, value) for key, value in agg_metrics.items() if key in metrics)
print(json.dumps(metrics_dic, indent=2))
# Save the model
mlp_predictor.serialize(pathlib.Path(model_dir))
return mlp_predictor
def backtest_metrics(
train_dataset: Optional[Dataset],
test_dataset: Dataset,
forecaster: Union[Estimator, Predictor],
evaluator=Evaluator(quantiles=(0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8,
0.9)),
num_samples: int = 100,
logging_file: Optional[str] = None,
use_symbol_block_predictor: Optional[bool] = False,
num_workers: Optional[int] = None,
num_prefetch: Optional[int] = None,
**kwargs,
):
"""
Parameters
----------
train_dataset
Dataset to use for training.
test_dataset
Dataset to use for testing.
forecaster
An estimator or a predictor to use for generating predictions.
evaluator
Evaluator to use.
num_samples
Number of samples to use when generating sample-based forecasts.
logging_file
If specified, information of the backtest is redirected to this file.
use_symbol_block_predictor
Use a :class:`SymbolBlockPredictor` during testing.
num_workers
The number of multiprocessing workers to use for data preprocessing.
By default 0, in which case no multiprocessing will be utilized.
num_prefetch
The number of prefetching batches only works if `num_workers` > 0.
If `prefetch` > 0, it allow worker process to prefetch certain batches before
acquiring data from iterators.
Note that using large prefetching batch will provide smoother bootstrapping performance,
but will consume more shared_memory. Using smaller number may forfeit the purpose of using
multiple worker processes, try reduce `num_workers` in this case.
By default it defaults to `num_workers * 2`.
Returns
-------
tuple
A tuple of aggregate metrics and per-time-series metrics obtained by
training `forecaster` on `train_dataset` and evaluating the resulting
`evaluator` provided on the `test_dataset`.
"""
if logging_file is not None:
log_formatter = logging.Formatter(
"[%(asctime)s %(levelname)s %(thread)d] %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
)
logger = logging.getLogger(__name__)
handler = logging.FileHandler(logging_file)
handler.setFormatter(log_formatter)
logger.addHandler(handler)
else:
logger = logging.getLogger(__name__)
if train_dataset is not None:
train_statistics = calculate_dataset_statistics(train_dataset)
serialize_message(logger, train_dataset_stats_key, train_statistics)
test_statistics = calculate_dataset_statistics(test_dataset)
serialize_message(logger, test_dataset_stats_key, test_statistics)
if isinstance(forecaster, Estimator):
serialize_message(logger, estimator_key, forecaster)
assert train_dataset is not None
predictor = forecaster.train(train_dataset)
if isinstance(forecaster, GluonEstimator) and isinstance(
predictor, GluonPredictor):
inference_data_loader = InferenceDataLoader(
dataset=test_dataset,
transform=predictor.input_transform,
batch_size=forecaster.trainer.batch_size,
ctx=forecaster.trainer.ctx,
dtype=forecaster.dtype,
num_workers=num_workers,
num_prefetch=num_prefetch,
**kwargs,
)
if forecaster.trainer.hybridize:
predictor.hybridize(batch=next(iter(inference_data_loader)))
if use_symbol_block_predictor:
predictor = predictor.as_symbol_block_predictor(
batch=next(iter(inference_data_loader)))
else:
predictor = forecaster
forecast_it, ts_it = make_evaluation_predictions(test_dataset,
predictor=predictor,
num_samples=num_samples)
agg_metrics, item_metrics = evaluator(ts_it,
forecast_it,
num_series=maybe_len(test_dataset))
# we only log aggregate metrics for now as item metrics may be very large
for name, value in agg_metrics.items():
serialize_message(logger, f"metric-{name}", value)
if logging_file is not None:
# Close the file handler to avoid letting the file open.
# https://stackoverflow.com/questions/24816456/python-logging-wont-shutdown
logger.removeHandler(handler)
del logger, handler
return agg_metrics, item_metrics
StepStrategy,
generate_rolling_dataset,
)
if __name__ == "__main__":
dataset = get_dataset("constant", regenerate=False)
estimator = SimpleFeedForwardEstimator(
prediction_length=dataset.metadata.prediction_length,
freq=dataset.metadata.freq,
trainer=Trainer(epochs=5, num_batches_per_epoch=10),
)
predictor = estimator.train(dataset.train)
# create the rolled dataset to use for forecasting and evaluation
dataset_rolled = generate_rolling_dataset(
dataset=dataset.test,
start_time=pd.Timestamp("2000-01-01-15", freq="1H"),
end_time=pd.Timestamp("2000-01-02-04", freq="1H"),
strategy=StepStrategy(
prediction_length=dataset.metadata.prediction_length, ),
)
forecast_it, ts_it = make_evaluation_predictions(
dataset_rolled, predictor=predictor, num_samples=len(dataset_rolled))
agg_metrics, _ = Evaluator()(ts_it, forecast_it)
pprint(agg_metrics)
def test_appendix_c():
"""
Test GluonTS paper examples from arxiv paper:
https://arxiv.org/abs/1906.05264
Appendix C
"""
from typing import List
from mxnet import gluon
from gluonts.model.estimator import GluonEstimator
from gluonts.model.predictor import Predictor
from gluonts.mx.model.predictor import RepresentableBlockPredictor
from gluonts.mx.trainer import Trainer
from gluonts.transform import (
InstanceSplitter,
Transformation,
ExpectedNumInstanceSampler,
)
from gluonts.core.component import validated
from gluonts.support.util import copy_parameters
class MyTrainNetwork(gluon.HybridBlock):
def __init__(self, prediction_length, cells, act_type, **kwargs):
super().__init__(**kwargs)
self.prediction_length = prediction_length
with self.name_scope():
# Set up a network that predicts the target
self.nn = gluon.nn.HybridSequential()
for c in cells:
self.nn.add(gluon.nn.Dense(units=c, activation=act_type))
self.nn.add(
gluon.nn.Dense(
units=self.prediction_length, activation=act_type
)
)
def hybrid_forward(self, F, past_target, future_target):
prediction = self.nn(past_target)
# calculate L1 loss to learn the median
return (prediction - future_target).abs().mean(axis=-1)
class MyPredNetwork(MyTrainNetwork):
# The prediction network only receives
# past target and returns predictions
def hybrid_forward(self, F, past_target):
prediction = self.nn(past_target)
return prediction.expand_dims(axis=1)
class MyEstimator(GluonEstimator):
@validated()
def __init__(
self,
freq: str,
prediction_length: int,
act_type: str = "relu",
context_length: int = 30,
cells: List[int] = [40, 40, 40],
trainer: Trainer = Trainer(epochs=10),
) -> None:
super().__init__(trainer=trainer)
self.freq = freq
self.prediction_length = prediction_length
self.act_type = act_type
self.context_length = context_length
self.cells = cells
def create_training_network(self) -> MyTrainNetwork:
return MyTrainNetwork(
prediction_length=self.prediction_length,
cells=self.cells,
act_type=self.act_type,
)
def create_predictor(
self,
transformation: Transformation,
trained_network: gluon.HybridBlock,
) -> Predictor:
prediction_network = MyPredNetwork(
prediction_length=self.prediction_length,
cells=self.cells,
act_type=self.act_type,
)
copy_parameters(trained_network, prediction_network)
return RepresentableBlockPredictor(
input_transform=transformation,
prediction_net=prediction_network,
batch_size=self.trainer.batch_size,
freq=self.freq,
prediction_length=self.prediction_length,
ctx=self.trainer.ctx,
)
def create_transformation(self):
# Model specific input transform
# Here we use a transformation that randomly
# selects training samples from all series.
return InstanceSplitter(
target_field=FieldName.TARGET,
is_pad_field=FieldName.IS_PAD,
start_field=FieldName.START,
forecast_start_field=FieldName.FORECAST_START,
train_sampler=ExpectedNumInstanceSampler(num_instances=1),
past_length=self.context_length,
future_length=self.prediction_length,
)
from gluonts.mx.trainer import Trainer
from gluonts.evaluation import Evaluator
from gluonts.evaluation.backtest import backtest_metrics
dataset_info, train_ds, test_ds = constant_dataset()
meta = dataset_info.metadata
estimator = MyEstimator(
freq=meta.freq,
prediction_length=1,
trainer=Trainer(epochs=1, batch_size=32),
)
predictor = estimator.train(train_ds)
evaluator = Evaluator(quantiles=(0.1, 0.5, 0.9))
agg_metrics, item_metrics = backtest_metrics(
test_dataset=test_ds, predictor=predictor, evaluator=evaluator,
)
def evaluate_deepar(
predictor: GluonPredictor,
train_data: ListDataset,
test_data: ListDataset,
hierarchy_dict: Dict[int, List[int]],
output_file: str = None,
output_mean: bool = True,
output_residuals: bool = True,
) -> Dict[Union[int, str], Dict[str, float]]:
""" aggregates error metrics for each level of the hierarchy, optionally writes predictions/in-sample residuals
to output file
Arguments:
predictor {GluonPredictor} -- predictor
train_data {ListDataset} -- train dataset
test_data {ListDataset} -- test dataset
hierarchy_dict {Dict[int, List[int]]} -- mapping from hierachy level to series prediction idxs included
in that level of hierarchy
Keyword Arguments:
output_file {str} -- output_file to save predictions (default: {None})
output_mean {bool} -- whether to output the mean (or median) predictions (default: {False})
output_residuals {bool} -- whether to output the residuals of in-sample predictions. If True,
the in-sample residuals will be prepended to the out-of-sample predictions. Thus,
if the in-sample data contains 24 timeteps, and the out-of-sample data contains 6 timesteps,
the output data frame will contain 30 rows (timesteps) (default: {True})
Returns:
Dict[Union[int, str], Dict[str, float]] -- mapping of hierarchy level (0-indexed) to dictionaries of aggregated metrics
for that level of the hierarchy
"""
eval_forecasts = []
output_forecasts = []
with tqdm(predictor.predict(train_data),
total=len(train_data),
desc="Making Predictions") as it, np.errstate(invalid='ignore'):
for forecast in it:
output_forecasts.append(
forecast.mean if output_mean else forecast.quantile(0.5))
eval_forecasts.append(forecast)
preds = np.array(output_forecasts)
if output_file:
if output_residuals:
preds = np.concatenate(
(predictor.prediction_net.residuals.asnumpy(), preds), axis=1)
out_df = pd.DataFrame(preds).T
out_df.to_csv(output_file, index=False)
eval_forecasts = np.array(eval_forecasts)
evaluator = Evaluator(quantiles=[0.5])
evaluations = {
level: evaluator([
to_pandas(series)
for series in np.array(list(test_data))[np.array(idxs)]
], eval_forecasts[np.array(idxs)])[0]
for level, idxs in hierarchy_dict.items()
}
evaluations['all'] = evaluator(
[to_pandas(series) for series in np.array(list(test_data))],
eval_forecasts)[0]
return evaluations
def test_instanciation_of_args_proj():
class MockedImplicitQuantileOutput(ImplicitQuantileOutput):
method_calls = 0
@classmethod
def set_args_proj(cls):
super().set_args_proj()
cls.method_calls += 1
dataset = get_dataset("constant")
metadata = dataset.metadata
distr_output = MockedImplicitQuantileOutput(output_domain="Real")
deepar_estimator = DeepAREstimator(
distr_output=distr_output,
freq=metadata.freq,
prediction_length=metadata.prediction_length,
trainer=Trainer(
device="cpu",
epochs=3,
learning_rate=1e-3,
num_batches_per_epoch=1,
batch_size=256,
),
input_size=48,
)
assert distr_output.method_calls == 1
deepar_predictor = deepar_estimator.train(dataset.train, num_workers=1)
# Method should be called when the MockedImplicitQuantileOutput is instanciated,
# and one more time because in_features is different from 1
assert distr_output.method_calls == 2
forecast_it, ts_it = make_evaluation_predictions(
dataset=dataset.test, # test dataset
predictor=deepar_predictor, # predictor
num_samples=100, # number of sample paths we want for evaluation
)
forecasts = list(forecast_it)
tss = list(ts_it)
evaluator = Evaluator(num_workers=0)
agg_metrics, item_metrics = evaluator(iter(tss),
iter(forecasts),
num_series=len(dataset.test))
assert distr_output.method_calls == 2
# Test that the implicit output module is proper reset
new_estimator = DeepAREstimator(
distr_output=MockedImplicitQuantileOutput(output_domain="Real"),
freq=metadata.freq,
prediction_length=metadata.prediction_length,
trainer=Trainer(
device="cpu",
epochs=3,
learning_rate=1e-3,
num_batches_per_epoch=1,
batch_size=256,
),
input_size=48,
)
assert distr_output.method_calls == 3
new_estimator.train(dataset.train, num_workers=1)
assert (
distr_output.method_calls == 3
) # Since in_feature is the same as before, there should be no additional call
forecast_it, ts_it = make_evaluation_predictions(dataset=dataset.test,
predictor=predictor)
forecasts_pytorch = list(f for f in forecast_it)
tss_pytorch = list(ts_it)
# figure plot
plt.figure(figsize=(20, 15))
for idx, (forecast,
ts) in islice(enumerate(zip(forecasts_pytorch, tss_pytorch)), 9):
ax = plt.subplot(3, 3, idx + 1)
plt.plot(ts[-5 * prediction_length:], label="target")
forecast.plot()
plt.gcf().tight_layout()
plt.legend()
plt.savefig(ASSETS_PATH / "ffnet-electricity.png", dpi=200)
# metrics
evaluator = Evaluator(quantiles=[0.5])
metrics_pytorch, _ = evaluator(iter(tss_pytorch),
iter(forecasts_pytorch),
num_series=len(dataset.test))
with open(ASSETS_PATH / "ffnet-electricity.metrics", "w") as f:
pd.DataFrame.from_records(metrics_pytorch,
index=["FFNet"]).to_markdown(f)
legend = ["observations", "median prediction"] + [
f"{k}% prediction interval" for k in prediction_intervals
][::-1]
fig, ax = plt.subplots(1, 1, figsize=(10, 7))
ts_entry[-plot_length:].plot(ax=ax) # plot the time series
forecast_entry.plot(prediction_intervals=prediction_intervals, color="g")
plt.grid(which="both")
plt.legend(legend, loc="upper left")
plt.show()
from gluonts.evaluation.backtest import make_evaluation_predictions
forecast_it, ts_it = make_evaluation_predictions(
dataset=dataset.test, predictor=predictor, num_samples=100
)
forecasts = list(forecast_it)
tss = list(ts_it)
plot_prob_forecasts(tss[0], forecasts[0])
from gluonts.evaluation import Evaluator
evaluator = Evaluator(quantiles=[0.1, 0.5, 0.9])
agg_metrics, item_metrics = evaluator(iter(tss), iter(forecasts), num_series=len(dataset.test))
if __name__ == "__main__":
pass
def eval(model,
validation_ds,
metadata,
configs,
names,
caiso_net_load_ramp=None,
eia_net_load_ramp=None):
predictor = model.load_model()
forecasts_it, series_it = make_evaluation_predictions(
dataset=validation_ds, # test dataset
predictor=predictor, # predictor
num_samples=128 # number of sample paths we want for evaluation
)
forecasts_list_it, forecast_it = tee(forecasts_it)
series_list_it, series_it = tee(series_it)
forecasts_ls = list(forecasts_list_it)
series_ls = list(series_list_it)
evaluator = Evaluator(quantiles=[0.10, 0.50, 0.90])
agg_metrics, item_metrics = evaluator(series_it,
forecast_it,
num_series=len(validation_ds))
for i in range(len(forecasts_ls)):
forecast = forecasts_ls[i]
name = names[i].lower()
print("*-" * 40)
print("*" * 40, end='')
print(f"Forecast {name}", "*" * 30)
print(f"Number of sample paths: {forecast.num_samples}")
print(f"Dimension of samples: {forecast.samples.shape}")
print(f"Start date of the forecast window: {forecast.start_date}")
print(f"Frequency of the time series: {forecast.freq}")
print("*-" * 40)
series = series_ls[i]
err_type = name + '_error_perc'
agg_metrics[err_type] = error_percentage(
predicted=forecast.samples,
true=series[-configs.pred_len:].to_numpy())
final_forecasts = ForecastMe(samples=forecast.samples,
start_date=forecast.start_date -
pd.Timedelta(value=8, unit='H'),
freq='H')
#final_forecasts = ForecastMe(samples=forecast.samples,start_date=forecast.start_date
# ,freq='H')
plot_forecasts(final_forecasts, agg_metrics, series, name, configs,
err_type)
# now do a similar calc for the combined forecasts
# log these values - I have logged the error percentages - Aditya
plt.close('all')
if configs.six_ramps:
target_ramp_caiso = series_ls[0] - series_ls[2] - series_ls[4]
target_ramp_eia = series_ls[1] - series_ls[3] - series_ls[5]
combined_forecast_samples = combine_forecast_components(
forecasts_ls, series_ls)
for cur_label, cur_samples in combined_forecast_samples.items():
if "caiso" in cur_label:
cur_target = caiso_net_load_ramp
err_type = 'caiso_error_perc'
ramp_name = 'CAISO Ramp Prediction Mean' if 'mean' in cur_label else 'CAISO Ramp Prediction Median'
target_series = target_ramp_caiso
else:
cur_target = eia_net_load_ramp
err_type = 'eia_error_perc'
ramp_name = 'EIA Ramp Prediction Mean' if 'mean' in cur_label else 'EIA Ramp Prediction Median'
target_series = target_ramp_eia
agg_metrics[err_type] = error_percentage(
predicted=cur_samples,
true=cur_target[-configs.pred_len:].reshape(-1))
#samples: np.ndarray, start_date: pd.Timestamp, freq: 'H'):
final_forecasts = ForecastMe(samples=cur_samples,
start_date=forecast.start_date -
pd.Timedelta(value=8, unit='H'),
freq='H')
#final_forecasts = ForecastMe(samples=cur_samples,start_date = forecast.start_date ,freq='H')
#final_forecasts = SampleForecast(samples=cur_samples,start_date=forecast.start_date,freq='H')
plot_forecasts(final_forecasts,
agg_metrics,
target_series,
cur_label,
configs,
err_type,
shift=False)
#plot_final_ramps(cur_samples, target_series, ramp_name, configs)
print(f"{cur_label} percent error: {agg_metrics[err_type]}")
print('\nTarget series forecast plots saved.')
log_eval(configs, agg_metrics)
shutil.make_archive(os.path.join(configs.model_save_path, 'plots'),
'zip',
base_dir=os.path.join(configs.model_save_path,
'plots'))
print('\nConfiguration and results logged.')
def backtest_metrics(
train_dataset: Optional[Dataset],
test_dataset: Dataset,
forecaster: Union[Estimator, Predictor],
evaluator=Evaluator(quantiles=(0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8,
0.9)),
num_eval_samples: int = 100,
logging_file: Optional[str] = None,
use_symbol_block_predictor: bool = False,
):
"""
Parameters
----------
train_dataset
Dataset to use for training.
test_dataset
Dataset to use for testing.
forecaster
An estimator or a predictor to use for generating predictions.
evaluator
Evaluator to use.
num_eval_samples
Number of samples to use when generating sample-based forecasts.
logging_file
If specified, information of the backtest is redirected to this file.
use_symbol_block_predictor
Use a :class:`SymbolBlockPredictor` during testing.
Returns
-------
tuple
A tuple of aggregate metrics and per-time-series metrics obtained by
training `forecaster` on `train_dataset` and evaluating the resulting
`evaluator` provided on the `test_dataset`.
"""
if logging_file is not None:
log_formatter = logging.Formatter(
"[%(asctime)s %(levelname)s %(thread)d] %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
)
logger = logging.getLogger(__name__)
handler = logging.FileHandler(logging_file)
handler.setFormatter(log_formatter)
logger.addHandler(handler)
else:
logger = logging.getLogger(__name__)
if train_dataset is not None:
train_statistics = calculate_dataset_statistics(train_dataset)
serialize_message(logger, train_dataset_stats_key, train_statistics)
test_statistics = calculate_dataset_statistics(test_dataset)
serialize_message(logger, test_dataset_stats_key, test_statistics)
if isinstance(forecaster, Estimator):
serialize_message(logger, estimator_key, forecaster)
predictor = forecaster.train(train_dataset)
if isinstance(forecaster, GluonEstimator) and isinstance(
predictor, GluonPredictor):
inference_data_loader = InferenceDataLoader(
dataset=test_dataset,
transform=predictor.input_transform,
batch_size=forecaster.trainer.batch_size,
ctx=forecaster.trainer.ctx,
dtype=forecaster.dtype,
)
if forecaster.trainer.hybridize:
predictor.hybridize(batch=next(iter(inference_data_loader)))
if use_symbol_block_predictor:
predictor = predictor.as_symbol_block_predictor(
batch=next(iter(inference_data_loader)))
else:
predictor = forecaster
forecast_it, ts_it = make_evaluation_predictions(
test_dataset, predictor=predictor, num_eval_samples=num_eval_samples)
agg_metrics, item_metrics = evaluator(ts_it,
forecast_it,
num_series=len(test_dataset))
# we only log aggregate metrics for now as item metrics may be very large
for name, value in agg_metrics.items():
serialize_message(logger, f"metric-{name}", value)
if logging_file is not None:
# Close the file handler to avoid letting the file open.
# https://stackoverflow.com/questions/24816456/python-logging-wont-shutdown
logger.removeHandler(handler)
del logger, handler
return agg_metrics, item_metrics
def forecast():
st.markdown("## About DeepAR")
st.markdown(
"""
The **Amazon SageMaker DeepAR** forecasting algorithm is a supervised learning algorithm for forecasting
scalar (one-dimensional) time series using recurrent neural networks (RNN). Classical forecasting
methods, such as autoregressive integrated moving average (ARIMA) or exponential smoothing (ETS),
fit a single model to each individual time series. They then use that model to extrapolate the time
series into the future. The basic algorithm uses **Recurrent Neural Networks.**
In many applications, however, you have many similar time series across a set of cross-sectional units. For example,
you might have time series groupings for demand for different products, server loads, and requests for webpages.
For this type of application, you can benefit from training a single model jointly over all of the time series.
DeepAR takes this approach. When your dataset contains hundreds of related time series, DeepAR outperforms the
standard ARIMA and ETS methods. You can also use the trained model to generate forecasts for new time series
that are similar to the ones it has been trained on.
"""
)
st.markdown('## Monthly Forecast using DeepAR')
df = pd.read_csv('Warehouse_monthly.csv')
df['Date'] = pd.to_datetime(df['Date'])
# df.info()
# df.tail()
df_ex = df.set_index('Date')
custom_dataset = df_ex.to_numpy().T
prediction_length = 12
freq = "M"
#custom_dataset = np.random.normal(size=(N, T))
start = pd.Timestamp("31-01-2012", freq=freq)
from gluonts.dataset.common import ListDataset
train_ds = ListDataset([{'target': x, 'start': start}
for x in custom_dataset[:, :-prediction_length]],
freq=freq)
test_ds = ListDataset([{'target': x, 'start': start}
for x in custom_dataset],
freq=freq)
# @st.cache(suppress_st_warning=True)
def train_depar(freq,prediction_length,train_ds):
estimator = DeepAREstimator(freq= freq, prediction_length=prediction_length, trainer=Trainer(epochs=2))
predictor = estimator.train(training_data=train_ds)
return predictor
predictor= train_depar(freq,prediction_length,train_ds)
from gluonts.evaluation.backtest import make_evaluation_predictions
forecast_it, ts_it = make_evaluation_predictions(
dataset=test_ds, # test dataset
predictor=predictor, # predictor
num_samples=100, # number of sample paths we want for evaluation
)
forecasts = list(forecast_it)
tss = list(ts_it)
ts_entry = tss[0]
np.array(ts_entry[:5]).reshape(-1,)
def plot_prob_forecasts(ts_entry, forecast_entry, wareh_name):
plot_length = 150
prediction_intervals = (50.0, 90.0)
legend = ["observations", "median prediction"] + [f"{k}% prediction interval" for k in prediction_intervals][::-1]
fig, ax = plt.subplots(1, 1, figsize=(30 ,16))
ts_entry[-plot_length:].plot(ax=ax) # plot the time series
forecast_entry.plot(prediction_intervals=prediction_intervals, color='r')
plt.grid(which="major")
plt.legend(legend, loc="upper left",fontsize=40)
plt.title(wareh_name,fontsize=50)
plt.xlabel('Date',fontsize=40)
plt.ylabel('Order Demand',fontsize=40)
plt.xticks(fontsize=30)
plt.yticks(fontsize=30)
st.pyplot()
ts_entry = tss[0]
forecast_entry = forecasts[0]
plot_prob_forecasts(ts_entry, forecast_entry, 'Warehouse A Forecast' )
ts_entry = tss[1]
forecast_entry = forecasts[1]
plot_prob_forecasts(ts_entry, forecast_entry, 'Warehouse C Forecast' )
# print("*************************************\n")
# print(forecast_entry.)
ts_entry = tss[2]
forecast_entry = forecasts[2]
plot_prob_forecasts(ts_entry, forecast_entry, 'Warehouse J Forecast' )
ts_entry = tss[3]
forecast_entry = forecasts[3]
plot_prob_forecasts(ts_entry, forecast_entry, 'Warehouse S Forecast' )
evaluator = Evaluator(quantiles=[0.1, 0.5, 0.9])
agg_metrics, item_metrics = evaluator(iter(tss), iter(forecasts), num_series=len(test_ds))
metric_df = item_metrics[['MAPE']]
list_w = ['Warehouse A', 'Warehouse C', 'Warehouse J', 'Warehouse S']
se = pd.Series(list_w)
metric_df['Warehouse'] = se.values
st.markdown("## Look at the MAPE values for different warehouses")
st.table(metric_df)
print("Success")
def run_test(
env: TrainEnv,
predictor: Predictor,
test_dataset: Dataset,
hyperparameters: dict,
) -> None:
len_original = maybe_len(test_dataset)
test_dataset = TransformedDataset(
test_dataset,
FilterTransformation(
lambda x: x["target"].shape[-1] > predictor.prediction_length),
)
len_filtered = len(test_dataset)
if len_original is not None and len_original > len_filtered:
logger.warning(
f"Not all time-series in the test-channel have "
f"enough data to be used for evaluation. Proceeding with "
f"{len_filtered}/{len_original} "
f"(~{int(len_filtered / len_original * 100)}%) items.")
forecast_it, ts_it = backtest.make_evaluation_predictions(
dataset=test_dataset, predictor=predictor, num_samples=100)
test_quantiles = ([
Quantile.parse(quantile).name
for quantile in hyperparameters["test_quantiles"]
] if "test_quantiles" in hyperparameters.keys() else None)
if isinstance(predictor, RepresentableBlockPredictor) and isinstance(
predictor.forecast_generator, QuantileForecastGenerator):
predictor_quantiles = predictor.forecast_generator.quantiles
if test_quantiles is None:
test_quantiles = predictor_quantiles
elif not set(test_quantiles).issubset(set(predictor_quantiles)):
logger.warning(
f"Some of the evaluation quantiles `{test_quantiles}` are "
f"not in the computed quantile forecasts `{predictor_quantiles}`."
)
test_quantiles = predictor_quantiles
if test_quantiles is not None:
logger.info(f"Using quantiles `{test_quantiles}` for evaluation.")
evaluator = Evaluator(quantiles=test_quantiles)
else:
evaluator = Evaluator()
agg_metrics, item_metrics = evaluator(
ts_iterator=ts_it,
fcst_iterator=forecast_it,
num_series=len(test_dataset),
)
# we only log aggregate metrics for now as item metrics may be very large
for name, score in agg_metrics.items():
logger.info(f"#test_score ({env.current_host}, {name}): {score}")
# store metrics
with open(env.path.model / "agg_metrics.json", "w") as agg_metric_file:
json.dump(agg_metrics, agg_metric_file)
with open(env.path.model / "item_metrics.csv", "w") as item_metrics_file:
item_metrics.to_csv(item_metrics_file, index=False)
def test_simple_model():
dsinfo, training_data, test_data = default_synthetic()
freq = dsinfo.metadata.freq
prediction_length = dsinfo.prediction_length
context_length = 2 * prediction_length
hidden_dimensions = [10, 10]
net = LightningFeedForwardNetwork(
freq=freq,
prediction_length=prediction_length,
context_length=context_length,
hidden_dimensions=hidden_dimensions,
distr_output=NormalOutput(),
batch_norm=True,
scaling=mean_abs_scaling,
)
transformation = AddObservedValuesIndicator(
target_field=FieldName.TARGET,
output_field=FieldName.OBSERVED_VALUES,
)
training_splitter = InstanceSplitter(
target_field=FieldName.TARGET,
is_pad_field=FieldName.IS_PAD,
start_field=FieldName.START,
forecast_start_field=FieldName.FORECAST_START,
instance_sampler=ExpectedNumInstanceSampler(
num_instances=1,
min_future=prediction_length,
),
past_length=context_length,
future_length=prediction_length,
time_series_fields=[FieldName.OBSERVED_VALUES],
)
data_loader = TrainDataLoader(
training_data,
batch_size=8,
stack_fn=batchify,
transform=transformation + training_splitter,
num_batches_per_epoch=5,
)
trainer = pl.Trainer(max_epochs=3, callbacks=[], weights_summary=None)
trainer.fit(net, train_dataloader=data_loader)
prediction_splitter = InstanceSplitter(
target_field=FieldName.TARGET,
is_pad_field=FieldName.IS_PAD,
start_field=FieldName.START,
forecast_start_field=FieldName.FORECAST_START,
instance_sampler=TestSplitSampler(),
past_length=context_length,
future_length=prediction_length,
time_series_fields=[FieldName.OBSERVED_VALUES],
)
predictor = net.get_predictor(transformation + prediction_splitter)
forecast_it, ts_it = make_evaluation_predictions(
dataset=test_data,
predictor=predictor,
num_samples=100,
)
evaluator = Evaluator(quantiles=[0.5, 0.9], num_workers=None)
agg_metrics, _ = evaluator(ts_it, forecast_it)
def evaluate(dataset_name, estimator, horizon):
train_ds, test_ds = get_custom_dataset(dataset_name, horizon)
estimator = estimator(
prediction_length=horizon,
freq=freq,
context_length=context_length,
#cardinality=len(train_ds)
)
print(
f"evaluating {estimator} on {dataset_name} dataset for {horizon} horizon"
)
predictor = estimator.train(train_ds)
forecast_it, ts_it = make_evaluation_predictions(test_ds,
predictor=predictor,
num_samples=n_samples)
print("Obtaining time series conditioning values ...")
tss = list(tqdm(ts_it, total=len(test_ds)))
print("Obtaining time series predictions ...")
forecasts = list(tqdm(forecast_it, total=len(test_ds)))
if plot:
print("Plotting time series predictions ...")
for i in tqdm(range(0, 361, 90)):
ts_entry = tss[i]
forecast_entry = forecasts[i]
plot_prob_forecasts(ts_entry, forecast_entry, i, horizon,
context_length)
print("Saving time series predictions ...")
series = int(len(forecasts) / len(train_ds))
sesies_q = np.empty((0, horizon * series), float)
q10_, q50_, q90_, indexes_ = sesies_q, sesies_q, sesies_q, np.empty(
(0, horizon * series), 'datetime64[s]')
for i in range(len(train_ds)):
q10, q50, q90, indexes = np.array([]), np.array([]), np.array(
[]), np.array([])
for z in range(series):
f_dict = forecasts[z * len(train_ds) + i].as_json_dict(
Config(output_types={OutputType.quantiles}))['quantiles']
q10 = np.append(q10, np.array(f_dict['0.1']))
q50 = np.append(q50, np.array(f_dict['0.5']))
q90 = np.append(q90, np.array(f_dict['0.9']))
indexes = np.append(
indexes,
np.array(list(forecasts[z * len(train_ds) + i].index)))
q10_ = np.vstack((q10_, q10))
q50_ = np.vstack((q50_, q50))
q90_ = np.vstack((q90_, q90))
indexes_ = np.vstack((indexes_, indexes))
if save:
save_file = r"./save/{}_{}_{}".format(
type(estimator).__name__, dataset_name, str(horizon))
np.savetxt('{}_q10.txt'.format(save_file), q10_)
np.savetxt('{}_q50.txt'.format(save_file), q50_)
np.savetxt('{}_q90.txt'.format(save_file), q90_)
np.savetxt('{}_index.txt'.format(save_file), indexes_, fmt='%s')
print("Calculating time series prediction metrics ...")
agg_metrics, item_metrics = Evaluator()(iter(tss),
iter(forecasts),
num_series=len(test_ds))
pprint.pprint(agg_metrics)
eval_dict = agg_metrics
eval_dict["dataset"] = dataset_name
eval_dict["estimator"] = type(estimator).__name__
eval_dict["horizon"] = str(horizon)
return eval_dict
def train(args):
# Parse arguments
epochs = args.epochs
pred_length = args.pred_length
num_layers = args.num_layers
num_cells = args.num_cells
dropout_rate = args.dropout_rate
batch_size = args.batch_size
lr = args.lr
model_dir = args.model_dir
data_dir = args.data_dir
num_gpus = args.num_gpus
output_dir = args.output_dir
device = "gpu" if num_gpus > 0 else "cpu"
FREQ = 'D'
# Get training data
target_df = pd.read_csv(os.path.join(data_dir, 'target_train.csv'))
target_df.set_index(target_df.columns[0], inplace=True)
target = target_df.values
num_steps, num_series = target_df.shape
start_dt = target_df.index[0]
custom_ds_metadata = {
'num_series': num_series,
'num_steps': num_steps,
'prediction_length': pred_length,
'freq': FREQ,
'start': [start_dt for _ in range(num_series)]
}
# Prepare GlounTS Dataset
train_lst = []
for i in range(0, num_series):
target_vec = target[:-pred_length, i]
dic = {FieldName.TARGET: target_vec, FieldName.START: start_dt}
train_lst.append(dic)
test_lst = []
for i in range(0, num_series):
target_vec = target[:, i]
dic = {FieldName.TARGET: target_vec, FieldName.START: start_dt}
test_lst.append(dic)
train_ds = ListDataset(train_lst, freq=FREQ)
test_ds = ListDataset(test_lst, freq=FREQ)
train_entry = next(iter(train_ds))
train_entry.keys()
# Define Estimator
trainer = Trainer(ctx=device,
epochs=epochs,
learning_rate=lr,
batch_size=batch_size)
deepar_estimator = DeepAREstimator(freq=FREQ,
prediction_length=pred_length,
num_cells=num_cells,
dropout_rate=dropout_rate,
num_layers=num_layers,
distr_output=StudentTOutput(),
trainer=trainer)
# Train the model
deepar_predictor = deepar_estimator.train(train_ds)
# Evaluate trained model on test data
forecast_it, ts_it = make_evaluation_predictions(test_ds,
deepar_predictor,
num_samples=100)
forecasts = list(forecast_it)
tss = list(ts_it)
evaluator = Evaluator(quantiles=[0.1, 0.5, 0.9])
agg_metrics, item_metrics = evaluator(iter(tss),
iter(forecasts),
num_series=len(test_ds))
metrics = [
'RMSE', 'MAPE', 'wQuantileLoss[0.1]', 'wQuantileLoss[0.5]',
'wQuantileLoss[0.9]', 'mean_wQuantileLoss'
]
metrics_dic = dict(
(key, value) for key, value in agg_metrics.items() if key in metrics)
print(json.dumps(metrics_dic, indent=2))
# Save the model
deepar_predictor.serialize(pathlib.Path(model_dir))
return deepar_predictor