def make_predictions(predictor, region_df_dict, test_date, regions_list, target_col, feat_dynamic_cols=None,
num_eval_samples=100):
if feat_dynamic_cols is not None:
test_data = ListDataset(
[{"item_id": region,
"start": region_df_dict[region].index[0],
"target": region_df_dict[region][target_col][:test_date + timedelta(hours=md.NB_HOURS_PRED)],
"feat_dynamic_real": [
region_df_dict[region][feat_dynamic_col][:test_date + timedelta(hours=md.NB_HOURS_PRED)]
for feat_dynamic_col in feat_dynamic_cols]
}
for region in regions_list],
freq=md.FREQ
)
else:
test_data = ListDataset(
[{"item_id": region,
"start": region_df_dict[region].index[0],
"target": region_df_dict[region][target_col][:test_date + timedelta(hours=md.NB_HOURS_PRED)],
}
for region in regions_list],
freq=md.FREQ
)
forecast_it, ts_it = make_evaluation_predictions(test_data, predictor=predictor, num_eval_samples=num_eval_samples)
return list(forecast_it), list(ts_it)
def evaluate(dataset_name, estimator):
dataset = get_dataset(dataset_name)
estimator = estimator(
prediction_length=dataset.metadata.prediction_length,
freq=dataset.metadata.time_granularity,
)
print(f"evaluating {estimator} on {dataset}")
predictor = estimator.train(dataset.train)
forecast_it, ts_it = make_evaluation_predictions(dataset.test,
predictor=predictor,
num_eval_samples=100)
agg_metrics, item_metrics = Evaluator()(ts_it,
forecast_it,
num_series=len(dataset.test))
pprint.pprint(agg_metrics)
eval_dict = agg_metrics
eval_dict["dataset"] = dataset_name
eval_dict["estimator"] = type(estimator).__name__
return eval_dict
def testDeepRenewal(type, hybridize, freq, num_feat_dynamic_real, cardinality):
prediction_length = 3
if type == "synthetic":
train_ds, test_ds = make_dummy_datasets_with_features(
prediction_length=prediction_length,
freq=freq,
num_feat_dynamic_real=num_feat_dynamic_real,
cardinality=cardinality,
)
else:
train_ds = make_constant_dataset(train_length=15, freq=freq)
test_ds = train_ds
trainer = Trainer(ctx="cpu", epochs=1,
hybridize=hybridize) # hybridize false for development
estimator = DeepRenewalEstimator(
prediction_length=prediction_length,
freq=freq,
trainer=trainer,
)
predictor = estimator.train(training_data=train_ds)
forecast_it, ts_it = make_evaluation_predictions(dataset=test_ds,
predictor=predictor,
num_samples=100)
evaluator = Evaluator(calculate_owa=False, num_workers=0)
agg_metrics, item_metrics = evaluator(ts_it,
forecast_it,
num_series=len(test_ds))
if type == "synthetic":
accuracy = 1.5
else:
accuracy = 1.3
assert agg_metrics["ND"] <= accuracy
def train_and_test(training_data,
test_data,
freq,
num_test_windows,
model,
require_train=False):
forecasts = []
tss = []
training_data = ListDataset(training_data, freq=freq)
test_data = ListDataset(test_data, freq=freq)
if require_train:
predictor = model.train(training_data=training_data)
else:
predictor = model
# Save the model locally for later deployment.
model_name = model.__class__.__name__
model_path = Path(f"models/{model_name}")
os.makedirs(model_path, exist_ok=True)
predictor.serialize(model_path)
# Do the forecast on the test set.
forecast_it, ts_it = make_evaluation_predictions(test_data,
predictor=predictor,
num_samples=100)
forecasts.extend(list(forecast_it))
tss.extend(list(ts_it))
return forecasts, tss
def _make_evaluation_predictions(self, predictor, test_list_dataset):
"""Evaluate predictor and generate sample forecasts.
Args:
predictor (gluonts.model.predictor.Predictor): Trained object used to make forecasts.
test_list_dataset (gluonts.dataset.common.ListDataset): ListDataset created with the GluonDataset class.
Returns:
Dictionary of aggregated metrics over all timeseries.
DataFrame of metrics for each timeseries (i.e., each target column).
List of gluonts.model.forecast.Forecast (objects storing the predicted distributions as samples).
"""
try:
forecast_it, ts_it = make_evaluation_predictions(
dataset=test_list_dataset,
predictor=predictor,
num_samples=100)
forecasts = list(forecast_it)
except Exception as err:
raise ModelPredictionError(
f"GluonTS '{self.model_name}' model crashed when making predictions. Full error: {err}"
)
evaluator = Evaluator(num_workers=min(2, multiprocessing.cpu_count()))
agg_metrics, item_metrics = evaluator(
ts_it, forecasts, num_series=len(test_list_dataset))
return agg_metrics, item_metrics, forecasts
def execute_gluonts_dataframe(model, trial, test_set):
""" Extract dataframe from the predictor """
predictor = model.train(training_data=trial['train'])
forecast_it, ts_it = make_evaluation_predictions(trial['test'],
predictor=predictor,
num_samples=10)
for forecast, ts in zip(forecast_it, ts_it):
df = pd.DataFrame(ts).rename(columns={0: 'actual'})
tmp = pd.concat([
df,
pd.DataFrame(
{
'predicted': forecast.mean,
'q25': forecast.quantile(0.25),
'q75': forecast.quantile(0.75),
'q95': forecast.quantile(0.95),
'q05': forecast.quantile(0.05),
},
index=forecast.index)
],
axis=1)
tmp.dropna(inplace=True)
tmp['test_set'] = test_set
return tmp
def test_training_with_implicit_quantile_output():
dataset = get_dataset("constant")
metadata = dataset.metadata
deepar_estimator = DeepAREstimator(
distr_output=ImplicitQuantileOutput(output_domain="Real"),
freq=metadata.freq,
prediction_length=metadata.prediction_length,
trainer=Trainer(
device="cpu",
epochs=5,
learning_rate=1e-3,
num_batches_per_epoch=3,
batch_size=256,
),
input_size=15,
)
deepar_predictor = deepar_estimator.train(dataset.train, num_workers=1)
forecast_it, ts_it = make_evaluation_predictions(
dataset=dataset.test, # test dataset
predictor=deepar_predictor, # predictor
num_samples=100, # number of sample paths we want for evaluation
)
forecasts = list(forecast_it)
tss = list(ts_it)
evaluator = Evaluator(num_workers=0)
agg_metrics, item_metrics = evaluator(iter(tss),
iter(forecasts),
num_series=len(dataset.test))
assert agg_metrics["MSE"] > 0
def run_test(env: TrainEnv, predictor: Predictor,
test_dataset: Dataset) -> None:
len_original = len(test_dataset)
test_dataset = TransformedDataset(
base_dataset=test_dataset,
transformations=[
FilterTransformation(
lambda x: x["target"].shape[-1] > predictor.prediction_length)
],
)
len_filtered = len(test_dataset)
if len_original > len_filtered:
logger.warning(
f"Not all time-series in the test-channel have "
f"enough data to be used for evaluation. Proceeding with "
f"{len_filtered}/{len_original} "
f"(~{int(len_filtered / len_original * 100)}%) items.")
forecast_it, ts_it = backtest.make_evaluation_predictions(
dataset=test_dataset, predictor=predictor, num_samples=100)
agg_metrics, _item_metrics = Evaluator()(
ts_iterator=ts_it,
fcst_iterator=forecast_it,
num_series=len(test_dataset),
)
# we only log aggregate metrics for now as item metrics may be very large
for name, score in agg_metrics.items():
logger.info(f"#test_score ({env.current_host}, {name}): {score}")
def evaluate(dataset_name, estimator):
dataset = get_dataset(dataset_name)
estimator = estimator(
prediction_length=dataset.metadata.prediction_length,
freq=dataset.metadata.freq,
use_feat_static_cat=True,
cardinality=[
feat_static_cat.cardinality
for feat_static_cat in dataset.metadata.feat_static_cat
],
)
print(f"evaluating {estimator} on {dataset}")
predictor = estimator.train(dataset.train)
forecast_it, ts_it = make_evaluation_predictions(dataset.test,
predictor=predictor,
num_samples=100)
agg_metrics, item_metrics = Evaluator()(ts_it,
forecast_it,
num_series=len(dataset.test))
pprint.pprint(agg_metrics)
eval_dict = agg_metrics
eval_dict["dataset"] = dataset_name
eval_dict["estimator"] = type(estimator).__name__
return eval_dict
def test_dynamic_integration(
train_length: int,
test_length: int,
prediction_length: int,
target_start: str,
rolling_start: str,
num_dynamic_feat: int,
):
"""
Trains an estimator on a rolled dataset with dynamic features.
Tests https://github.com/awslabs/gluon-ts/issues/1390
"""
train_ds = create_dynamic_dataset(target_start, train_length,
num_dynamic_feat)
rolled_ds = generate_rolling_dataset(
dataset=create_dynamic_dataset(target_start, test_length,
num_dynamic_feat),
strategy=StepStrategy(prediction_length=prediction_length),
start_time=pd.Timestamp(rolling_start),
)
estimator = DeepAREstimator(
freq="D",
prediction_length=prediction_length,
context_length=2 * prediction_length,
use_feat_dynamic_real=True,
trainer=Trainer(epochs=1),
)
predictor = estimator.train(training_data=train_ds)
forecast_it, ts_it = make_evaluation_predictions(rolled_ds,
predictor=predictor,
num_samples=100)
training_agg_metrics, _ = Evaluator(num_workers=0)(ts_it, forecast_it)
# it should have failed by this point if the dynamic features were wrong
assert training_agg_metrics
def simple_main():
import mxnet as mx
from pprint import pprint
dataset = get_dataset("electricity", regenerate=False)
trainer = Trainer(
ctx=mx.cpu(0),
epochs=10,
num_batches_per_epoch=200,
learning_rate=1e-3,
hybridize=False,
)
cardinality = int(dataset.metadata.feat_static_cat[0].cardinality)
estimator = DeepFactorEstimator(
trainer=trainer,
context_length=168,
cardinality=[cardinality],
prediction_length=dataset.metadata.prediction_length,
freq=dataset.metadata.freq,
)
predictor = estimator.train(dataset.train)
forecast_it, ts_it = make_evaluation_predictions(dataset.test,
predictor=predictor,
num_eval_samples=100)
agg_metrics, item_metrics = Evaluator()(ts_it,
forecast_it,
num_series=len(dataset.test))
pprint(agg_metrics)
def evaluate(self) -> None:
forecast_it, ts_it = make_evaluation_predictions(
dataset=self.test_dataset,
predictor=self.predictor,
num_samples=self.num_samples)
self.agg_metrics, self.ind_metrics = self.evaluator(ts_it, forecast_it)
seasonal_errors = get_seasonal_errors(self.past_datas)
def evaluate_forecast(test_ds, predictor, train_ds):
forecast_it, ts_it = make_evaluation_predictions(test_ds,
predictor=predictor,
num_samples=100)
forecasts = list(forecast_it)
tss = list(ts_it)
for target, forecast in islice(zip(tss, forecasts), 1):
return forecast.mean_ts, target
def plot_forecasts(test_ds, predictor):
forecast_it, ts_it = make_evaluation_predictions(test_ds,
predictor=predictor,
num_samples=100)
forecasts = list(forecast_it)
tss = list(ts_it)
for target, forecast in islice(zip(tss, forecasts), 1):
return forecast.copy_dim(0).mean_ts, target[0]
def test_simple_model():
dsinfo, training_data, test_data = default_synthetic()
freq = dsinfo.metadata.freq
prediction_length = dsinfo.prediction_length
context_length = 2 * prediction_length
hidden_dimensions = [10, 10]
net = LightningFeedForwardNetwork(
freq=freq,
prediction_length=prediction_length,
context_length=context_length,
hidden_dimensions=hidden_dimensions,
distr_output=NormalOutput(),
batch_norm=True,
scaling=mean_abs_scaling,
)
transformation = Chain([
AddObservedValuesIndicator(
target_field=FieldName.TARGET,
output_field=FieldName.OBSERVED_VALUES,
),
InstanceSplitter(
target_field=FieldName.TARGET,
is_pad_field=FieldName.IS_PAD,
start_field=FieldName.START,
forecast_start_field=FieldName.FORECAST_START,
train_sampler=ExpectedNumInstanceSampler(num_instances=1),
past_length=context_length,
future_length=prediction_length,
time_series_fields=[FieldName.OBSERVED_VALUES],
),
])
data_loader = TrainDataLoader(
training_data,
batch_size=8,
stack_fn=batchify,
transform=transformation,
num_batches_per_epoch=5,
)
trainer = pl.Trainer(max_epochs=3, callbacks=[], weights_summary=None)
trainer.fit(net, train_dataloader=data_loader)
predictor = net.get_predictor(transformation)
forecast_it, ts_it = make_evaluation_predictions(
dataset=test_data,
predictor=predictor,
num_samples=100,
)
evaluator = Evaluator(quantiles=[0.5, 0.9], num_workers=None)
agg_metrics, _ = evaluator(ts_it, forecast_it)
def make_predictions(self, eval_ds):
if self.algorithm == 'DeepAR':
forecast_it, ts_it = make_evaluation_predictions(
eval_ds, predictor=self.predictor, num_samples=100)
elif self.algorithm == 'Prophet':
train_ds = copy.deepcopy(eval_ds)
for p in range(len(list_products)):
train_ds.list_data[p]['target'] = train_ds.list_data[p][
'target'][:-prediction_length]
forecast_it = self.predictor.predict(train_ds)
ts_it = []
for p in range(len(list_products)):
ts_it.append(
pd.DataFrame({0: eval_ds.list_data[p]['target']},
index=pd.date_range(
min_date,
periods=len(
eval_ds.list_data[p]['target']),
freq=freq,
tz=None)))
elif self.algorithm == 'ARIMA':
ts_it = []
period_list_pred = pd.date_range(
min_date,
periods=len(eval_ds.list_data[0]['target']),
freq=freq,
tz=None)[-prediction_length:]
for p in range(len(list_products)):
AMIMA_predictor = self.train_ARIMA_predictor(eval_ds, p)
pred = AMIMA_predictor.predict(n_periods=prediction_length)
if p == 0:
forecast_it = pd.DataFrame({
'OrderDate': period_list_pred,
'Product': pred
})
else:
temp = pd.DataFrame({
'OrderDate': period_list_pred,
'Product': pred
})
forecast_it = forecast_it.merge(temp,
on='OrderDate',
how='left')
forecast_it = forecast_it.rename(
columns={'Product': self.list_products_names[p]})
ts_it.append(
pd.DataFrame({0: eval_ds.list_data[p]['target']},
index=pd.date_range(
min_date,
periods=len(
eval_ds.list_data[p]['target']),
freq=freq,
tz=None)))
return forecast_it, ts_it
return list(forecast_it), list(ts_it)
def evaluate_forecast(args):
is_gpu = mx.context.num_gpus() > 0
dataset = get_dataset(args.datasource,
regenerate=args.regenerate_datasource)
prediction_length = dataset.metadata.prediction_length
freq = dataset.metadata.freq
cardinality = (ast.literal_eval(
dataset.metadata.feat_static_cat[0].cardinality)
if args.use_feat_static_cat else None)
train_ds = dataset.train
test_ds = dataset.test
trainer = Trainer(
ctx=mx.context.gpu() if is_gpu & args.use_cuda else mx.context.cpu(),
batch_size=args.batch_size,
learning_rate=args.learning_rate,
epochs=args.max_epochs,
num_batches_per_epoch=args.number_of_batches_per_epoch,
clip_gradient=args.clip_gradient,
weight_decay=args.weight_decay,
hybridize=True,
) # hybridize false for development
estimator = DeepRenewalEstimator(
prediction_length=prediction_length,
context_length=prediction_length * args.context_length_multiplier,
num_layers=args.num_layers,
num_cells=args.num_cells,
cell_type=args.cell_type,
dropout_rate=args.dropout_rate,
scaling=True,
lags_seq=np.arange(1, args.num_lags + 1).tolist(),
freq=freq,
use_feat_dynamic_real=args.use_feat_dynamic_real,
use_feat_static_cat=args.use_feat_static_cat,
use_feat_static_real=args.use_feat_static_real,
cardinality=cardinality if args.use_feat_static_cat else None,
trainer=trainer,
)
predictor = estimator.train(train_ds, test_ds)
os.makedirs(args.model_save_dir, exist_ok=True)
os.makedirs(Path(args.model_save_dir) / "deep_renewal", exist_ok=True)
predictor.serialize(Path(args.model_save_dir) / "deep_renewal")
print("Generating DeepRenewal forecasts.......")
# predictor = Predictor.deserialize(Path(args.model_save_dir)/"deeprenewal")
deep_renewal_flat_forecast_it, ts_it = make_evaluation_predictions(
dataset=test_ds, predictor=predictor, num_samples=100)
tss = list(tqdm(ts_it, total=len(test_ds)))
deep_renewal_flat_forecasts = list(
tqdm(deep_renewal_flat_forecast_it, total=len(test_ds)))
evaluator = IntermittentEvaluator(quantiles=[0.25, 0.5, 0.75],
median=args.point_forecast == "median",
calculate_spec=args.calculate_spec)
deep_renewal_flat_agg_metrics, deep_renewal_flat_item_metrics = evaluator(
iter(tss), iter(deep_renewal_flat_forecasts), num_series=len(test_ds))
print(deep_renewal_flat_agg_metrics)
def inference(self, model_input):
"""
Internal inference methods
:param model_input: transformed model input data list
:return: list of inference output in numpy array
"""
forecast_it, ts_it = make_evaluation_predictions(model_input,
self.model,
num_samples=100)
return forecast_it
def test_lstnet(
skip_size,
ar_window,
lead_time,
prediction_length,
hybridize,
scaling,
dtype,
):
estimator = LSTNetEstimator(
skip_size=skip_size,
ar_window=ar_window,
num_series=NUM_SERIES,
channels=6,
kernel_size=2,
context_length=4,
freq=freq,
lead_time=lead_time,
prediction_length=prediction_length,
trainer=Trainer(epochs=1,
batch_size=2,
learning_rate=0.01,
hybridize=hybridize),
scaling=scaling,
dtype=dtype,
)
predictor = estimator.train(dataset.train)
forecast_it, ts_it = make_evaluation_predictions(dataset=dataset.test,
predictor=predictor,
num_samples=NUM_SAMPLES)
forecasts = list(forecast_it)
tss = list(ts_it)
assert len(forecasts) == len(tss) == len(dataset.test)
test_ds = dataset.test.list_data[0]
for fct in forecasts:
assert fct.freq == freq
assert fct.samples.shape == (
NUM_SAMPLES,
prediction_length,
NUM_SERIES,
)
assert fct.start_date == pd.date_range(
start=str(test_ds["start"]),
periods=test_ds["target"].shape[1], # number of test periods
freq=freq,
)[-prediction_length]
evaluator = MultivariateEvaluator(
quantiles=[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9])
agg_metrics, item_metrics = evaluator(iter(tss),
iter(forecasts),
num_series=len(dataset.test))
assert agg_metrics["ND"] < 1.0
def run_test(env: TrainEnv, predictor: Predictor,
test_dataset: Dataset) -> None:
forecast_it, ts_it = backtest.make_evaluation_predictions(
test_dataset, predictor=predictor, num_eval_samples=100)
agg_metrics, _item_metrics = Evaluator()(ts_it,
forecast_it,
num_series=len(test_dataset))
# we only log aggregate metrics for now as item metrics may be
# very large
log_metrics(env, agg_metrics)
def run_test(self, dataset, estimator, predictor):
test_dataset = TransformedDataset(
dataset,
transformations=[
FilterTransformation(lambda el: el['target'].shape[-1] >
predictor.prediction_length)
],
)
len_orig = len(dataset)
len_filtered = len(test_dataset)
if len_orig > len_filtered:
logging.warning(
'Not all time-series in the test-channel have '
'enough data to be used for evaluation. Proceeding with '
f'{len_filtered}/{len_orig} '
f'(~{int(len_filtered/len_orig*100)}%) items.')
try:
log.metric('test_dataset_stats', test_dataset.calc_stats())
except GluonTSDataError as error:
logging.error(
f"Failure whilst calculating stats for test dataset: {error}")
return
if isinstance(estimator, GluonEstimator) and isinstance(
predictor, GluonPredictor):
inference_data_loader = InferenceDataLoader(
dataset=test_dataset,
transform=predictor.input_transform,
batch_size=estimator.trainer.batch_size,
ctx=estimator.trainer.ctx,
float_type=estimator.float_type,
)
if estimator.trainer.hybridize:
predictor.hybridize(batch=next(iter(inference_data_loader)))
if self.hyperparameters.get('use_symbol_block_predictor'):
predictor = predictor.as_symbol_block_predictor(
batch=next(iter(inference_data_loader)))
num_eval_samples = self.hyperparameters.get('num_eval_samples', 100)
quantiles = self.hyperparameters.get(
'quantiles', (0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9))
# we only log aggregate metrics for now as item metrics may be
# very large
predictions, input_timeseries = backtest.make_evaluation_predictions(
test_dataset, predictor, num_eval_samples)
agg_metrics, _item_metrics = Evaluator(quantiles=quantiles)(
input_timeseries, predictions, num_series=len_filtered)
log.metric("agg_metrics", agg_metrics)
def train(epochs, prediction_length, num_layers, dropout_rate):
#create train dataset
df = pd.read_csv(filepath_or_buffer=os.environ['SM_CHANNEL_TRAIN'] +
"/train.csv",
header=0,
index_col=0)
training_data = ListDataset([{
"start": df.index[0],
"target": df.value[:]
}],
freq="5min")
#define DeepAR estimator
deepar_estimator = DeepAREstimator(freq="5min",
prediction_length=prediction_length,
dropout_rate=dropout_rate,
num_layers=num_layers,
trainer=Trainer(epochs=epochs))
#train the model
deepar_predictor = deepar_estimator.train(training_data=training_data)
#create test dataset
df = pd.read_csv(filepath_or_buffer=os.environ['SM_CHANNEL_TEST'] +
"/test.csv",
header=0,
index_col=0)
test_data = ListDataset([{
"start": df.index[0],
"target": df.value[:]
}],
freq="5min")
#evaluate trained model on test data
forecast_it, ts_it = make_evaluation_predictions(test_data,
deepar_predictor,
num_samples=100)
forecasts = list(forecast_it)
tss = list(ts_it)
evaluator = Evaluator(quantiles=[0.1, 0.5, 0.9])
agg_metrics, item_metrics = evaluator(iter(tss),
iter(forecasts),
num_series=len(test_data))
print("MSE:", agg_metrics["MSE"])
#save the model
deepar_predictor.serialize(pathlib.Path(os.environ['SM_MODEL_DIR']))
return deepar_predictor
def train(arguments):
"""
Generic train method that trains a specified estimator on a specified
dataset.
"""
logger.info("Downloading estimator config.")
estimator_config = Path(arguments.estimator) / "estimator.json"
with estimator_config.open() as config_file:
estimator = serde.load_json(config_file.read())
logger.info("Downloading dataset.")
if arguments.s3_dataset is None:
# load built in dataset
dataset = datasets.get_dataset(arguments.dataset)
else:
# load custom dataset
s3_dataset_dir = Path(arguments.s3_dataset)
dataset = common.load_datasets(
metadata=s3_dataset_dir,
train=s3_dataset_dir / "train",
test=s3_dataset_dir / "test",
)
logger.info("Starting model training.")
predictor = estimator.train(dataset.train)
forecast_it, ts_it = backtest.make_evaluation_predictions(
dataset=dataset.test,
predictor=predictor,
num_samples=int(arguments.num_samples),
)
logger.info("Starting model evaluation.")
evaluator = Evaluator(quantiles=eval(arguments.quantiles))
agg_metrics, item_metrics = evaluator(ts_it,
forecast_it,
num_series=len(list(dataset.test)))
# required for metric tracking.
for name, value in agg_metrics.items():
logger.info(f"gluonts[metric-{name}]: {value}")
# save the evaluation results
metrics_output_dir = Path(arguments.output_data_dir)
with open(metrics_output_dir / "agg_metrics.json", "w") as f:
json.dump(agg_metrics, f)
with open(metrics_output_dir / "item_metrics.csv", "w") as f:
item_metrics.to_csv(f, index=False)
# save the model
model_output_dir = Path(arguments.model_dir)
predictor.serialize(model_output_dir)
def mse(net,test):
predictor = estimator.create_predictor(transformation,net)
forecast_it, ts_it = make_evaluation_predictions(
dataset=test, # test dataset
predictor=predictor, # predictor
num_samples=100, # number of sample paths we want for evaluation
)
forecasts = list(forecast_it)
tss = list(ts_it)
evaluator = Evaluator(quantiles=[0.1, 0.5, 0.9])
agg_metrics, item_metrics = evaluator(iter(tss), iter(forecasts), num_series=len(test))
return agg_metrics['MSE']
def evaluate(
model: Predictor, test_data: ListDataset, num_samples: int
) -> Tuple[List[Forecast], List[pd.Series], Dict[str, float], pd.DataFrame]:
forecast_it, ts_it = make_evaluation_predictions(dataset=test_data,
predictor=model,
num_samples=num_samples)
forecasts = list(forecast_it)
tss = list(ts_it)
evaluator = MultivariateEvaluator()
agg_metrics, item_metrics = evaluator(iter(tss),
iter(forecasts),
num_series=len(test_data))
return forecasts, tss, agg_metrics, item_metrics
def plot(
self,
num_viz: int = 10,
) -> None:
forecasts, _ = make_evaluation_predictions(
dataset=self.test_dataset,
predictor=self.predictor,
num_samples=self.num_samples)
fig, ax = plt.subplots(num_viz,
1,
sharex=True,
sharey=True,
figsize=(10, 7))
for i, (series,
fcast) in enumerate(zip(
self.original_series,
self.forecasts,
)):
if i == num_viz:
break
# plot original, uninterpolated series
index = pd.date_range(start=series['start'],
freq=self.predictor.freq,
periods=series['target'].shape[-1])
if series['target'].ndim > 1:
ts = pd.Series(series['target'][0], index=index)
else:
ts = pd.Series(series['target'], index=index)
ts.plot(ax=ax[i], legend=False)
# plot mean/median forecast
if self.predictor.freq == 'W':
offset = pd.DateOffset(weeks=self.predictor.prediction_length -
1)
elif self.predictor.freq == 'M':
offset = pd.DateOffset(
months=self.predictor.prediction_length - 1)
fcast_index = pd.date_range(
start=index[-1] - offset,
freq=self.predictor.freq,
periods=self.predictor.prediction_length)
fcast = pd.Series(fcast, index=fcast_index)
fcast.plot(color='g', ax=ax[i], legend=False)
plt.savefig(f'plots/visualize_forecast_gluon.png')
def test_training_external_features(self):
prediction_length = 2
frequency = "3M"
gluon_dataset = ListDataset(self.timeseries, freq=frequency)
estimator = AutoARIMAEstimator(prediction_length=prediction_length, freq=frequency, season_length=4, use_feat_dynamic_real=True)
predictor = estimator.train(gluon_dataset)
forecast_it, ts_it = make_evaluation_predictions(dataset=gluon_dataset, predictor=predictor, num_samples=100)
timeseries = list(ts_it)
forecasts = list(forecast_it)
assert forecasts[1].samples.shape == (100, 2)
evaluator = Evaluator()
agg_metrics, item_metrics = evaluator(iter(timeseries), iter(forecasts), num_series=len(gluon_dataset))
assert agg_metrics["MAPE"] is not None
def run_test(
env: TrainEnv, predictor: Predictor, test_dataset: Dataset
) -> None:
len_original = maybe_len(test_dataset)
test_dataset = TransformedDataset(
test_dataset,
FilterTransformation(
lambda x: x["target"].shape[-1] > predictor.prediction_length
),
)
len_filtered = len(test_dataset)
if len_original is not None and len_original > len_filtered:
logger.warning(
f"Not all time-series in the test-channel have "
f"enough data to be used for evaluation. Proceeding with "
f"{len_filtered}/{len_original} "
f"(~{int(len_filtered / len_original * 100)}%) items."
)
forecast_it, ts_it = backtest.make_evaluation_predictions(
dataset=test_dataset, predictor=predictor, num_samples=100
)
if isinstance(predictor, RepresentableBlockPredictor) and isinstance(
predictor.forecast_generator, QuantileForecastGenerator
):
quantiles = predictor.forecast_generator.quantiles
logger.info(f"Using quantiles `{quantiles}` for evaluation.")
evaluator = Evaluator(quantiles=quantiles)
else:
evaluator = Evaluator()
agg_metrics, item_metrics = evaluator(
ts_iterator=ts_it,
fcst_iterator=forecast_it,
num_series=len(test_dataset),
)
# we only log aggregate metrics for now as item metrics may be very large
for name, score in agg_metrics.items():
logger.info(f"#test_score ({env.current_host}, {name}): {score}")
# store metrics
with open(env.path.model / "agg_metrics.json", "w") as agg_metric_file:
json.dump(agg_metrics, agg_metric_file)
with open(env.path.model / "item_metrics.csv", "w") as item_metrics_file:
item_metrics.to_csv(item_metrics_file, index=False)
def build_deepar_model():
# get the financial data "exchange_rate"
gluon_data = get_dataset("exchange_rate", regenerate=True)
train_data = next(iter(gluon_data.train))
test_data = next(iter(gluon_data.test))
meta_data = gluon_data.metadata
# data set visualisation
fig, ax = plt.subplots(1, 1, figsize=(11, 8))
to_pandas(train_data).plot(ax=ax)
ax.grid(which="both")
ax.legend(["train data"], loc="upper left")
plt.savefig("dataset.png")
# visualize various members of the 'gluon_data.*'
print(train_data.keys())
print(test_data.keys())
print(meta_data)
# convert dataset into an object recognised by GluonTS
training_data = common.ListDataset(gluon_data.train, freq=meta_data.freq)
testing_data = common.ListDataset(gluon_data.test, freq=meta_data.freq)
# create an Estimator with DeepAR
# an object of Trainer() class is used to customize Estimator
estimator = deepar.DeepAREstimator(
freq=meta_data.freq,
prediction_length=meta_data.prediction_length,
trainer=Trainer(ctx="cpu", epochs=100, learning_rate=1e-4))
# create a Predictor by training the Estimator with training dataset
predictor = estimator.train(training_data=training_data)
# make predictions
forecasts, test_series = make_evaluation_predictions(dataset=testing_data,
predictor=predictor,
num_samples=10)
# visualise forecasts
prediction_intervals = (50.0, 90.0)
legend = ["actual data", "median forecast"
] + [f"{k}% forecast interval"
for k in prediction_intervals][::-1]
fig, ax = plt.subplots(1, 1, figsize=(11, 8))
list(test_series)[0][-150:].plot(ax=ax) # plot the time series
list(forecasts)[0].plot(prediction_intervals=prediction_intervals,
color='r')
plt.grid(which="both")
plt.legend(legend, loc="upper left")
plt.savefig("deepar-model.png")
def deep_state(seed=42, data="m4_quarterly", epochs=100, batches=50):
mx.random.seed(seed)
np.random.seed(seed)
dataset = get_dataset(data)
trainer = Trainer(
ctx=mx.cpu(0),
# ctx=mx.gpu(0),
epochs=epochs,
num_batches_per_epoch=batches,
learning_rate=1e-3,
)
cardinality = int(dataset.metadata.feat_static_cat[0].cardinality)
estimator = DeepStateEstimator(
trainer=trainer,
cardinality=[cardinality],
prediction_length=dataset.metadata.prediction_length,
freq=dataset.metadata.freq,
use_feat_static_cat=True,
)
predictor = estimator.train(dataset.train)
# predictor = estimator.train(training_data=dataset.train,
# validation_data=dataset.test)
forecast_it, ts_it = make_evaluation_predictions(dataset.test,
predictor=predictor,
num_samples=100)
agg_metrics, item_metrics = Evaluator()(ts_it,
forecast_it,
num_series=len(dataset.test))
metrics = [
"MASE", "sMAPE", "MSIS", "wQuantileLoss[0.5]", "wQuantileLoss[0.9]"
]
output = {
key: round(value, 8)
for key, value in agg_metrics.items() if key in metrics
}
output["epochs"] = epochs
output["seed"] = seed
df = pd.DataFrame([output])
return df