def test_training_with_implicit_quantile_output():
dataset = get_dataset("constant")
metadata = dataset.metadata
deepar_estimator = DeepAREstimator(
distr_output=ImplicitQuantileOutput(output_domain="Real"),
freq=metadata.freq,
prediction_length=metadata.prediction_length,
trainer=Trainer(device="cpu",
epochs=5,
learning_rate=1e-3,
num_batches_per_epoch=3,
batch_size=256,
num_workers=1,
),
input_size=48,
)
deepar_predictor = deepar_estimator.train(dataset.train)
forecast_it, ts_it = make_evaluation_predictions(
dataset=dataset.test, # test dataset
predictor=deepar_predictor, # predictor
num_samples=100, # number of sample paths we want for evaluation
)
forecasts = list(forecast_it)
tss = list(ts_it)
evaluator = Evaluator()
agg_metrics, item_metrics = evaluator(iter(tss), iter(forecasts), num_series=len(dataset.test))
assert agg_metrics["MSE"] > 0
def test_metrics(timeseries, res, has_nans, input_type):
ts_datastructure = pd.Series
evaluator = Evaluator(quantiles=QUANTILES)
agg_metrics, item_metrics = calculate_metrics(
timeseries,
evaluator,
ts_datastructure,
has_nans=has_nans,
input_type=input_type,
)
for metric, score in agg_metrics.items():
if metric in res.keys():
assert abs(score - res[metric]) < 0.001, (
"Scores for the metric {} do not match: \nexpected: {} "
"\nobtained: {}".format(metric, res[metric], score))
def test_evaluation_with_QuantileForecast():
start = "2012-01-11"
target = [2.4, 1.0, 3.0, 4.4, 5.5, 4.9] * 11
index = pd.date_range(start=start, freq="1D", periods=len(target))
ts = pd.Series(index=index, data=target)
ev = Evaluator(quantiles=("0.1", "0.2", "0.5"))
fcst = [
QuantileForecast(
start_date=pd.Timestamp("2012-01-11"),
freq="D",
forecast_arrays=np.array([[2.4, 9.0, 3.0, 2.4, 5.5, 4.9] * 10]),
forecast_keys=["0.5"],
)
]
agg_metric, _ = ev(iter([ts]), iter(fcst))
assert np.isfinite(agg_metric["wQuantileLoss[0.5]"])
def test_MASE_sMAPE_M4(timeseries, res):
ts_datastructure = pd.Series
evaluator = Evaluator(quantiles=QUANTILES)
agg_df, item_df = calculate_metrics(timeseries, evaluator,
ts_datastructure)
assert abs((agg_df["MASE"] - res["MASE"]) / res["MASE"]) < 0.001, (
"Scores for the metric MASE do not match: "
"\nexpected: {} \nobtained: {}".format(res["MASE"], agg_df["MASE"]))
assert abs((agg_df["MAPE"] - res["MAPE"]) / res["MAPE"]) < 0.001, (
"Scores for the metric MAPE do not match: \nexpected: {} "
"\nobtained: {}".format(res["MAPE"], agg_df["MAPE"]))
assert abs((agg_df["sMAPE"] - res["sMAPE"]) / res["sMAPE"]) < 0.001, (
"Scores for the metric sMAPE do not match: \nexpected: {} "
"\nobtained: {}".format(res["sMAPE"], agg_df["sMAPE"]))
assert (
sum(abs(item_df["seasonal_error"].values - res["seasonal_error"])) <
0.001
), ("Scores for the metric seasonal_error do not match: \nexpected: {} "
"\nobtained: {}".format(res["seasonal_error"],
item_df["seasonal_error"].values))
def test_instanciation_of_args_proj():
class MockedImplicitQuantileOutput(ImplicitQuantileOutput):
method_calls = 0
@classmethod
def set_args_proj(cls):
super().set_args_proj()
cls.method_calls += 1
dataset = get_dataset("constant")
metadata = dataset.metadata
distr_output = MockedImplicitQuantileOutput(output_domain="Real")
deepar_estimator = DeepAREstimator(
distr_output=distr_output,
freq=metadata.freq,
prediction_length=metadata.prediction_length,
trainer=Trainer(device="cpu",
epochs=1,
learning_rate=1e-3,
num_batches_per_epoch=1,
batch_size=256,
num_workers=1,
),
input_size=48,
)
assert distr_output.method_calls == 1
deepar_predictor = deepar_estimator.train(dataset.train)
# Method should be called when the MockedImplicitQuantileOutput is instanciated,
# and one more time because in_features is different from 1
assert distr_output.method_calls == 2
forecast_it, ts_it = make_evaluation_predictions(
dataset=dataset.test, # test dataset
predictor=deepar_predictor, # predictor
num_samples=100, # number of sample paths we want for evaluation
)
forecasts = list(forecast_it)
tss = list(ts_it)
evaluator = Evaluator()
agg_metrics, item_metrics = evaluator(iter(tss), iter(forecasts), num_series=len(dataset.test))
assert distr_output.method_calls == 2
# Test that the implicit output module is proper reset
new_estimator = DeepAREstimator(
distr_output=MockedImplicitQuantileOutput(output_domain="Real"),
freq=metadata.freq,
prediction_length=metadata.prediction_length,
trainer=Trainer(device="cpu",
epochs=1,
learning_rate=1e-3,
num_batches_per_epoch=1,
batch_size=256,
num_workers=1,
),
input_size=48,
)
assert distr_output.method_calls == 3
new_estimator.train(dataset.train)
assert distr_output.method_calls == 3 # Since in_feature is the same as before, there should be no additional call