def test_training_with_implicit_quantile_output(): dataset = get_dataset("constant") metadata = dataset.metadata deepar_estimator = DeepAREstimator( distr_output=ImplicitQuantileOutput(output_domain="Real"), freq=metadata.freq, prediction_length=metadata.prediction_length, trainer=Trainer(device="cpu", epochs=5, learning_rate=1e-3, num_batches_per_epoch=3, batch_size=256, num_workers=1, ), input_size=48, ) deepar_predictor = deepar_estimator.train(dataset.train) forecast_it, ts_it = make_evaluation_predictions( dataset=dataset.test, # test dataset predictor=deepar_predictor, # predictor num_samples=100, # number of sample paths we want for evaluation ) forecasts = list(forecast_it) tss = list(ts_it) evaluator = Evaluator() agg_metrics, item_metrics = evaluator(iter(tss), iter(forecasts), num_series=len(dataset.test)) assert agg_metrics["MSE"] > 0
def test_metrics(timeseries, res, has_nans, input_type): ts_datastructure = pd.Series evaluator = Evaluator(quantiles=QUANTILES) agg_metrics, item_metrics = calculate_metrics( timeseries, evaluator, ts_datastructure, has_nans=has_nans, input_type=input_type, ) for metric, score in agg_metrics.items(): if metric in res.keys(): assert abs(score - res[metric]) < 0.001, ( "Scores for the metric {} do not match: \nexpected: {} " "\nobtained: {}".format(metric, res[metric], score))
def test_evaluation_with_QuantileForecast(): start = "2012-01-11" target = [2.4, 1.0, 3.0, 4.4, 5.5, 4.9] * 11 index = pd.date_range(start=start, freq="1D", periods=len(target)) ts = pd.Series(index=index, data=target) ev = Evaluator(quantiles=("0.1", "0.2", "0.5")) fcst = [ QuantileForecast( start_date=pd.Timestamp("2012-01-11"), freq="D", forecast_arrays=np.array([[2.4, 9.0, 3.0, 2.4, 5.5, 4.9] * 10]), forecast_keys=["0.5"], ) ] agg_metric, _ = ev(iter([ts]), iter(fcst)) assert np.isfinite(agg_metric["wQuantileLoss[0.5]"])
def test_MASE_sMAPE_M4(timeseries, res): ts_datastructure = pd.Series evaluator = Evaluator(quantiles=QUANTILES) agg_df, item_df = calculate_metrics(timeseries, evaluator, ts_datastructure) assert abs((agg_df["MASE"] - res["MASE"]) / res["MASE"]) < 0.001, ( "Scores for the metric MASE do not match: " "\nexpected: {} \nobtained: {}".format(res["MASE"], agg_df["MASE"])) assert abs((agg_df["MAPE"] - res["MAPE"]) / res["MAPE"]) < 0.001, ( "Scores for the metric MAPE do not match: \nexpected: {} " "\nobtained: {}".format(res["MAPE"], agg_df["MAPE"])) assert abs((agg_df["sMAPE"] - res["sMAPE"]) / res["sMAPE"]) < 0.001, ( "Scores for the metric sMAPE do not match: \nexpected: {} " "\nobtained: {}".format(res["sMAPE"], agg_df["sMAPE"])) assert ( sum(abs(item_df["seasonal_error"].values - res["seasonal_error"])) < 0.001 ), ("Scores for the metric seasonal_error do not match: \nexpected: {} " "\nobtained: {}".format(res["seasonal_error"], item_df["seasonal_error"].values))
def test_instanciation_of_args_proj(): class MockedImplicitQuantileOutput(ImplicitQuantileOutput): method_calls = 0 @classmethod def set_args_proj(cls): super().set_args_proj() cls.method_calls += 1 dataset = get_dataset("constant") metadata = dataset.metadata distr_output = MockedImplicitQuantileOutput(output_domain="Real") deepar_estimator = DeepAREstimator( distr_output=distr_output, freq=metadata.freq, prediction_length=metadata.prediction_length, trainer=Trainer(device="cpu", epochs=1, learning_rate=1e-3, num_batches_per_epoch=1, batch_size=256, num_workers=1, ), input_size=48, ) assert distr_output.method_calls == 1 deepar_predictor = deepar_estimator.train(dataset.train) # Method should be called when the MockedImplicitQuantileOutput is instanciated, # and one more time because in_features is different from 1 assert distr_output.method_calls == 2 forecast_it, ts_it = make_evaluation_predictions( dataset=dataset.test, # test dataset predictor=deepar_predictor, # predictor num_samples=100, # number of sample paths we want for evaluation ) forecasts = list(forecast_it) tss = list(ts_it) evaluator = Evaluator() agg_metrics, item_metrics = evaluator(iter(tss), iter(forecasts), num_series=len(dataset.test)) assert distr_output.method_calls == 2 # Test that the implicit output module is proper reset new_estimator = DeepAREstimator( distr_output=MockedImplicitQuantileOutput(output_domain="Real"), freq=metadata.freq, prediction_length=metadata.prediction_length, trainer=Trainer(device="cpu", epochs=1, learning_rate=1e-3, num_batches_per_epoch=1, batch_size=256, num_workers=1, ), input_size=48, ) assert distr_output.method_calls == 3 new_estimator.train(dataset.train) assert distr_output.method_calls == 3 # Since in_feature is the same as before, there should be no additional call