def owa(
target: np.ndarray,
forecast: np.ndarray,
past_data: np.ndarray,
seasonal_error: float,
start_date: pd.Timestamp,
) -> float:
r"""
.. math::
owa = 0.5*(smape/smape_naive + mase/mase_naive)
https://www.m4.unic.ac.cy/wp-content/uploads/2018/03/M4-Competitors-Guide.pdf
"""
# avoid import error due to circular dependency
from gluonts.model.naive_2 import naive_2
# calculate the forecast of the seasonal naive predictor
naive_median_fcst = naive_2(
past_data, len(target), freq=start_date.freqstr
)
owa = 0.5 * (
(
Evaluator.smape(target, forecast)
/ Evaluator.smape(target, naive_median_fcst)
)
+ (
Evaluator.mase(target, forecast, seasonal_error)
/ Evaluator.mase(target, naive_median_fcst, seasonal_error)
)
)
return owa
def owa(
target: np.ndarray,
forecast: np.ndarray,
past_data: np.ndarray,
seasonal_error: float,
start_date: pd.Timestamp,
) -> float:
r"""
.. math::
owa = 0.5*(smape / smape\_naive + mase / mase\_naive)
See [SSA20]_ for more details.
"""
# avoid import error due to circular dependency
from gluonts.model.naive_2 import naive_2
# calculate the forecast of the seasonal naive predictor
naive_median_fcst = naive_2(
past_data, len(target), freq=start_date.freqstr
)
return 0.5 * (
(smape(target, forecast) / smape(target, naive_median_fcst))
+ (
mase(target, forecast, seasonal_error)
/ mase(target, naive_median_fcst, seasonal_error)
)
)
def test_naive_2(prediction_length=FH, season_length=FRQ):
r_naive_2_inputs, r_naive_2_outputs = load_naive_2_data()
predictions = []
for i in range(len(r_naive_2_inputs)):
predictions.append(
naive_2(
r_naive_2_inputs[i],
prediction_length=prediction_length,
season_length=season_length,
))
predictions = np.array(predictions)
assert np.allclose(r_naive_2_outputs, predictions)
def get_metrics_per_ts(
self, time_series: Union[pd.Series, pd.DataFrame], forecast: Forecast
) -> Mapping[str, Union[float, str, None, np.ma.core.MaskedConstant]]:
pred_target = np.array(self.extract_pred_target(time_series, forecast))
past_data = np.array(self.extract_past_data(time_series, forecast))
if self.ignore_invalid_values:
past_data = np.ma.masked_invalid(past_data)
pred_target = np.ma.masked_invalid(pred_target)
try:
mean_fcst = getattr(forecast, "mean", None)
except NotImplementedError:
mean_fcst = None
median_fcst = forecast.quantile(0.5)
seasonal_error = calculate_seasonal_error(past_data, forecast,
self.seasonality)
metrics: Dict[str, Union[float, str, None]] = {
"item_id": forecast.item_id,
"MSE":
mse(pred_target, mean_fcst) if mean_fcst is not None else None,
"abs_error": abs_error(pred_target, median_fcst),
"abs_target_sum": abs_target_sum(pred_target),
"abs_target_mean": abs_target_mean(pred_target),
"seasonal_error": seasonal_error,
"MASE": mase(pred_target, median_fcst, seasonal_error),
"MAPE": mape(pred_target, median_fcst),
"sMAPE": smape(pred_target, median_fcst),
}
if self.custom_eval_fn is not None:
for k, (eval_fn, _, fcst_type) in self.custom_eval_fn.items():
if fcst_type == "mean":
if mean_fcst is not None:
target_fcst = mean_fcst
else:
logging.warning(
"mean_fcst is None, therefore median_fcst is used."
)
target_fcst = median_fcst
else:
target_fcst = median_fcst
try:
val = {
k: eval_fn(
pred_target,
target_fcst,
)
}
except Exception:
logging.warning(f"Error occured when evaluating {k}.")
val = {k: np.nan}
metrics.update(val)
try:
metrics["MSIS"] = msis(
pred_target,
forecast.quantile(self.alpha / 2),
forecast.quantile(1.0 - self.alpha / 2),
seasonal_error,
self.alpha,
)
except Exception:
logging.warning("Could not calculate MSIS metric.")
metrics["MSIS"] = np.nan
if self.calculate_owa:
from gluonts.model.naive_2 import naive_2
naive_median_forecast = naive_2(past_data,
len(pred_target),
freq=forecast.start_date.freqstr)
metrics["sMAPE_naive2"] = smape(pred_target, naive_median_forecast)
metrics["MASE_naive2"] = mase(pred_target, naive_median_forecast,
seasonal_error)
for quantile in self.quantiles:
forecast_quantile = forecast.quantile(quantile.value)
metrics[quantile.loss_name] = quantile_loss(
pred_target, forecast_quantile, quantile.value)
metrics[quantile.coverage_name] = coverage(pred_target,
forecast_quantile)
return metrics
