def owa( target: np.ndarray, forecast: np.ndarray, past_data: np.ndarray, seasonal_error: float, start_date: pd.Timestamp, ) -> float: r""" .. math:: owa = 0.5*(smape/smape_naive + mase/mase_naive) https://www.m4.unic.ac.cy/wp-content/uploads/2018/03/M4-Competitors-Guide.pdf """ # avoid import error due to circular dependency from gluonts.model.naive_2 import naive_2 # calculate the forecast of the seasonal naive predictor naive_median_fcst = naive_2( past_data, len(target), freq=start_date.freqstr ) owa = 0.5 * ( ( Evaluator.smape(target, forecast) / Evaluator.smape(target, naive_median_fcst) ) + ( Evaluator.mase(target, forecast, seasonal_error) / Evaluator.mase(target, naive_median_fcst, seasonal_error) ) ) return owa
def owa( target: np.ndarray, forecast: np.ndarray, past_data: np.ndarray, seasonal_error: float, start_date: pd.Timestamp, ) -> float: r""" .. math:: owa = 0.5*(smape / smape\_naive + mase / mase\_naive) See [SSA20]_ for more details. """ # avoid import error due to circular dependency from gluonts.model.naive_2 import naive_2 # calculate the forecast of the seasonal naive predictor naive_median_fcst = naive_2( past_data, len(target), freq=start_date.freqstr ) return 0.5 * ( (smape(target, forecast) / smape(target, naive_median_fcst)) + ( mase(target, forecast, seasonal_error) / mase(target, naive_median_fcst, seasonal_error) ) )
def test_naive_2(prediction_length=FH, season_length=FRQ): r_naive_2_inputs, r_naive_2_outputs = load_naive_2_data() predictions = [] for i in range(len(r_naive_2_inputs)): predictions.append( naive_2( r_naive_2_inputs[i], prediction_length=prediction_length, season_length=season_length, )) predictions = np.array(predictions) assert np.allclose(r_naive_2_outputs, predictions)
def get_metrics_per_ts( self, time_series: Union[pd.Series, pd.DataFrame], forecast: Forecast ) -> Mapping[str, Union[float, str, None, np.ma.core.MaskedConstant]]: pred_target = np.array(self.extract_pred_target(time_series, forecast)) past_data = np.array(self.extract_past_data(time_series, forecast)) if self.ignore_invalid_values: past_data = np.ma.masked_invalid(past_data) pred_target = np.ma.masked_invalid(pred_target) try: mean_fcst = getattr(forecast, "mean", None) except NotImplementedError: mean_fcst = None median_fcst = forecast.quantile(0.5) seasonal_error = calculate_seasonal_error(past_data, forecast, self.seasonality) metrics: Dict[str, Union[float, str, None]] = { "item_id": forecast.item_id, "MSE": mse(pred_target, mean_fcst) if mean_fcst is not None else None, "abs_error": abs_error(pred_target, median_fcst), "abs_target_sum": abs_target_sum(pred_target), "abs_target_mean": abs_target_mean(pred_target), "seasonal_error": seasonal_error, "MASE": mase(pred_target, median_fcst, seasonal_error), "MAPE": mape(pred_target, median_fcst), "sMAPE": smape(pred_target, median_fcst), } if self.custom_eval_fn is not None: for k, (eval_fn, _, fcst_type) in self.custom_eval_fn.items(): if fcst_type == "mean": if mean_fcst is not None: target_fcst = mean_fcst else: logging.warning( "mean_fcst is None, therefore median_fcst is used." ) target_fcst = median_fcst else: target_fcst = median_fcst try: val = { k: eval_fn( pred_target, target_fcst, ) } except Exception: logging.warning(f"Error occured when evaluating {k}.") val = {k: np.nan} metrics.update(val) try: metrics["MSIS"] = msis( pred_target, forecast.quantile(self.alpha / 2), forecast.quantile(1.0 - self.alpha / 2), seasonal_error, self.alpha, ) except Exception: logging.warning("Could not calculate MSIS metric.") metrics["MSIS"] = np.nan if self.calculate_owa: from gluonts.model.naive_2 import naive_2 naive_median_forecast = naive_2(past_data, len(pred_target), freq=forecast.start_date.freqstr) metrics["sMAPE_naive2"] = smape(pred_target, naive_median_forecast) metrics["MASE_naive2"] = mase(pred_target, naive_median_forecast, seasonal_error) for quantile in self.quantiles: forecast_quantile = forecast.quantile(quantile.value) metrics[quantile.loss_name] = quantile_loss( pred_target, forecast_quantile, quantile.value) metrics[quantile.coverage_name] = coverage(pred_target, forecast_quantile) return metrics