def get_metrics_per_ts(
self, time_series: Union[pd.Series, pd.DataFrame],
forecast: Forecast) -> Dict[str, Union[float, str, None]]:
pred_target = np.array(self.extract_pred_target(time_series, forecast))
pred_target = np.ma.masked_invalid(pred_target)
# required for seasonal_error and owa calculation
past_data = np.array(self.extract_past_data(time_series, forecast))
past_data = np.ma.masked_invalid(past_data)
try:
mean_fcst = forecast.mean
except:
mean_fcst = None
median_fcst = forecast.quantile(0.5)
seasonal_error = self.seasonal_error(past_data, forecast)
metrics = {
"item_id": forecast.item_id,
"MSE": self.mse(pred_target, mean_fcst)
if mean_fcst is not None else None,
"abs_error": self.abs_error(pred_target, median_fcst),
"abs_target_sum": self.abs_target_sum(pred_target),
"abs_target_mean": self.abs_target_mean(pred_target),
"seasonal_error": seasonal_error,
"MASE": self.mase(pred_target, median_fcst, seasonal_error),
"MAPE": self.mape(pred_target, median_fcst),
"sMAPE": self.smape(pred_target, median_fcst),
"OWA": np.nan, # by default not calculated
}
try:
metrics["MSIS"] = self.msis(
pred_target,
forecast.quantile(self.alpha / 2),
forecast.quantile(1.0 - self.alpha / 2),
seasonal_error,
self.alpha,
)
except Exception:
logging.warning("Could not calculate MSIS metric.")
metrics["MSIS"] = np.nan
if self.calculate_owa:
metrics["OWA"] = self.owa(
pred_target,
median_fcst,
past_data,
seasonal_error,
forecast.start_date,
)
for quantile in self.quantiles:
forecast_quantile = forecast.quantile(quantile.value)
metrics[quantile.loss_name] = self.quantile_loss(
pred_target, forecast_quantile, quantile.value)
metrics[quantile.coverage_name] = self.coverage(
pred_target, forecast_quantile)
return metrics
def get_metrics_per_ts(
self, time_series: Union[pd.Series, pd.DataFrame],
forecast: Forecast) -> Dict[str, Union[float, str, None]]:
# Inverse tranformation (if any), then clip to 0.
if self.gt_inverse_transform is not None:
time_series = self.gt_inverse_transform(time_series)
if self.clip_at_zero:
time_series = time_series.clip(lower=0.0)
# Compute the built-in metrics
metrics = super().get_metrics_per_ts(time_series, forecast)
# Write forecast results to output file.
result: Dict[str, Any] = {
"item_id": str(forecast.item_id),
**forecast.as_json_dict(output_configuration)
}
json.dump(result, self.out_f)
self.out_f.write("\n")
# region: custom metrics.
# Follow gluonts.evaluation.Evaluator who uses median
pred_target = np.array(self.extract_pred_target(time_series, forecast))
pred_target = np.ma.masked_invalid(pred_target)
median_fcst = forecast.quantile(0.5)
# And here comes custom metrics...
metrics["wMAPE"] = wmape(pred_target, median_fcst, version=2)
# endregion: custom metrics
# Add to montage
self.plot_prob_forecasts(self.mp.pop(forecast.item_id), time_series,
forecast, self.plot_ci)
return metrics
def get_metrics_per_ts(
self, time_series: Union[pd.Series, pd.DataFrame],
forecast: Forecast) -> Dict[str, Union[float, str, None]]:
pred_target = np.array(self.extract_pred_target(time_series, forecast))
pred_target = np.ma.masked_invalid(pred_target)
try:
mean_fcst = forecast.mean
except:
mean_fcst = None
median_fcst = forecast.quantile(0.5)
seasonal_error = self.seasonal_error(time_series, forecast)
# For MSIS: alpha/2 quantile may not exist. Find the closest.
lower_q = min(self.quantiles,
key=lambda q: abs(q.value - self.alpha / 2))
upper_q = min(
reversed(self.quantiles),
key=lambda q: abs(q.value - (1 - self.alpha / 2)),
)
metrics = {
"item_id":
forecast.item_id,
"MSE":
self.mse(pred_target, mean_fcst)
if mean_fcst is not None else None,
"abs_error":
self.abs_error(pred_target, median_fcst),
"abs_target_sum":
self.abs_target_sum(pred_target),
"abs_target_mean":
self.abs_target_mean(pred_target),
"seasonal_error":
seasonal_error,
"MASE":
self.mase(pred_target, median_fcst, seasonal_error),
"sMAPE":
self.smape(pred_target, median_fcst),
"MSIS":
self.msis(
pred_target,
forecast.quantile(lower_q.value),
forecast.quantile(upper_q.value),
seasonal_error,
self.alpha,
),
}
for quantile in self.quantiles:
forecast_quantile = forecast.quantile(quantile.value)
metrics[quantile.loss_name] = self.quantile_loss(
pred_target, forecast_quantile, quantile.value)
metrics[quantile.coverage_name] = self.coverage(
pred_target, forecast_quantile)
return metrics
def get_metrics_per_ts(self, time_series: Union[pd.Series, pd.DataFrame],
forecast: Forecast) -> Dict[str, float]:
pred_target = np.array(self.extract_pred_target(time_series, forecast))
pred_target = np.ma.masked_invalid(pred_target)
mean_fcst = forecast.mean
median_fcst = forecast.quantile(0.5)
seasonal_error = self.seasonal_error(time_series, forecast)
# For MSIS: alpha/2 quantile may not exist. Find the closest.
lower_q = min(self.quantile_values,
key=lambda x: abs(x - self.alpha / 2))
upper_q = min(
list(reversed(self.quantile_values)),
key=lambda x: abs(x - (1 - self.alpha / 2)),
)
metrics = {
"item_id":
forecast.item_id,
"MSE":
self.mse(pred_target, mean_fcst),
"abs_error":
self.abs_error(pred_target, median_fcst),
"abs_target_sum":
self.abs_target_sum(pred_target),
"abs_target_mean":
self.abs_target_mean(pred_target),
"seasonal_error":
seasonal_error,
"MASE":
self.mase(pred_target, median_fcst, seasonal_error),
'sMAPE':
self.smape(pred_target, median_fcst),
'MSIS':
self.msis(
pred_target,
forecast.quantile(lower_q),
forecast.quantile(upper_q),
seasonal_error,
self.alpha,
),
}
for q, q_name in zip(self.quantile_values, self.quantile_names):
m = 'QuantileLoss[{}]'.format(q_name)
metrics[m] = self.quantile_loss(pred_target, forecast.quantile(q),
q)
m = 'Coverage[{}]'.format(q_name)
metrics[m] = self.coverage(pred_target, forecast.quantile(q))
return metrics
def get_target_dimensionality(forecast: Forecast) -> int:
target_dim = forecast.dim()
assert target_dim > 1, (
f'the dimensionality of the forecast should be larger than 1, but got {target_dim}. '
f'Please use the Evaluator to evaluate 1D forecasts.'
)
return target_dim
def __init__(
self,
input_names: List[str],
prediction_net: BlockType,
batch_size: int,
prediction_length: int,
freq: str,
ctx: mx.Context,
input_transform: Transformation,
output_transform: Optional[Callable[[DataEntry, np.ndarray],
np.ndarray]] = None,
float_type: DType = np.float32,
forecast_cls_name: str = "SampleForecast",
forecast_kwargs: Optional[Dict] = None,
) -> None:
super().__init__(prediction_length, freq)
forecast_dict = {c.__name__: c for c in Forecast.__subclasses__()}
assert forecast_cls_name in forecast_dict
self.input_names = input_names
self.prediction_net = prediction_net
self.batch_size = batch_size
self.input_transform = input_transform
self.output_transform = output_transform
self.ctx = ctx
self.float_type = float_type
self.forecast_cls_name = forecast_cls_name
self._forecast_cls = forecast_dict[forecast_cls_name]
self.forecast_kwargs = forecast_kwargs if forecast_kwargs else {}
def process(forecast: Forecast) -> dict:
prediction = {}
if 'samples' in config.output_types:
if isinstance(forecast, SampleForecast):
prediction['samples'] = forecast.samples.tolist()
else:
prediction['samples'] = []
if 'mean' in config.output_types:
prediction['mean'] = forecast.mean.tolist()
if 'quantiles' in config.output_types:
prediction['quantiles'] = {
q: forecast.quantile(q).tolist()
for q in config.quantiles
}
return prediction
def get_metrics_per_ts(
self, time_series: Union[pd.Series, pd.DataFrame], forecast: Forecast
) -> Mapping[str, Union[float, str, None, np.ma.core.MaskedConstant]]:
pred_target = np.array(self.extract_pred_target(time_series, forecast))
past_data = np.array(self.extract_past_data(time_series, forecast))
if self.ignore_invalid_values:
past_data = np.ma.masked_invalid(past_data)
pred_target = np.ma.masked_invalid(pred_target)
try:
mean_fcst = getattr(forecast, "mean", None)
except NotImplementedError:
mean_fcst = None
median_fcst = forecast.quantile(0.5)
seasonal_error = calculate_seasonal_error(past_data, forecast,
self.seasonality)
metrics: Dict[str, Union[float, str, None]] = {
"item_id": forecast.item_id,
"MSE":
mse(pred_target, mean_fcst) if mean_fcst is not None else None,
"abs_error": abs_error(pred_target, median_fcst),
"abs_target_sum": abs_target_sum(pred_target),
"abs_target_mean": abs_target_mean(pred_target),
"seasonal_error": seasonal_error,
"MASE": mase(pred_target, median_fcst, seasonal_error),
"MAPE": mape(pred_target, median_fcst),
"sMAPE": smape(pred_target, median_fcst),
}
if self.custom_eval_fn is not None:
for k, (eval_fn, _, fcst_type) in self.custom_eval_fn.items():
if fcst_type == "mean":
if mean_fcst is not None:
target_fcst = mean_fcst
else:
logging.warning(
"mean_fcst is None, therefore median_fcst is used."
)
target_fcst = median_fcst
else:
target_fcst = median_fcst
try:
val = {
k: eval_fn(
pred_target,
target_fcst,
)
}
except Exception:
logging.warning(f"Error occured when evaluating {k}.")
val = {k: np.nan}
metrics.update(val)
try:
metrics["MSIS"] = msis(
pred_target,
forecast.quantile(self.alpha / 2),
forecast.quantile(1.0 - self.alpha / 2),
seasonal_error,
self.alpha,
)
except Exception:
logging.warning("Could not calculate MSIS metric.")
metrics["MSIS"] = np.nan
if self.calculate_owa:
from gluonts.model.naive_2 import naive_2
naive_median_forecast = naive_2(past_data,
len(pred_target),
freq=forecast.start_date.freqstr)
metrics["sMAPE_naive2"] = smape(pred_target, naive_median_forecast)
metrics["MASE_naive2"] = mase(pred_target, naive_median_forecast,
seasonal_error)
for quantile in self.quantiles:
forecast_quantile = forecast.quantile(quantile.value)
metrics[quantile.loss_name] = quantile_loss(
pred_target, forecast_quantile, quantile.value)
metrics[quantile.coverage_name] = coverage(pred_target,
forecast_quantile)
return metrics
def get_metrics_per_ts(
self, time_series: Union[pd.Series, pd.DataFrame],
forecast: Forecast) -> Dict[str, Union[float, str, None]]:
pred_target = np.array(self.extract_pred_target(time_series, forecast))
pred_target = np.ma.masked_invalid(pred_target)
# required for seasonal_error and owa calculation
past_data = np.array(self.extract_past_data(time_series, forecast))
past_data = np.ma.masked_invalid(past_data)
if self.median:
fcst = forecast.quantile(0.5)
else:
fcst = forecast.mean
naive_fc = self.naive_fc(time_series, forecast)
seasonal_error = np.abs(naive_fc - pred_target)
seasonal_sq_error = np.square(naive_fc - pred_target)
nonzero_mask = pred_target > 0
metrics = {
"item_id":
forecast.item_id,
"non_zero_n":
np.sum(nonzero_mask),
"n":
len(pred_target),
"abs_target_sum":
self.abs_target_sum(pred_target),
"abs_target_mean":
self.abs_target_mean(pred_target),
"seasonal_error":
seasonal_error.sum(),
"seasonal_sq_error":
seasonal_sq_error.sum(),
"abs_error":
self.abs_error(pred_target, fcst),
"cum_error":
self.cum_error(pred_target, fcst),
"CFE":
self.signed_error(pred_target, fcst),
"CFE_min":
np.min(self.cum_error(pred_target, fcst)),
"CFE_max":
np.max(self.cum_error(pred_target, fcst)),
"NOSp":
self.nos_p(pred_target, fcst),
"PIS":
self.pis(pred_target, fcst),
"MSE":
self.mse(pred_target, fcst),
"RMSE":
np.sqrt(self.mse(pred_target, fcst)),
"MAE":
self.mae(pred_target, fcst),
"MRAE":
self.mean_relative_abs_error(pred_target, fcst, naive_fc),
# "MSEn": self.mse(pred_target[nonzero_mask], fcst[nonzero_mask]),
# "MAEn": self.mae(pred_target[nonzero_mask], fcst[nonzero_mask]),
"MASE":
self.mase(pred_target, fcst, seasonal_error.mean()),
"MAPE":
self.mape(pred_target, fcst),
"RelMAE":
self.mae(pred_target, fcst) / self.mae(pred_target, naive_fc),
"RelRMSE":
np.sqrt(self.mse(pred_target, fcst)) /
np.sqrt(self.mse(pred_target, naive_fc)),
"PBMAE":
self.percent_better(pred_target, fcst, naive_fc),
"MAAPE":
self.maape(pred_target, fcst),
"sMAPE":
self.smape(pred_target, fcst),
"SPEC_0.75":
self.spec(pred_target, fcst, a1=0.75, a2=0.25),
"SPEC_0.5":
self.spec(pred_target, fcst, a1=0.5, a2=0.5),
"SPEC_0.25":
self.spec(pred_target, fcst, a1=0.25, a2=0.75)
}
if self.calculate_spec:
metrics["SPEC_0.75"] = self.spec(pred_target,
fcst,
a1=0.75,
a2=0.25),
metrics["SPEC_0.5"] = self.spec(pred_target, fcst, a1=0.5, a2=0.5),
metrics["SPEC_0.25"] = self.spec(pred_target,
fcst,
a1=0.25,
a2=0.75)
for quantile in self.quantiles:
forecast_quantile = forecast.quantile(quantile.value)
metrics[quantile.loss_name] = self.quantile_loss(
pred_target, forecast_quantile, quantile.value)
metrics[quantile.coverage_name] = self.coverage(
pred_target, forecast_quantile)
return metrics
def get_metrics_per_ts(
self, time_series: Union[pd.Series, pd.DataFrame],
forecast: Forecast) -> Dict[str, Union[float, str, None]]:
pred_target = np.array(self.extract_pred_target(time_series, forecast))
pred_target = np.ma.masked_invalid(pred_target)
# required for seasonal_error and owa calculation
past_data = np.array(self.extract_past_data(time_series, forecast))
past_data = np.ma.masked_invalid(past_data)
try:
mean_fcst = forecast.mean
except:
mean_fcst = None
median_fcst = forecast.quantile(0.5)
seasonal_error = calculate_seasonal_error(past_data, forecast,
self.seasonality)
metrics: Dict[str, Union[float, str, None]] = {
"item_id": forecast.item_id,
"MSE":
mse(pred_target, mean_fcst) if mean_fcst is not None else None,
"abs_error": abs_error(pred_target, median_fcst),
"abs_target_sum": abs_target_sum(pred_target),
"abs_target_mean": abs_target_mean(pred_target),
"seasonal_error": seasonal_error,
"MASE": mase(pred_target, median_fcst, seasonal_error),
"MAPE": mape(pred_target, median_fcst),
"sMAPE": smape(pred_target, median_fcst),
"OWA": np.nan, # by default not calculated
}
if self.custom_eval_fn is not None:
for k, (eval_fn, _, fcst_type) in self.custom_eval_fn.items():
if fcst_type == "mean":
if mean_fcst is not None:
target_fcst = mean_fcst
else:
logging.warning(
"mean_fcst is None, therefore median_fcst is used."
)
target_fcst = median_fcst
else:
target_fcst = median_fcst
try:
val = {
k: eval_fn(
pred_target,
target_fcst,
)
}
except:
val = {k: np.nan}
metrics.update(val)
try:
metrics["MSIS"] = msis(
pred_target,
forecast.quantile(self.alpha / 2),
forecast.quantile(1.0 - self.alpha / 2),
seasonal_error,
self.alpha,
)
except Exception:
logging.warning("Could not calculate MSIS metric.")
metrics["MSIS"] = np.nan
if self.calculate_owa:
metrics["OWA"] = owa(
pred_target,
median_fcst,
past_data,
seasonal_error,
forecast.start_date,
)
for quantile in self.quantiles:
forecast_quantile = forecast.quantile(quantile.value)
metrics[quantile.loss_name] = quantile_loss(
pred_target, forecast_quantile, quantile.value)
metrics[quantile.coverage_name] = coverage(pred_target,
forecast_quantile)
return metrics
