def _evaluate(self, true, pred):
"""Evaluate continuous ranked probability score (CRPS).
CRPS depends on the mean of the observations and, in general, the full predictive distribution.
Currently implemented only for normal predictive distributions, for which a closed-form expression exists.
For arbitrary distributions (given as samples), an expression suitable for direct implementation is given by Equ. 3 in
Eric P. Grimit, Tilmann Gneiting, Veronica J. Berrocal, Nicholas A. Johnson:
The continuous ranked probability score for circular variables and its application to mesoscale forecast ensemble verification,
Quarterly Journal of the Royal Meteorological Society 132(621C): 2925--2942, 2006. DOI 10.1256/qj.05.235
Parameters:
true: observed property distributions; requires only means
pred: predictive property distributions
Returns:
sequence of metric values
continuous ranked probability scores as a NumPy vector of floating point numbers
"""
true = params.distribution(true)
pred = params.normal_distribution(pred)
if np.any(pred.stddev == 0):
warn(
f"Some uncertainties are zero. Metric {self.__class__.__name__}"
"may return nan.",
RuntimeWarning,
)
strue = (true.mean -
pred.mean) / pred.stddev # re-used intermediate quantity
crps = pred.stddev * (strue * (2 * sp.stats.norm.cdf(strue) - 1) + 2 *
sp.stats.norm.pdf(strue) - 1 / np.sqrt(np.pi))
return crps
def _evaluate(self, true, pred):
r"""Logarithmic Predictive Density (LPD).
Assumes a normal predictive distribution.
\[
\log p (y_i = t_i | x_i)
= - ( \log \sqrt{2\pi} + \log \sigma_i + \frac{1}{2} ( \frac{y_i - t_i}{\sigma_i} )^2 )
\]
See, for example,
Joaquin Quinonero-Candela, Carl Edward Rasmussen, Fabian Sinz, Olivier Bousquet, and Bernhard Schölkopf.
Evaluating predictive uncertainty challenge, p. 1-27, 2005. In Joaquin Quinonero-Candela, Ido Dagan,
Bernardo Magnini, and Florence d'Alché Buc (editors), Proceedings of the First PASCAL Machine Learning
Challenges Workshop (MLCW 2005), Southampton, United Kingdom, April 11–13, 2005.
Parameters:
true: observed property distribution; requires only means
pred: predictive property distribution; must be normal
Returns:
logarithmic predictive densities as a NumPy vector of floating point numbers
"""
true = params.distribution(true)
pred = params.normal_distribution(pred)
if np.any(pred.stddev == 0):
warn(
f"Some uncertainties are zero. Metric {self.__class__.__name__}"
"may return nan.",
RuntimeWarning,
)
lpd = -(np.log(np.sqrt(2 * np.pi)) + np.log(pred.stddev) +
0.5 * np.square((true.mean - pred.mean) / pred.stddev))
return lpd
def _evaluate(self, true, pred):
"""Root mean squared error divided by standard deviation of labels.
stdRMSE = RMSE / std. dev.
See class docstring for details.
Parameters:
true: observed property distribution; requires only means
pred: predictive property distribution; requires only means
Returns:
standardized root mean squared error as a floating point number
"""
true = params.distribution(true)
# ensure sufficiently many samples
n = len(true.mean)
if n <= 1:
raise InvalidParameterError(
"enough samples to compute standard deviation", f"{n} samples")
# compute RMSE and standard deviation
rmse = super()._evaluate(true, pred)
stddev = np.std(true.mean, ddof=self._bias_correction)
# ensure sufficient variance in samples
if stddev <= 1e-3: # hard-coded, could be initialization parameter
raise InvalidParameterError(
"sufficient label variance for non-zero standard deviation",
f"standard deviation of {stddev}",
)
return float(rmse / stddev)
def _evaluate(self, true, pred):
"""Evaluate prediction error residuals.
residuals = predicted - observed
Parameters:
true: observed property distribution; requires only means
pred: predictive property distribution; requires only means
Returns:
residuals as NumPy array
"""
true = params.distribution(true).mean
pred = params.distribution(pred).mean
return pred - true
def _evaluate(self, true, pred):
"""Compute standard confidence
Parameters:
true: observed property distributions; requires only means
pred: predictive property distributions
Returns:
standard confidence
"""
true = params.distribution(true)
pred = params.normal_distribution(pred)
abs_residual = np.abs(true.mean - pred.mean)
is_less = abs_residual < pred.stddev
stdconf = np.mean(is_less)
return stdconf
def _evaluate(self, true, pred):
"""Compute Uncertainty Correlation
Parameters:
true: observed property distributions; requires only means
pred: predictive property distributions
Returns:
uncertainty correlation
"""
true = params.distribution(true)
pred = params.normal_distribution(pred)
abs_residual = np.abs(true.mean - pred.mean)
uc_corr = np.corrcoef(
abs_residual,
pred.stddev)[0, 1] # get off-diagonal of correlation matrix
return uc_corr
def _evaluate(self, true, pred):
"""Compute RMSSE.
Parameters:
true: observed property distributions; requires only means
pred: predictive property distributions
Returns:
RMSSE
"""
true = params.distribution(true)
pred = params.normal_distribution(pred)
if np.any(pred.stddev == 0):
warn(
f"Some uncertainties are zero. Metric {self.__class__.__name__}"
"will be nan.",
RuntimeWarning,
)
return np.nan
strue = (true.mean - pred.mean) / pred.stddev
rmsse = np.sqrt(np.mean(np.power(strue, 2)))
return rmsse