def test_regression_metrics(n_samples=50):
y_true = np.arange(n_samples)
y_pred = y_true + 1
y_pred_2 = y_true - 1
assert_almost_equal(mean_squared_error(y_true, y_pred), 1.0)
assert_almost_equal(
mean_squared_log_error(y_true, y_pred),
mean_squared_error(np.log(1 + y_true), np.log(1 + y_pred)),
)
assert_almost_equal(mean_absolute_error(y_true, y_pred), 1.0)
assert_almost_equal(mean_pinball_loss(y_true, y_pred), 0.5)
assert_almost_equal(mean_pinball_loss(y_true, y_pred_2), 0.5)
assert_almost_equal(mean_pinball_loss(y_true, y_pred, alpha=0.4), 0.6)
assert_almost_equal(mean_pinball_loss(y_true, y_pred_2, alpha=0.4), 0.4)
assert_almost_equal(median_absolute_error(y_true, y_pred), 1.0)
mape = mean_absolute_percentage_error(y_true, y_pred)
assert np.isfinite(mape)
assert mape > 1e6
assert_almost_equal(max_error(y_true, y_pred), 1.0)
assert_almost_equal(r2_score(y_true, y_pred), 0.995, 2)
assert_almost_equal(explained_variance_score(y_true, y_pred), 1.0)
assert_almost_equal(
mean_tweedie_deviance(y_true, y_pred, power=0),
mean_squared_error(y_true, y_pred),
)
assert_almost_equal(d2_tweedie_score(y_true, y_pred, power=0),
r2_score(y_true, y_pred))
# Tweedie deviance needs positive y_pred, except for p=0,
# p>=2 needs positive y_true
# results evaluated by sympy
y_true = np.arange(1, 1 + n_samples)
y_pred = 2 * y_true
n = n_samples
assert_almost_equal(
mean_tweedie_deviance(y_true, y_pred, power=-1),
5 / 12 * n * (n**2 + 2 * n + 1),
)
assert_almost_equal(mean_tweedie_deviance(y_true, y_pred, power=1),
(n + 1) * (1 - np.log(2)))
assert_almost_equal(mean_tweedie_deviance(y_true, y_pred, power=2),
2 * np.log(2) - 1)
assert_almost_equal(
mean_tweedie_deviance(y_true, y_pred, power=3 / 2),
((6 * np.sqrt(2) - 8) / n) * np.sqrt(y_true).sum(),
)
assert_almost_equal(mean_tweedie_deviance(y_true, y_pred, power=3),
np.sum(1 / y_true) / (4 * n))
dev_mean = 2 * np.mean(xlogy(y_true, 2 * y_true / (n + 1)))
assert_almost_equal(
d2_tweedie_score(y_true, y_pred, power=1),
1 - (n + 1) * (1 - np.log(2)) / dev_mean,
)
dev_mean = 2 * np.log((n + 1) / 2) - 2 / n * np.log(factorial(n))
assert_almost_equal(d2_tweedie_score(y_true, y_pred, power=2),
1 - (2 * np.log(2) - 1) / dev_mean)
def test_tweedie_score(regression_data, power, link):
"""Test that GLM score equals d2_tweedie_score for Tweedie losses."""
X, y = regression_data
# make y positive
y = np.abs(y) + 1.0
glm = TweedieRegressor(power=power, link=link).fit(X, y)
assert glm.score(X, y) == pytest.approx(
d2_tweedie_score(y, glm.predict(X), power=power))
def test_regression_metrics_at_limits():
# Single-sample case
# Note: for r2 and d2_tweedie see also test_regression_single_sample
assert_almost_equal(mean_squared_error([0.0], [0.0]), 0.0)
assert_almost_equal(mean_squared_error([0.0], [0.0], squared=False), 0.0)
assert_almost_equal(mean_squared_log_error([0.0], [0.0]), 0.0)
assert_almost_equal(mean_absolute_error([0.0], [0.0]), 0.0)
assert_almost_equal(mean_pinball_loss([0.0], [0.0]), 0.0)
assert_almost_equal(mean_absolute_percentage_error([0.0], [0.0]), 0.0)
assert_almost_equal(median_absolute_error([0.0], [0.0]), 0.0)
assert_almost_equal(max_error([0.0], [0.0]), 0.0)
assert_almost_equal(explained_variance_score([0.0], [0.0]), 1.0)
# Perfect cases
assert_almost_equal(r2_score([0.0, 1], [0.0, 1]), 1.0)
assert_almost_equal(d2_pinball_score([0.0, 1], [0.0, 1]), 1.0)
# Non-finite cases
# R² and explained variance have a fix by default for non-finite cases
for s in (r2_score, explained_variance_score):
assert_almost_equal(s([0, 0], [1, -1]), 0.0)
assert_almost_equal(s([0, 0], [1, -1], force_finite=False), -np.inf)
assert_almost_equal(s([1, 1], [1, 1]), 1.0)
assert_almost_equal(s([1, 1], [1, 1], force_finite=False), np.nan)
msg = ("Mean Squared Logarithmic Error cannot be used when targets "
"contain negative values.")
with pytest.raises(ValueError, match=msg):
mean_squared_log_error([-1.0], [-1.0])
msg = ("Mean Squared Logarithmic Error cannot be used when targets "
"contain negative values.")
with pytest.raises(ValueError, match=msg):
mean_squared_log_error([1.0, 2.0, 3.0], [1.0, -2.0, 3.0])
msg = ("Mean Squared Logarithmic Error cannot be used when targets "
"contain negative values.")
with pytest.raises(ValueError, match=msg):
mean_squared_log_error([1.0, -2.0, 3.0], [1.0, 2.0, 3.0])
# Tweedie deviance error
power = -1.2
assert_allclose(mean_tweedie_deviance([0], [1.0], power=power),
2 / (2 - power),
rtol=1e-3)
msg = "can only be used on strictly positive y_pred."
with pytest.raises(ValueError, match=msg):
mean_tweedie_deviance([0.0], [0.0], power=power)
with pytest.raises(ValueError, match=msg):
d2_tweedie_score([0.0] * 2, [0.0] * 2, power=power)
assert_almost_equal(mean_tweedie_deviance([0.0], [0.0], power=0), 0.0, 2)
power = 1.0
msg = "only be used on non-negative y and strictly positive y_pred."
with pytest.raises(ValueError, match=msg):
mean_tweedie_deviance([0.0], [0.0], power=power)
with pytest.raises(ValueError, match=msg):
d2_tweedie_score([0.0] * 2, [0.0] * 2, power=power)
power = 1.5
assert_allclose(mean_tweedie_deviance([0.0], [1.0], power=power),
2 / (2 - power))
msg = "only be used on non-negative y and strictly positive y_pred."
with pytest.raises(ValueError, match=msg):
mean_tweedie_deviance([0.0], [0.0], power=power)
with pytest.raises(ValueError, match=msg):
d2_tweedie_score([0.0] * 2, [0.0] * 2, power=power)
power = 2.0
assert_allclose(mean_tweedie_deviance([1.0], [1.0], power=power),
0.00,
atol=1e-8)
msg = "can only be used on strictly positive y and y_pred."
with pytest.raises(ValueError, match=msg):
mean_tweedie_deviance([0.0], [0.0], power=power)
with pytest.raises(ValueError, match=msg):
d2_tweedie_score([0.0] * 2, [0.0] * 2, power=power)
power = 3.0
assert_allclose(mean_tweedie_deviance([1.0], [1.0], power=power),
0.00,
atol=1e-8)
msg = "can only be used on strictly positive y and y_pred."
with pytest.raises(ValueError, match=msg):
mean_tweedie_deviance([0.0], [0.0], power=power)
with pytest.raises(ValueError, match=msg):
d2_tweedie_score([0.0] * 2, [0.0] * 2, power=power)
power = 0.5
with pytest.raises(ValueError,
match="is only defined for power<=0 and power>=1"):
mean_tweedie_deviance([0.0], [0.0], power=power)
with pytest.raises(ValueError,
match="is only defined for power<=0 and power>=1"):
d2_tweedie_score([0.0] * 2, [0.0] * 2, power=power)
def test_regression_metrics_at_limits():
assert_almost_equal(mean_squared_error([0.0], [0.0]), 0.0)
assert_almost_equal(mean_squared_error([0.0], [0.0], squared=False), 0.0)
assert_almost_equal(mean_squared_log_error([0.0], [0.0]), 0.0)
assert_almost_equal(mean_absolute_error([0.0], [0.0]), 0.0)
assert_almost_equal(mean_pinball_loss([0.0], [0.0]), 0.0)
assert_almost_equal(mean_absolute_percentage_error([0.0], [0.0]), 0.0)
assert_almost_equal(median_absolute_error([0.0], [0.0]), 0.0)
assert_almost_equal(max_error([0.0], [0.0]), 0.0)
assert_almost_equal(explained_variance_score([0.0], [0.0]), 1.0)
assert_almost_equal(r2_score([0.0, 1], [0.0, 1]), 1.0)
msg = ("Mean Squared Logarithmic Error cannot be used when targets "
"contain negative values.")
with pytest.raises(ValueError, match=msg):
mean_squared_log_error([-1.0], [-1.0])
msg = ("Mean Squared Logarithmic Error cannot be used when targets "
"contain negative values.")
with pytest.raises(ValueError, match=msg):
mean_squared_log_error([1.0, 2.0, 3.0], [1.0, -2.0, 3.0])
msg = ("Mean Squared Logarithmic Error cannot be used when targets "
"contain negative values.")
with pytest.raises(ValueError, match=msg):
mean_squared_log_error([1.0, -2.0, 3.0], [1.0, 2.0, 3.0])
# Tweedie deviance error
power = -1.2
assert_allclose(mean_tweedie_deviance([0], [1.0], power=power),
2 / (2 - power),
rtol=1e-3)
msg = "can only be used on strictly positive y_pred."
with pytest.raises(ValueError, match=msg):
mean_tweedie_deviance([0.0], [0.0], power=power)
with pytest.raises(ValueError, match=msg):
d2_tweedie_score([0.0] * 2, [0.0] * 2, power=power)
assert_almost_equal(mean_tweedie_deviance([0.0], [0.0], power=0), 0.0, 2)
power = 1.0
msg = "only be used on non-negative y and strictly positive y_pred."
with pytest.raises(ValueError, match=msg):
mean_tweedie_deviance([0.0], [0.0], power=power)
with pytest.raises(ValueError, match=msg):
d2_tweedie_score([0.0] * 2, [0.0] * 2, power=power)
power = 1.5
assert_allclose(mean_tweedie_deviance([0.0], [1.0], power=power),
2 / (2 - power))
msg = "only be used on non-negative y and strictly positive y_pred."
with pytest.raises(ValueError, match=msg):
mean_tweedie_deviance([0.0], [0.0], power=power)
with pytest.raises(ValueError, match=msg):
d2_tweedie_score([0.0] * 2, [0.0] * 2, power=power)
power = 2.0
assert_allclose(mean_tweedie_deviance([1.0], [1.0], power=power),
0.00,
atol=1e-8)
msg = "can only be used on strictly positive y and y_pred."
with pytest.raises(ValueError, match=msg):
mean_tweedie_deviance([0.0], [0.0], power=power)
with pytest.raises(ValueError, match=msg):
d2_tweedie_score([0.0] * 2, [0.0] * 2, power=power)
power = 3.0
assert_allclose(mean_tweedie_deviance([1.0], [1.0], power=power),
0.00,
atol=1e-8)
msg = "can only be used on strictly positive y and y_pred."
with pytest.raises(ValueError, match=msg):
mean_tweedie_deviance([0.0], [0.0], power=power)
with pytest.raises(ValueError, match=msg):
d2_tweedie_score([0.0] * 2, [0.0] * 2, power=power)
power = 0.5
with pytest.raises(ValueError,
match="is only defined for power<=0 and power>=1"):
mean_tweedie_deviance([0.0], [0.0], power=power)
with pytest.raises(ValueError,
match="is only defined for power<=0 and power>=1"):
d2_tweedie_score([0.0] * 2, [0.0] * 2, power=power)
