def test_extreme_cllr(self):
self.assertEqual(np.inf, metrics.cllr(*Xn_to_Xy([np.inf, 1], [1, 1])))
self.assertEqual(np.inf, metrics.cllr(*Xn_to_Xy([np.inf, 0], [1, 1])))
self.assertEqual(np.inf, metrics.cllr(*Xn_to_Xy([1, 1], [0, 1])))
self.assertAlmostEqual(.5, metrics.cllr(*Xn_to_Xy([0, 0], [1, 1])))
self.assertAlmostEqual(.5, metrics.cllr(*Xn_to_Xy([1, 1], [np.inf, np.inf])))
self.assertAlmostEqual(0, metrics.cllr(*Xn_to_Xy([0, 0], [np.inf, np.inf])))
self.assertAlmostEqual(np.inf, metrics.cllr(*Xn_to_Xy([np.inf, np.inf], [0, 0])))
self.assertEqual(np.inf, metrics.cllr(*Xn_to_Xy([1], [1.e-317]))) # value near zero for which 1/value causes an overflow
def test_lr_1(self):
score_class0 = np.arange(0, 1, .1)
score_class1 = np.arange(0, 1, .1)
X, y = Xn_to_Xy(score_class0, score_class1)
irc = IsotonicCalibrator(add_one=False)
lr0, lr1 = Xy_to_Xn(irc.fit_transform(X, y), y)
self.assertEqual(score_class0.shape, lr0.shape)
self.assertEqual(score_class1.shape, lr1.shape)
np.testing.assert_almost_equal(lr0, [1.] * lr0.shape[0])
np.testing.assert_almost_equal(lr1, [1.] * lr1.shape[0])
def run_cllrmin(self, lr0, lr1, places=7):
lr0 = np.array(lr0)
lr1 = np.array(lr1)
X, y = Xn_to_Xy(lr0, lr1)
cllr = _cllr(lr0, lr1)
irc = IsotonicCalibrator(add_one=False)
lrmin0, lrmin1 = Xy_to_Xn(irc.fit_transform(X / (X + 1), y), y)
cllrmin = _cllr(lrmin0, lrmin1)
self.assertAlmostEqual(cllr, cllrmin, places=places)
def test_lr_almost_1(self):
score_class0 = np.arange(0, 1, .1)
score_class1 = np.arange(.05, 1.05, .1)
X, y = Xn_to_Xy(score_class0, score_class1)
irc = IsotonicCalibrator(add_one=False)
lr0, lr1 = Xy_to_Xn(irc.fit_transform(X, y), y)
self.assertEqual(score_class0.shape, lr0.shape)
self.assertEqual(score_class1.shape, lr1.shape)
np.testing.assert_almost_equal(
lr0, np.concatenate([[0], [1.] * (lr0.shape[0] - 1)]))
np.testing.assert_almost_equal(
lr1, np.concatenate([[1.] * (lr1.shape[0] - 1), [np.inf]]))
def test_fraction_calibrator(self):
points_h0 = np.array([ 1, 2, 4, 8 ])
points_h1 = np.array([ 2, 6, 8, 9 ])
p0 = np.array([1., 1., .75, .5, .5 , .25, .25, .25, .25, 0., 0.])
p1 = np.array([0., 0., .25, .25, .25, .25, .5, .5, .75, 1, 1])
cal = FractionCalibrator(value_range=[0,10])
cal.fit(*Xn_to_Xy(points_h0, points_h1))
lr = cal.transform(np.arange(11))
np.testing.assert_almost_equal(cal.p0, p0)
np.testing.assert_almost_equal(cal.p1, p1)
with np.errstate(divide='ignore'):
np.testing.assert_almost_equal(lr, p1/p0)
def test_log_odds_version(self):
X, y = Xn_to_Xy(self.score_class0, self.score_class1)
X = to_probability(X)
desired = [
3.59562799e-02, 1.75942116e-11, 2.59633540e-12, 1.36799721e-12,
8.15673411e-03, 2.10030624e-02, 3.70456430e-05, 1.40710861e-18,
1.04459592e-10, 3.14589737e+03, 2.59568527e+02, 1.08519904e+02,
8.56459139e+01, 3.81243702e+00, 6.23873841e+01, 1.43844114e+02,
2.64913149e+02, 1.49097168e+05
]
calibrator = KDECalibrator()
calibrator.fit(X, y)
lrs_cal = calibrator.transform(X)
np.testing.assert_allclose(lrs_cal, desired)
def test_log_odds_version(self):
X, y = Xn_to_Xy(self.score_class0, self.score_class1)
X = to_probability(X)
desired = [
1.79732352e-01, 4.16251897e-04, 2.90464504e-04, 2.58097514e-04,
8.75801433e-02, 1.36880766e-01, 1.19709662e-02, 2.54277585e-05,
5.87902757e-04, 5.83462439e+02, 7.25669320e+01, 3.62580218e+01,
3.03795948e+01, 3.59619089e+00, 2.41271821e+01, 4.50261471e+01,
7.38162334e+01, 4.73670703e+03
]
calibrator = LogitCalibrator()
calibrator.fit(X, y)
lrs_cal = calibrator.transform(X)
np.testing.assert_allclose(lrs_cal, desired)
def test_log_odds_version(self):
X, y = Xn_to_Xy(self.score_class0, self.score_class1)
X = to_probability(X)
desired = [
3.06533372e-02, 5.92376598e-09, 1.96126088e-09, 1.35801711e-09,
6.71884519e-03, 1.74203890e-02, 6.47951643e-05, 6.33299969e-13,
1.67740156e-08, 2.38575352e+03, 3.62962152e+02, 1.65709775e+02,
1.33992382e+02, 6.90693424e+00, 1.00822553e+02, 2.13452386e+02,
3.69664971e+02, 7.74656845e+03
]
calibrator = GaussianCalibrator()
calibrator.fit(X, y)
lrs_cal = calibrator.transform(X)
np.testing.assert_allclose(lrs_cal, desired)
def test_illegal_cllr(self):
self.assertTrue(np.isnan(metrics.cllr(*Xn_to_Xy([np.nan, 1], [1, 1]))))
self.assertTrue(np.isnan(metrics.cllr(*Xn_to_Xy([1, 1], [1, np.nan]))))
self.assertTrue(np.isnan(metrics.cllr(*Xn_to_Xy([np.nan, np.nan], [np.nan, np.nan]))))
def test_calculate_cllr(self):
self.assertAlmostEqual(1, metrics.cllr(*Xn_to_Xy([1, 1], [1, 1])))
self.assertAlmostEqual(2, metrics.cllr(*Xn_to_Xy([3.]*2, [1/3.]*2)))
self.assertAlmostEqual(2, metrics.cllr(*Xn_to_Xy([3.]*20, [1/3.]*20)))
self.assertAlmostEqual(0.4150374992788437, metrics.cllr(*Xn_to_Xy([1/3.]*2, [3.]*2)))
self.assertAlmostEqual(0.7075187496394219, metrics.cllr(*Xn_to_Xy([1/3.]*2, [1])))
self.assertAlmostEqual(0.507177646488535, metrics.cllr(*Xn_to_Xy([1/100.]*100, [1])))
self.assertAlmostEqual(0.5400680236656377, metrics.cllr(*Xn_to_Xy([1/100.]*100 + [100], [1])))
self.assertAlmostEqual(0.5723134914863265, metrics.cllr(*Xn_to_Xy([1/100.]*100 + [100]*2, [1])))
self.assertAlmostEqual(0.6952113122368764, metrics.cllr(*Xn_to_Xy([1/100.]*100 + [100]*6, [1])))
self.assertAlmostEqual(1.0000000000000000, metrics.cllr(*Xn_to_Xy([1], [1])))
self.assertAlmostEqual(1.0849625007211563, metrics.cllr(*Xn_to_Xy([2], [2]*2)))
self.assertAlmostEqual(1.6699250014423126, metrics.cllr(*Xn_to_Xy([8], [8]*8)))
