def test_extreme_values(self):
S, y = get_extreme_binary_example()
self.cal = FullDirichletCalibrator()
self.cal.fit(S, y)
predictions = self.cal.predict_proba(S).argmax(axis=1)
acc = accuracy_score(y, predictions)
self.assertGreater(acc, 0.99, "accuracy must be superior to 99 percent")
def test_fit_predict(self):
for S, y in (get_simple_binary_example(),
get_simple_ternary_example()):
self.cal = FullDirichletCalibrator()
self.cal.fit(S, y)
predictions = self.cal.predict_proba(S).argmax(axis=1)
acc = accuracy_score(y, predictions)
self.assertGreater(acc, 0.98,
"accuracy must be superior to 99 percent")
class TestFullDirichlet(unittest.TestCase):
def setUp(self):
self.cal = FullDirichletCalibrator()
def test_fit_predict(self):
S, y = get_simple_binary_example()
self.cal = FullDirichletCalibrator()
self.cal.fit(S, y)
predictions = self.cal.predict_proba(S).argmax(axis=1)
acc = accuracy_score(y, predictions)
self.assertGreater(acc, 0.99, "accuracy must be superior to 99 percent")
S, y = get_simple_ternary_example()
self.cal = FullDirichletCalibrator()
self.cal.fit(S, y)
predictions = self.cal.predict_proba(S).argmax(axis=1)
acc = accuracy_score(y, predictions)
self.assertGreater(acc, 0.98, "accuracy must be superior to 99 percent")
def test_extreme_values(self):
S, y = get_extreme_binary_example()
self.cal = FullDirichletCalibrator()
self.cal.fit(S, y)
predictions = self.cal.predict_proba(S).argmax(axis=1)
acc = accuracy_score(y, predictions)
self.assertGreater(acc, 0.99, "accuracy must be superior to 99 percent")
def test_lambda(self):
S, y = get_simple_ternary_example()
l2_odir = 1e-2
cal = FullDirichletCalibrator(reg_lambda=l2_odir,
reg_mu=l2_odir,
reg_norm=False)
cal.fit(S, y)
predictions = cal.predict_proba(S).argmax(axis=1)
acc = accuracy_score(y, predictions)
self.assertGreater(acc, 0.98,
"accuracy must be superior to 99 percent")
l2_odir = 1e2
cal = FullDirichletCalibrator(reg_lambda=l2_odir,
reg_mu=l2_odir,
reg_norm=False)
cal.fit(S, y)
predictions = cal.predict_proba(S).argmax(axis=1)
acc = accuracy_score(y, predictions)
self.assertLess(acc, 0.6, "accuracy must be smaller than 60 percent")
class TestFullDirichlet(unittest.TestCase):
def setUp(self):
self.cal = FullDirichletCalibrator()
def test_fit_predict(self):
for S, y in (get_simple_binary_example(),
get_simple_ternary_example()):
self.cal = FullDirichletCalibrator()
self.cal.fit(S, y)
predictions = self.cal.predict_proba(S).argmax(axis=1)
acc = accuracy_score(y, predictions)
self.assertGreater(acc, 0.98,
"accuracy must be superior to 99 percent")
def test_extreme_values(self):
S, y = get_extreme_binary_example()
self.cal = FullDirichletCalibrator()
self.cal.fit(S, y)
predictions = self.cal.predict_proba(S).argmax(axis=1)
acc = accuracy_score(y, predictions)
self.assertGreater(acc, 0.99,
"accuracy must be superior to 99 percent")
def test_lambda(self):
S, y = get_simple_ternary_example()
l2_odir = 1e-2
cal = FullDirichletCalibrator(reg_lambda=l2_odir,
reg_mu=l2_odir,
reg_norm=False)
cal.fit(S, y)
predictions = cal.predict_proba(S).argmax(axis=1)
acc = accuracy_score(y, predictions)
self.assertGreater(acc, 0.98,
"accuracy must be superior to 99 percent")
l2_odir = 1e2
cal = FullDirichletCalibrator(reg_lambda=l2_odir,
reg_mu=l2_odir,
reg_norm=False)
cal.fit(S, y)
predictions = cal.predict_proba(S).argmax(axis=1)
acc = accuracy_score(y, predictions)
self.assertLess(acc, 0.6, "accuracy must be smaller than 60 percent")
def setUp(self):
self.cal = FullDirichletCalibrator()
x_train, x_test, y_train, y_test = train_test_split(x, y, random_state=1,
test_size=0.3)
skf = StratifiedKFold(n_splits=3, shuffle=True, random_state=0)
classifier = GaussianNB()
print('Training a classifier with cross-validation')
scores = cross_val_score(classifier, x_train, y_train, cv=skf,
scoring='neg_log_loss')
print('Crossval scores: {}'.format(scores))
print('Average neg log loss {:.3f}'.format(scores.mean()))
classifier.fit(x_train, y_train)
cla_scores_train = classifier.predict_proba(x_train)
reg = [1e-1, 1e-2, 1e-3, 1e-4, 1e-5]
# Full Dirichlet
calibrator = FullDirichletCalibrator(reg_lambda=reg, reg_mu=None)
# ODIR Dirichlet
#calibrator = FullDirichletCalibrator(reg_lambda=reg, reg_mu=reg)
gscv = GridSearchCV(calibrator, param_grid={'reg_lambda': reg,
'reg_mu': [None]},
cv=skf, scoring='neg_log_loss')
gscv.fit(cla_scores_train, y_train)
print('Grid of parameters cross-validated')
print(gscv.param_grid)
print('Best parameters: {}'.format(gscv.best_params_))
cla_scores_test = classifier.predict_proba(x_test)
cal_scores_test = gscv.predict_proba(cla_scores_test)
cla_loss = log_loss(y_test, cla_scores_test)
cal_loss = log_loss(y_test, cal_scores_test)