def test_extreme_values(self): S, y = get_extreme_binary_example() self.cal = FullDirichletCalibrator() self.cal.fit(S, y) predictions = self.cal.predict_proba(S).argmax(axis=1) acc = accuracy_score(y, predictions) self.assertGreater(acc, 0.99, "accuracy must be superior to 99 percent")
def test_fit_predict(self): for S, y in (get_simple_binary_example(), get_simple_ternary_example()): self.cal = FullDirichletCalibrator() self.cal.fit(S, y) predictions = self.cal.predict_proba(S).argmax(axis=1) acc = accuracy_score(y, predictions) self.assertGreater(acc, 0.98, "accuracy must be superior to 99 percent")
class TestFullDirichlet(unittest.TestCase): def setUp(self): self.cal = FullDirichletCalibrator() def test_fit_predict(self): S, y = get_simple_binary_example() self.cal = FullDirichletCalibrator() self.cal.fit(S, y) predictions = self.cal.predict_proba(S).argmax(axis=1) acc = accuracy_score(y, predictions) self.assertGreater(acc, 0.99, "accuracy must be superior to 99 percent") S, y = get_simple_ternary_example() self.cal = FullDirichletCalibrator() self.cal.fit(S, y) predictions = self.cal.predict_proba(S).argmax(axis=1) acc = accuracy_score(y, predictions) self.assertGreater(acc, 0.98, "accuracy must be superior to 99 percent") def test_extreme_values(self): S, y = get_extreme_binary_example() self.cal = FullDirichletCalibrator() self.cal.fit(S, y) predictions = self.cal.predict_proba(S).argmax(axis=1) acc = accuracy_score(y, predictions) self.assertGreater(acc, 0.99, "accuracy must be superior to 99 percent")
def test_lambda(self): S, y = get_simple_ternary_example() l2_odir = 1e-2 cal = FullDirichletCalibrator(reg_lambda=l2_odir, reg_mu=l2_odir, reg_norm=False) cal.fit(S, y) predictions = cal.predict_proba(S).argmax(axis=1) acc = accuracy_score(y, predictions) self.assertGreater(acc, 0.98, "accuracy must be superior to 99 percent") l2_odir = 1e2 cal = FullDirichletCalibrator(reg_lambda=l2_odir, reg_mu=l2_odir, reg_norm=False) cal.fit(S, y) predictions = cal.predict_proba(S).argmax(axis=1) acc = accuracy_score(y, predictions) self.assertLess(acc, 0.6, "accuracy must be smaller than 60 percent")
class TestFullDirichlet(unittest.TestCase): def setUp(self): self.cal = FullDirichletCalibrator() def test_fit_predict(self): for S, y in (get_simple_binary_example(), get_simple_ternary_example()): self.cal = FullDirichletCalibrator() self.cal.fit(S, y) predictions = self.cal.predict_proba(S).argmax(axis=1) acc = accuracy_score(y, predictions) self.assertGreater(acc, 0.98, "accuracy must be superior to 99 percent") def test_extreme_values(self): S, y = get_extreme_binary_example() self.cal = FullDirichletCalibrator() self.cal.fit(S, y) predictions = self.cal.predict_proba(S).argmax(axis=1) acc = accuracy_score(y, predictions) self.assertGreater(acc, 0.99, "accuracy must be superior to 99 percent") def test_lambda(self): S, y = get_simple_ternary_example() l2_odir = 1e-2 cal = FullDirichletCalibrator(reg_lambda=l2_odir, reg_mu=l2_odir, reg_norm=False) cal.fit(S, y) predictions = cal.predict_proba(S).argmax(axis=1) acc = accuracy_score(y, predictions) self.assertGreater(acc, 0.98, "accuracy must be superior to 99 percent") l2_odir = 1e2 cal = FullDirichletCalibrator(reg_lambda=l2_odir, reg_mu=l2_odir, reg_norm=False) cal.fit(S, y) predictions = cal.predict_proba(S).argmax(axis=1) acc = accuracy_score(y, predictions) self.assertLess(acc, 0.6, "accuracy must be smaller than 60 percent")
def setUp(self): self.cal = FullDirichletCalibrator()
x_train, x_test, y_train, y_test = train_test_split(x, y, random_state=1, test_size=0.3) skf = StratifiedKFold(n_splits=3, shuffle=True, random_state=0) classifier = GaussianNB() print('Training a classifier with cross-validation') scores = cross_val_score(classifier, x_train, y_train, cv=skf, scoring='neg_log_loss') print('Crossval scores: {}'.format(scores)) print('Average neg log loss {:.3f}'.format(scores.mean())) classifier.fit(x_train, y_train) cla_scores_train = classifier.predict_proba(x_train) reg = [1e-1, 1e-2, 1e-3, 1e-4, 1e-5] # Full Dirichlet calibrator = FullDirichletCalibrator(reg_lambda=reg, reg_mu=None) # ODIR Dirichlet #calibrator = FullDirichletCalibrator(reg_lambda=reg, reg_mu=reg) gscv = GridSearchCV(calibrator, param_grid={'reg_lambda': reg, 'reg_mu': [None]}, cv=skf, scoring='neg_log_loss') gscv.fit(cla_scores_train, y_train) print('Grid of parameters cross-validated') print(gscv.param_grid) print('Best parameters: {}'.format(gscv.best_params_)) cla_scores_test = classifier.predict_proba(x_test) cal_scores_test = gscv.predict_proba(cla_scores_test) cla_loss = log_loss(y_test, cla_scores_test) cal_loss = log_loss(y_test, cal_scores_test)