def testFallBackNBest(self):
ensbuilder = EnsembleBuilder(backend=self.backend,
dataset_name="TEST",
task_type=1, #Binary Classification
metric=roc_auc,
limit=-1, # not used,
seed=0, # important to find the test files
ensemble_nbest=1
)
ensbuilder.read_ensemble_preds()
filename = os.path.join(
self.backend.temporary_directory,
".auto-sklearn/predictions_ensemble/predictions_ensemble_0_2.npy"
)
ensbuilder.read_preds[filename]["ens_score"] = -1
filename = os.path.join(
self.backend.temporary_directory,
".auto-sklearn/predictions_ensemble/predictions_ensemble_0_1.npy"
)
ensbuilder.read_preds[filename]["ens_score"] = -1
sel_keys = ensbuilder.get_n_best_preds()
fixture = os.path.join(
self.backend.temporary_directory,
".auto-sklearn/predictions_ensemble/predictions_ensemble_0_1.npy"
)
self.assertEquals(sel_keys[0], fixture)
def testGetValidTestPreds(self):
ensbuilder = EnsembleBuilder(backend=self.backend,
dataset_name="TEST",
task_type=1, #Binary Classification
metric=roc_auc,
limit=-1, # not used,
seed=0, # important to find the test files
ensemble_nbest=1
)
ensbuilder.read_ensemble_preds()
d2 = os.path.join(
self.backend.temporary_directory,
".auto-sklearn/predictions_ensemble/predictions_ensemble_0_2.npy"
)
d1 = os.path.join(
self.backend.temporary_directory,
".auto-sklearn/predictions_ensemble/predictions_ensemble_0_1.npy"
)
sel_keys = ensbuilder.get_n_best_preds()
ensbuilder.get_valid_test_preds(selected_keys=sel_keys)
# selected --> read valid and test predictions
self.assertIsNotNone(ensbuilder.read_preds[d2][Y_VALID])
self.assertIsNotNone(ensbuilder.read_preds[d2][Y_TEST])
# not selected --> should still be None
self.assertIsNone(ensbuilder.read_preds[d1][Y_VALID])
self.assertIsNone(ensbuilder.read_preds[d1][Y_TEST])
def testNBest(self):
for ensemble_nbest, models_in_disc, exp in (
(1, None, 1),
(1.0, None, 2),
(0.1, None, 1),
(0.9, None, 1),
(1, 2, 1),
(2, 1, 1),
):
ensbuilder = EnsembleBuilder(
backend=self.backend,
dataset_name="TEST",
task_type=1, # Binary Classification
metric=roc_auc,
limit=-1, # not used,
seed=0, # important to find the test files
ensemble_nbest=ensemble_nbest,
max_models_on_disc=models_in_disc,
)
ensbuilder.read_ensemble_preds()
sel_keys = ensbuilder.get_n_best_preds()
self.assertEqual(len(sel_keys), exp)
fixture = os.path.join(
self.backend.temporary_directory,
".auto-sklearn/predictions_ensemble/predictions_ensemble_0_2_0.0.npy"
)
self.assertEqual(sel_keys[0], fixture)
def testGetValidTestPreds(self):
ensbuilder = EnsembleBuilder(backend=self.backend,
dataset_name="TEST",
task_type=1, #Binary Classification
metric=roc_auc,
limit=-1, # not used,
seed=0, # important to find the test files
ensemble_nbest=1
)
ensbuilder.read_ensemble_preds()
d2 = os.path.join(
self.backend.temporary_directory,
".auto-sklearn/predictions_ensemble/predictions_ensemble_0_2.npy"
)
d1 = os.path.join(
self.backend.temporary_directory,
".auto-sklearn/predictions_ensemble/predictions_ensemble_0_1.npy"
)
sel_keys = ensbuilder.get_n_best_preds()
ensbuilder.get_valid_test_preds(selected_keys=sel_keys)
# selected --> read valid and test predictions
self.assertIsNotNone(ensbuilder.read_preds[d2][Y_VALID])
self.assertIsNotNone(ensbuilder.read_preds[d2][Y_TEST])
# not selected --> should still be None
self.assertIsNone(ensbuilder.read_preds[d1][Y_VALID])
self.assertIsNone(ensbuilder.read_preds[d1][Y_TEST])
def testFallBackNBest(self):
ensbuilder = EnsembleBuilder(backend=self.backend,
dataset_name="TEST",
task_type=1, #Binary Classification
metric=roc_auc,
limit=-1, # not used,
seed=0, # important to find the test files
ensemble_nbest=1
)
ensbuilder.read_ensemble_preds()
filename = os.path.join(
self.backend.temporary_directory,
".auto-sklearn/predictions_ensemble/predictions_ensemble_0_2.npy"
)
ensbuilder.read_preds[filename]["ens_score"] = -1
filename = os.path.join(
self.backend.temporary_directory,
".auto-sklearn/predictions_ensemble/predictions_ensemble_0_1.npy"
)
ensbuilder.read_preds[filename]["ens_score"] = -1
sel_keys = ensbuilder.get_n_best_preds()
fixture = os.path.join(
self.backend.temporary_directory,
".auto-sklearn/predictions_ensemble/predictions_ensemble_0_1.npy"
)
self.assertEquals(sel_keys[0], fixture)
def testEntireEnsembleBuilder(self):
ensbuilder = EnsembleBuilder(
backend=self.backend,
dataset_name="TEST",
task_type=1, #Binary Classification
metric=roc_auc,
limit=-1, # not used,
seed=0, # important to find the test files
ensemble_nbest=2,
)
ensbuilder.SAVE2DISC = False
ensbuilder.read_ensemble_preds()
d2 = os.path.join(
self.backend.temporary_directory,
".auto-sklearn/predictions_ensemble/predictions_ensemble_0_2.npy"
)
sel_keys = ensbuilder.get_n_best_preds()
self.assertGreater(len(sel_keys), 0)
ensemble = ensbuilder.fit_ensemble(selected_keys=sel_keys)
print(ensemble, sel_keys)
n_sel_valid, n_sel_test = ensbuilder.get_valid_test_preds(selected_keys=sel_keys)
# both valid and test prediction files are available
self.assertGreater(len(n_sel_valid), 0)
self.assertEqual(n_sel_valid, n_sel_test)
y_valid = ensbuilder.predict(
set_="valid",
ensemble=ensemble,
selected_keys=n_sel_valid,
n_preds=len(sel_keys),
index_run=1,
)
y_test = ensbuilder.predict(
set_="test",
ensemble=ensemble,
selected_keys=n_sel_test,
n_preds=len(sel_keys),
index_run=1,
)
# predictions for valid and test are the same
# --> should results in the same predictions
np.testing.assert_array_almost_equal(y_valid, y_test)
# since d2 provides perfect predictions
# it should get a higher weight
# so that y_valid should be exactly y_valid_d2
y_valid_d2 = ensbuilder.read_preds[d2][Y_VALID][:, 1]
np.testing.assert_array_almost_equal(y_valid, y_valid_d2)
def testEntireEnsembleBuilder(self):
ensbuilder = EnsembleBuilder(
backend=self.backend,
dataset_name="TEST",
task_type=1, #Binary Classification
metric=roc_auc,
limit=-1, # not used,
seed=0, # important to find the test files
ensemble_nbest=2,
)
ensbuilder.SAVE2DISC = False
ensbuilder.read_ensemble_preds()
d2 = os.path.join(
self.backend.temporary_directory,
".auto-sklearn/predictions_ensemble/predictions_ensemble_0_2.npy"
)
sel_keys = ensbuilder.get_n_best_preds()
self.assertGreater(len(sel_keys), 0)
ensemble = ensbuilder.fit_ensemble(selected_keys=sel_keys)
print(ensemble, sel_keys)
n_sel_valid, n_sel_test = ensbuilder.get_valid_test_preds(selected_keys=sel_keys)
# both valid and test prediction files are available
self.assertGreater(len(n_sel_valid), 0)
self.assertEqual(n_sel_valid, n_sel_test)
y_valid = ensbuilder.predict(
set_="valid",
ensemble=ensemble,
selected_keys=n_sel_valid,
n_preds=len(sel_keys),
index_run=1,
)
y_test = ensbuilder.predict(
set_="test",
ensemble=ensemble,
selected_keys=n_sel_test,
n_preds=len(sel_keys),
index_run=1,
)
# predictions for valid and test are the same
# --> should results in the same predictions
np.testing.assert_array_almost_equal(y_valid, y_test)
# since d2 provides perfect predictions
# it should get a higher weight
# so that y_valid should be exactly y_valid_d2
y_valid_d2 = ensbuilder.read_preds[d2][Y_VALID][:, 1]
np.testing.assert_array_almost_equal(y_valid, y_valid_d2)
def testRead(self):
ensbuilder = EnsembleBuilder(
backend=self.backend,
dataset_name="TEST",
task_type=1, #Binary Classification
metric=roc_auc,
limit=-1, # not used,
seed=0, # important to find the test files
)
success = ensbuilder.read_ensemble_preds()
self.assertTrue(success, str(ensbuilder.read_preds))
self.assertEqual(len(ensbuilder.read_preds), 2)
filename = os.path.join(
self.backend.temporary_directory,
".auto-sklearn/predictions_ensemble/predictions_ensemble_0_1.npy"
)
self.assertEqual(ensbuilder.read_preds[filename]["ens_score"], 0.5)
filename = os.path.join(
self.backend.temporary_directory,
".auto-sklearn/predictions_ensemble/predictions_ensemble_0_2.npy"
)
self.assertEqual(ensbuilder.read_preds[filename]["ens_score"], 1.0)
def testRead(self):
ensbuilder = EnsembleBuilder(
backend=self.backend,
dataset_name="TEST",
task_type=1, #Binary Classification
metric=roc_auc,
limit=-1, # not used,
seed=0, # important to find the test files
)
success = ensbuilder.read_ensemble_preds()
self.assertTrue(success, str(ensbuilder.read_preds))
self.assertEqual(len(ensbuilder.read_preds), 2)
filename = os.path.join(
self.backend.temporary_directory,
".auto-sklearn/predictions_ensemble/predictions_ensemble_0_1.npy"
)
self.assertEqual(ensbuilder.read_preds[filename]["ens_score"], 0.5)
filename = os.path.join(
self.backend.temporary_directory,
".auto-sklearn/predictions_ensemble/predictions_ensemble_0_2.npy"
)
self.assertEqual(ensbuilder.read_preds[filename]["ens_score"], 1.0)