def testFallBackNBest(self): ensbuilder = EnsembleBuilder(backend=self.backend, dataset_name="TEST", task_type=1, #Binary Classification metric=roc_auc, limit=-1, # not used, seed=0, # important to find the test files ensemble_nbest=1 ) ensbuilder.read_ensemble_preds() filename = os.path.join( self.backend.temporary_directory, ".auto-sklearn/predictions_ensemble/predictions_ensemble_0_2.npy" ) ensbuilder.read_preds[filename]["ens_score"] = -1 filename = os.path.join( self.backend.temporary_directory, ".auto-sklearn/predictions_ensemble/predictions_ensemble_0_1.npy" ) ensbuilder.read_preds[filename]["ens_score"] = -1 sel_keys = ensbuilder.get_n_best_preds() fixture = os.path.join( self.backend.temporary_directory, ".auto-sklearn/predictions_ensemble/predictions_ensemble_0_1.npy" ) self.assertEquals(sel_keys[0], fixture)
def testGetValidTestPreds(self): ensbuilder = EnsembleBuilder(backend=self.backend, dataset_name="TEST", task_type=1, #Binary Classification metric=roc_auc, limit=-1, # not used, seed=0, # important to find the test files ensemble_nbest=1 ) ensbuilder.read_ensemble_preds() d2 = os.path.join( self.backend.temporary_directory, ".auto-sklearn/predictions_ensemble/predictions_ensemble_0_2.npy" ) d1 = os.path.join( self.backend.temporary_directory, ".auto-sklearn/predictions_ensemble/predictions_ensemble_0_1.npy" ) sel_keys = ensbuilder.get_n_best_preds() ensbuilder.get_valid_test_preds(selected_keys=sel_keys) # selected --> read valid and test predictions self.assertIsNotNone(ensbuilder.read_preds[d2][Y_VALID]) self.assertIsNotNone(ensbuilder.read_preds[d2][Y_TEST]) # not selected --> should still be None self.assertIsNone(ensbuilder.read_preds[d1][Y_VALID]) self.assertIsNone(ensbuilder.read_preds[d1][Y_TEST])
def testNBest(self): for ensemble_nbest, models_in_disc, exp in ( (1, None, 1), (1.0, None, 2), (0.1, None, 1), (0.9, None, 1), (1, 2, 1), (2, 1, 1), ): ensbuilder = EnsembleBuilder( backend=self.backend, dataset_name="TEST", task_type=1, # Binary Classification metric=roc_auc, limit=-1, # not used, seed=0, # important to find the test files ensemble_nbest=ensemble_nbest, max_models_on_disc=models_in_disc, ) ensbuilder.read_ensemble_preds() sel_keys = ensbuilder.get_n_best_preds() self.assertEqual(len(sel_keys), exp) fixture = os.path.join( self.backend.temporary_directory, ".auto-sklearn/predictions_ensemble/predictions_ensemble_0_2_0.0.npy" ) self.assertEqual(sel_keys[0], fixture)
def testEntireEnsembleBuilder(self): ensbuilder = EnsembleBuilder( backend=self.backend, dataset_name="TEST", task_type=1, #Binary Classification metric=roc_auc, limit=-1, # not used, seed=0, # important to find the test files ensemble_nbest=2, ) ensbuilder.SAVE2DISC = False ensbuilder.read_ensemble_preds() d2 = os.path.join( self.backend.temporary_directory, ".auto-sklearn/predictions_ensemble/predictions_ensemble_0_2.npy" ) sel_keys = ensbuilder.get_n_best_preds() self.assertGreater(len(sel_keys), 0) ensemble = ensbuilder.fit_ensemble(selected_keys=sel_keys) print(ensemble, sel_keys) n_sel_valid, n_sel_test = ensbuilder.get_valid_test_preds(selected_keys=sel_keys) # both valid and test prediction files are available self.assertGreater(len(n_sel_valid), 0) self.assertEqual(n_sel_valid, n_sel_test) y_valid = ensbuilder.predict( set_="valid", ensemble=ensemble, selected_keys=n_sel_valid, n_preds=len(sel_keys), index_run=1, ) y_test = ensbuilder.predict( set_="test", ensemble=ensemble, selected_keys=n_sel_test, n_preds=len(sel_keys), index_run=1, ) # predictions for valid and test are the same # --> should results in the same predictions np.testing.assert_array_almost_equal(y_valid, y_test) # since d2 provides perfect predictions # it should get a higher weight # so that y_valid should be exactly y_valid_d2 y_valid_d2 = ensbuilder.read_preds[d2][Y_VALID][:, 1] np.testing.assert_array_almost_equal(y_valid, y_valid_d2)
def testRead(self): ensbuilder = EnsembleBuilder( backend=self.backend, dataset_name="TEST", task_type=1, #Binary Classification metric=roc_auc, limit=-1, # not used, seed=0, # important to find the test files ) success = ensbuilder.read_ensemble_preds() self.assertTrue(success, str(ensbuilder.read_preds)) self.assertEqual(len(ensbuilder.read_preds), 2) filename = os.path.join( self.backend.temporary_directory, ".auto-sklearn/predictions_ensemble/predictions_ensemble_0_1.npy" ) self.assertEqual(ensbuilder.read_preds[filename]["ens_score"], 0.5) filename = os.path.join( self.backend.temporary_directory, ".auto-sklearn/predictions_ensemble/predictions_ensemble_0_2.npy" ) self.assertEqual(ensbuilder.read_preds[filename]["ens_score"], 1.0)