def testGetValidTestPreds(self): ensbuilder = EnsembleBuilder(backend=self.backend, dataset_name="TEST", task_type=1, #Binary Classification metric=roc_auc, limit=-1, # not used, seed=0, # important to find the test files ensemble_nbest=1 ) ensbuilder.read_ensemble_preds() d2 = os.path.join( self.backend.temporary_directory, ".auto-sklearn/predictions_ensemble/predictions_ensemble_0_2.npy" ) d1 = os.path.join( self.backend.temporary_directory, ".auto-sklearn/predictions_ensemble/predictions_ensemble_0_1.npy" ) sel_keys = ensbuilder.get_n_best_preds() ensbuilder.get_valid_test_preds(selected_keys=sel_keys) # selected --> read valid and test predictions self.assertIsNotNone(ensbuilder.read_preds[d2][Y_VALID]) self.assertIsNotNone(ensbuilder.read_preds[d2][Y_TEST]) # not selected --> should still be None self.assertIsNone(ensbuilder.read_preds[d1][Y_VALID]) self.assertIsNone(ensbuilder.read_preds[d1][Y_TEST])
def testGetValidTestPreds(self): ensbuilder = EnsembleBuilder( backend=self.backend, dataset_name="TEST", task_type=1, # Binary Classification metric=roc_auc, limit=-1, # not used, seed=0, # important to find the test files ensemble_nbest=1) ensbuilder.score_ensemble_preds() d1 = os.path.join( self.backend.temporary_directory, ".auto-sklearn/predictions_ensemble/predictions_ensemble_0_1_0.0.npy" ) d2 = os.path.join( self.backend.temporary_directory, ".auto-sklearn/predictions_ensemble/predictions_ensemble_0_2_0.0.npy" ) d3 = os.path.join( self.backend.temporary_directory, ".auto-sklearn/predictions_ensemble/predictions_ensemble_0_3_100.0.npy" ) sel_keys = ensbuilder.get_n_best_preds() self.assertEqual(len(sel_keys), 1) ensbuilder.get_valid_test_preds(selected_keys=sel_keys) # Number of read files should be three and # predictions_ensemble_0_4_0.0.npy must not be in there self.assertEqual(len(ensbuilder.read_preds), 3) self.assertNotIn( os.path.join( self.backend.temporary_directory, ".auto-sklearn/predictions_ensemble/predictions_ensemble_0_4_0.0.npy" ), ensbuilder.read_preds) # not selected --> should still be None self.assertIsNone(ensbuilder.read_preds[d1][Y_VALID]) self.assertIsNone(ensbuilder.read_preds[d1][Y_TEST]) self.assertIsNone(ensbuilder.read_preds[d3][Y_VALID]) self.assertIsNone(ensbuilder.read_preds[d3][Y_TEST]) # selected --> read valid and test predictions self.assertIsNotNone(ensbuilder.read_preds[d2][Y_VALID]) self.assertIsNotNone(ensbuilder.read_preds[d2][Y_TEST])
def testGetValidTestPreds(ensemble_backend): ensbuilder = EnsembleBuilder( backend=ensemble_backend, dataset_name="TEST", task_type=BINARY_CLASSIFICATION, metric=roc_auc, seed=0, # important to find the test files ensemble_nbest=1) ensbuilder.compute_loss_per_model() # d1 is a dummt prediction. d2 and d3 have the same prediction with # different name. num_run=2 is selected when doing sorted() d1 = os.path.join( ensemble_backend.temporary_directory, ".auto-sklearn/runs/0_1_0.0/predictions_ensemble_0_1_0.0.npy") d2 = os.path.join( ensemble_backend.temporary_directory, ".auto-sklearn/runs/0_2_0.0/predictions_ensemble_0_2_0.0.npy") d3 = os.path.join( ensemble_backend.temporary_directory, ".auto-sklearn/runs/0_3_100.0/predictions_ensemble_0_3_100.0.npy") sel_keys = ensbuilder.get_n_best_preds() assert len(sel_keys) == 1 ensbuilder.get_valid_test_preds(selected_keys=sel_keys) # Number of read files should be three and # predictions_ensemble_0_4_0.0.npy must not be in there assert len(ensbuilder.read_preds) == 3 assert os.path.join( ensemble_backend.temporary_directory, ".auto-sklearn/runs/0_4_0.0/predictions_ensemble_0_4_0.0.npy" ) not in ensbuilder.read_preds # not selected --> should still be None assert ensbuilder.read_preds[d1][Y_VALID] is None assert ensbuilder.read_preds[d1][Y_TEST] is None assert ensbuilder.read_preds[d3][Y_VALID] is None assert ensbuilder.read_preds[d3][Y_TEST] is None # selected --> read valid and test predictions assert ensbuilder.read_preds[d2][Y_VALID] is not None assert ensbuilder.read_preds[d2][Y_TEST] is not None
def testEntireEnsembleBuilder(self): ensbuilder = EnsembleBuilder( backend=self.backend, dataset_name="TEST", task_type=1, # Binary Classification metric=roc_auc, limit=-1, # not used, seed=0, # important to find the test files ensemble_nbest=2, ) ensbuilder.SAVE2DISC = False ensbuilder.score_ensemble_preds() d2 = os.path.join( self.backend.temporary_directory, ".auto-sklearn/predictions_ensemble/predictions_ensemble_0_2_0.0.npy" ) sel_keys = ensbuilder.get_n_best_preds() self.assertGreater(len(sel_keys), 0) ensemble = ensbuilder.fit_ensemble(selected_keys=sel_keys) print(ensemble, sel_keys) n_sel_valid, n_sel_test = ensbuilder.get_valid_test_preds( selected_keys=sel_keys) # both valid and test prediction files are available self.assertGreater(len(n_sel_valid), 0) self.assertEqual(n_sel_valid, n_sel_test) y_valid = ensbuilder.predict( set_="valid", ensemble=ensemble, selected_keys=n_sel_valid, n_preds=len(sel_keys), index_run=1, ) y_test = ensbuilder.predict( set_="test", ensemble=ensemble, selected_keys=n_sel_test, n_preds=len(sel_keys), index_run=1, ) # predictions for valid and test are the same # --> should results in the same predictions np.testing.assert_array_almost_equal(y_valid, y_test) # since d2 provides perfect predictions # it should get a higher weight # so that y_valid should be exactly y_valid_d2 y_valid_d2 = ensbuilder.read_preds[d2][Y_VALID][:, 1] np.testing.assert_array_almost_equal(y_valid, y_valid_d2)
def testEntireEnsembleBuilder(self): ensbuilder = EnsembleBuilder( backend=self.backend, dataset_name="TEST", task_type=1, #Binary Classification metric=roc_auc, limit=-1, # not used, seed=0, # important to find the test files ensemble_nbest=2, ) ensbuilder.SAVE2DISC = False ensbuilder.read_ensemble_preds() d2 = os.path.join( self.backend.temporary_directory, ".auto-sklearn/predictions_ensemble/predictions_ensemble_0_2.npy" ) sel_keys = ensbuilder.get_n_best_preds() self.assertGreater(len(sel_keys), 0) ensemble = ensbuilder.fit_ensemble(selected_keys=sel_keys) print(ensemble, sel_keys) n_sel_valid, n_sel_test = ensbuilder.get_valid_test_preds(selected_keys=sel_keys) # both valid and test prediction files are available self.assertGreater(len(n_sel_valid), 0) self.assertEqual(n_sel_valid, n_sel_test) y_valid = ensbuilder.predict( set_="valid", ensemble=ensemble, selected_keys=n_sel_valid, n_preds=len(sel_keys), index_run=1, ) y_test = ensbuilder.predict( set_="test", ensemble=ensemble, selected_keys=n_sel_test, n_preds=len(sel_keys), index_run=1, ) # predictions for valid and test are the same # --> should results in the same predictions np.testing.assert_array_almost_equal(y_valid, y_test) # since d2 provides perfect predictions # it should get a higher weight # so that y_valid should be exactly y_valid_d2 y_valid_d2 = ensbuilder.read_preds[d2][Y_VALID][:, 1] np.testing.assert_array_almost_equal(y_valid, y_valid_d2)
def testEntireEnsembleBuilder(ensemble_backend): ensbuilder = EnsembleBuilder( backend=ensemble_backend, dataset_name="TEST", task_type=BINARY_CLASSIFICATION, metric=roc_auc, seed=0, # important to find the test files ensemble_nbest=2, ) ensbuilder.SAVE2DISC = False ensbuilder.compute_loss_per_model() d2 = os.path.join( ensemble_backend.temporary_directory, ".auto-sklearn/runs/0_2_0.0/predictions_ensemble_0_2_0.0.npy") sel_keys = ensbuilder.get_n_best_preds() assert len(sel_keys) > 0 ensemble = ensbuilder.fit_ensemble(selected_keys=sel_keys) print(ensemble, sel_keys) n_sel_valid, n_sel_test = ensbuilder.get_valid_test_preds( selected_keys=sel_keys) # both valid and test prediction files are available assert len(n_sel_valid) > 0 assert n_sel_valid == n_sel_test y_valid = ensbuilder.predict( set_="valid", ensemble=ensemble, selected_keys=n_sel_valid, n_preds=len(sel_keys), index_run=1, ) y_test = ensbuilder.predict( set_="test", ensemble=ensemble, selected_keys=n_sel_test, n_preds=len(sel_keys), index_run=1, ) # predictions for valid and test are the same # --> should results in the same predictions np.testing.assert_array_almost_equal(y_valid, y_test) # since d2 provides perfect predictions # it should get a higher weight # so that y_valid should be exactly y_valid_d2 y_valid_d2 = ensbuilder.read_preds[d2][Y_VALID][:, 1] np.testing.assert_array_almost_equal(y_valid, y_valid_d2)