def test_best_config_metric(self): my_pipe_optimizer = Hyperpipe.Optimization('grid_search', {}, [], 'balanced_accuracy', None) self.assertTrue(my_pipe_optimizer.maximize_metric) my_pipe_optimizer = Hyperpipe.Optimization('grid_search', {}, [], 'mean_squared_error', None) self.assertFalse(my_pipe_optimizer.maximize_metric)
def setUp(self): super(InnerFoldTests, self).setUp() self.pipe = PhotonPipeline([ ("StandardScaler", PipelineElement("StandardScaler")), ("PCA", PipelineElement("PCA")), ("RidgeClassifier", PipelineElement("RidgeClassifier")), ]) self.config = { "PCA__n_components": 5, "RidgeClassifier__solver": "svd", "RidgeClassifier__random_state": 42, } self.outer_fold_id = "TestID" self.inner_cv = KFold(n_splits=4) self.X, self.y = load_breast_cancer(True) self.cross_validation = Hyperpipe.CrossValidation( self.inner_cv, None, True, 0.2, True, False) self.cross_validation.inner_folds = { self.outer_fold_id: { i: FoldInfo(i, i + 1, train, test) for i, (train, test) in enumerate(self.inner_cv.split(self.X, self.y)) } } self.optimization = Hyperpipe.Optimization( "grid_search", {}, ["accuracy", "recall", "specificity"], "accuracy", None)
def setUp(self): super(InnerFoldTests, self).setUp() self.pipe = PhotonPipeline([ ('StandardScaler', PipelineElement('StandardScaler')), ('PCA', PipelineElement('PCA')), ('RidgeClassifier', PipelineElement('RidgeClassifier')) ]) self.config = { 'PCA__n_components': 5, 'RidgeClassifier__solver': 'svd', 'RidgeClassifier__random_state': 42 } self.outer_fold_id = 'TestID' self.inner_cv = KFold(n_splits=4) self.X, self.y = load_breast_cancer(return_X_y=True) self.cross_validation = Hyperpipe.CrossValidation( self.inner_cv, None, True, 0.2, True, False, False, None) self.cross_validation.inner_folds = { self.outer_fold_id: { i: FoldInfo(i, i + 1, train, test) for i, (train, test) in enumerate(self.inner_cv.split(self.X, self.y)) } } self.optimization = Hyperpipe.Optimization( 'grid_search', {}, ['accuracy', 'recall', 'specificity'], 'accuracy', None)
def test_get_optimum_config_outer_folds(self): my_pipe_optimizer = Hyperpipe.Optimization( "grid_search", {}, [], "balanced_accuracy", None ) outer_fold_list = list() for i in range(10): outer_fold = MDBOuterFold() outer_fold.best_config = MDBConfig() outer_fold.best_config.best_config_score = MDBInnerFold() outer_fold.best_config.best_config_score.validation = MDBScoreInformation() # again fold 5 wins if i == 5: outer_fold.best_config.best_config_score.validation.metrics = { "balanced_accuracy": 0.99 } else: outer_fold.best_config.best_config_score.validation.metrics = { "balanced_accuracy": 0.5 } outer_fold_list.append(outer_fold) best_config_outer_folds = my_pipe_optimizer.get_optimum_config_outer_folds( outer_fold_list ) self.assertEqual( best_config_outer_folds.best_config_score.validation.metrics[ "balanced_accuracy" ], 0.99, ) self.assertIs(best_config_outer_folds, outer_fold_list[5].best_config)
def test_get_optimum_config(self): my_pipe_optimizer = Hyperpipe.Optimization( "grid_search", {}, [], "balanced_accuracy", None ) list_of_tested_configs = list() metric_default = MDBFoldMetric( metric_name="balanced_accuracy", operation=FoldOperations.MEAN, value=0.5 ) metric_best = MDBFoldMetric( metric_name="balanced_accuracy", operation=FoldOperations.MEAN, value=0.99 ) # we add looser configs, one good config, and one good config that failed # and check if the good non-failing config is chosen for i in range(10): config = MDBConfig() # number 5 is the winner if i == 5 or i == 8: config.metrics_test = [metric_best] else: config.metrics_test = [metric_default] if i == 8: config.config_failed = True list_of_tested_configs.append(config) winner_config = my_pipe_optimizer.get_optimum_config(list_of_tested_configs) self.assertIs(winner_config, list_of_tested_configs[5]) self.assertEqual(winner_config.metrics_test[0].value, 0.99)
def test_optmizer_input_str(self): with self.assertRaises(ValueError): my_pipe_optimizer = Hyperpipe.Optimization('unknown_optimizer', {}, [], 'accuracy', None) for name, opt_class in Hyperpipe.Optimization.OPTIMIZER_DICTIONARY.items( ): def get_optimizer(name): my_pipe_optimizer = Hyperpipe.Optimization( name, {}, [], 'accuracy', None) return my_pipe_optimizer.get_optimizer() if name == 'smac': with self.assertRaises(ModuleNotFoundError): get_optimizer(name) else: self.assertIsInstance(get_optimizer(name), opt_class)
def setUp(self): super(OuterFoldTests, self).setUp() self.fold_nr_inner_cv = 5 self.inner_cv = ShuffleSplit(n_splits=self.fold_nr_inner_cv, random_state=42) self.outer_cv = ShuffleSplit(n_splits=1, test_size=0.2, random_state=42) self.cv_info = Hyperpipe.CrossValidation( inner_cv=self.inner_cv, outer_cv=self.outer_cv, eval_final_performance=True, test_size=0.2, calculate_metrics_per_fold=True, calculate_metrics_across_folds=False, learning_curves=False, learning_curves_cut=None) self.X, self.y = load_boston(return_X_y=True) self.outer_fold_id = "TestFoldOuter1" self.cv_info.outer_folds = { self.outer_fold_id: FoldInfo(0, 1, train, test) for train, test in self.outer_cv.split(self.X, self.y) } self.config_num = 2 self.optimization_info = Hyperpipe.Optimization( metrics=['mean_absolute_error', 'mean_squared_error'], best_config_metric='mean_absolute_error', optimizer_input='grid_search', optimizer_params={}, performance_constraints=None) self.elements = [ PipelineElement('StandardScaler'), PipelineElement('PCA', {'n_components': [4, 7]}), PipelineElement('DecisionTreeRegressor', random_state=42) ] self.pipe = PhotonPipeline([(p.name, p) for p in self.elements])
def get_optimizer(name): my_pipe_optimizer = Hyperpipe.Optimization( name, {}, [], "accuracy", None ) return my_pipe_optimizer.get_optimizer()