def test_save_optimum_pipe_custom_element(self): tmp_path = os.path.join(self.tmp_folder_path, 'optimum_pipypipe') settings = OutputSettings(project_folder=tmp_path, overwrite_results=True) my_pipe = Hyperpipe('hyperpipe', optimizer='random_grid_search', optimizer_params={'n_configurations': 1}, metrics=['accuracy', 'precision', 'recall'], best_config_metric='f1_score', outer_cv=KFold(n_splits=2), inner_cv=KFold(n_splits=2), verbosity=1, output_settings=settings) my_pipe += PipelineElement('KerasDnnClassifier', {}, epochs=1, hidden_layer_sizes=[5]) my_pipe.fit(self.__X, self.__y) model_path = os.path.join(my_pipe.output_settings.results_folder, 'photon_best_model.photon') self.assertTrue(os.path.exists(model_path)) # check if load_optimum_pipe also works # check if we have the meta information recovered loaded_optimum_pipe = Hyperpipe.load_optimum_pipe(model_path) self.assertIsNotNone(loaded_optimum_pipe._meta_information)
def test_register_element(self): with self.assertRaises(ValueError): self.registry.register('MyCustomEstimator', 'custom_estimator.CustomEstimator', 'WrongType') self.registry.register('MyCustomEstimator', 'custom_estimator.CustomEstimator', 'Estimator') self.registry.activate() settings = OutputSettings(save_output=False, project_folder='./tmp/') # DESIGN YOUR PIPELINE pipe = Hyperpipe('custom_estimator_pipe', optimizer='random_grid_search', optimizer_params={'n_configurations': 2}, metrics=['accuracy', 'precision', 'recall', 'balanced_accuracy'], best_config_metric='accuracy', outer_cv=KFold(n_splits=2), inner_cv=KFold(n_splits=2), verbosity=1, output_settings=settings) pipe += PipelineElement('MyCustomEstimator') pipe.fit(np.random.randn(30, 30), np.random.randint(0, 2, 30)) self.registry.delete('MyCustomEstimator') os.remove(os.path.join(self.custom_folder, 'CustomElements.json'))
def test_huge_combinations(self): hp = Hyperpipe( "huge_combinations", metrics=["accuracy"], best_config_metric="accuracy", output_settings=OutputSettings( project_folder=self.tmp_folder_path), ) hp += PipelineElement("PCA", hyperparameters={"n_components": [5, 10]}) stack = Stack("ensemble") for i in range(20): stack += PipelineElement( "SVC", hyperparameters={ "C": FloatRange(0.001, 5), "kernel": ["linear", "rbf", "sigmoid", "polynomial"], }, ) hp += stack hp += PipelineElement( "SVC", hyperparameters={"kernel": ["linear", "rbf", "sigmoid"]}) X, y = load_breast_cancer(True) with self.assertRaises(Warning): hp.fit(X, y)
def generate_hyperpipes(self): if self.atlas_info_object.roi_names_runtime: self.rois = self.atlas_info_object.roi_names_runtime # # self.outer_pipe = Hyperpipe(self.atlas_name + 'outer_pipe', optimizer='grid_search', # metrics=['accuracy'], hyperparameter_specific_config_cv_object= # ShuffleSplit(n_splits=1, test_size=0.2, random_state=3), # hyperparameter_search_cv_object= # ShuffleSplit(n_splits=1, test_size=0.2, random_state=3), # eval_final_performance=True) inner_pipe_list = {} for i in range(len(self.rois)): tmp_inner_pipe = Hyperpipe(self.atlas_name + '_' + str(self.rois[i]), optimizer='grid_search', inner_cv=ShuffleSplit(n_splits=1, test_size=0.2, random_state=3), eval_final_performance=False, verbose=logging.verbosity_level, best_config_metric=self.best_config_metric, metrics=self.metrics) # at first set a filter element roi_filter_element = RoiFilterElement(i) tmp_inner_pipe.filter_element = roi_filter_element # secondly add all other items for pipe_item in self.hyperpipe_elements: tmp_inner_pipe += PipelineElement.create(pipe_item[0], pipe_item[1], **pipe_item[2]) inner_pipe_list[self.rois[i]] = tmp_inner_pipe self.pipeline_fusion = Stack('multiple_source_pipes', inner_pipe_list.values(), voting=False)
def test_three_levels_of_feature_importances(self): hyperpipe = Hyperpipe( "fimps", inner_cv=KFold(n_splits=4), outer_cv=KFold(n_splits=3), metrics=["mean_absolute_error", "mean_squared_error"], best_config_metric="mean_squared_error", output_settings=OutputSettings( project_folder=self.tmp_folder_path), ) hyperpipe += PipelineElement("StandardScaler") hyperpipe += PipelineElement("DecisionTreeRegressor") X, y = load_boston(True) hyperpipe.fit(X, y) exepcted_nr_of_feature_importances = X.shape[1] self.assertTrue( len(hyperpipe.results.best_config_feature_importances) == exepcted_nr_of_feature_importances) for outer_fold in hyperpipe.results.outer_folds: self.assertTrue( len(outer_fold.best_config.best_config_score. feature_importances) == exepcted_nr_of_feature_importances) for inner_fold in outer_fold.best_config.inner_folds: self.assertTrue( len(inner_fold.feature_importances) == exepcted_nr_of_feature_importances)
def test_sanity(self): # make sure that no metrics means raising an error with self.assertRaises(ValueError): hyperpipe = Hyperpipe("hp_name", inner_cv=self.inner_cv_object) # make sure that if no best config metric is given, PHOTON raises a warning with self.assertRaises(Warning): hyperpipe = Hyperpipe("hp_name", inner_cv=self.inner_cv_object, metrics=["accuracy", "f1_score"]) with self.assertRaises(Warning): hyperpipe = Hyperpipe("hp_name", inner_cv=self.inner_cv_object, best_config_metric=["accuracy", "f1_score"]) with self.assertRaises(NotImplementedError): hyperpipe = Hyperpipe("hp_name", inner_cv=self.inner_cv_object, best_config_metric='accuracy', metrics=["accuracy"], calculate_metrics_across_folds=False, calculate_metrics_per_fold=False) with self.assertRaises(AttributeError): hyperpipe = Hyperpipe("hp_name", best_config_metric='accuracy', metrics=["accuracy"]) data = np.random.random((500, 50)) with self.assertRaises(ValueError): targets = np.random.randint(0, 1, (500, 2)) self.hyperpipe.fit(data, targets)
def test_register_element(self): with self.assertRaises(ValueError): self.registry.register("MyCustomEstimator", "custom_estimator.CustomEstimator", "WrongType") self.registry.register("MyCustomEstimator", "custom_estimator.CustomEstimator", "Estimator") self.registry.activate() settings = OutputSettings(save_output=False, project_folder="./tmp/") # DESIGN YOUR PIPELINE pipe = Hyperpipe( "custom_estimator_pipe", optimizer="random_grid_search", optimizer_params={"n_configurations": 2}, metrics=["accuracy", "precision", "recall", "balanced_accuracy"], best_config_metric="accuracy", outer_cv=KFold(n_splits=2), inner_cv=KFold(n_splits=2), verbosity=1, output_settings=settings, ) pipe += PipelineElement("MyCustomEstimator") pipe.fit(np.random.randn(30, 30), np.random.randint(0, 2, 30)) self.registry.delete("MyCustomEstimator") os.remove(os.path.join(self.custom_folder, "CustomElements.json"))
def setUp(self): self.time_limit = 60 * 2 settings = OutputSettings(project_folder="./tmp/") self.smac_helper = {"data": None, "initial_runs": None} # DESIGN YOUR PIPELINE self.pipe = Hyperpipe( "basic_svm_pipe", # the name of your pipeline optimizer="smac", # which optimizer PHOTON shall use optimizer_params={ "wallclock_limit": self.time_limit, "smac_helper": self.smac_helper, "run_limit": 20, }, metrics=["accuracy"], # the performance metrics of your interest best_config_metric="accuracy", inner_cv=KFold( n_splits=3 ), # test each configuration ten times respectively, verbosity=0, output_settings=settings, )
def setUp(self): self.s_split = ShuffleSplit(n_splits=3, test_size=0.2, random_state=42) self.time_limit = 20 settings = OutputSettings(project_folder='./tmp/') self.smac_helper = {"data": None, "initial_runs": None} # Scenario object scenario_dict = { "run_obj": "quality", "deterministic": "true", "wallclock_limit": self.time_limit } # DESIGN YOUR PIPELINE self.pipe = Hyperpipe('basic_svm_pipe', optimizer='smac', optimizer_params={ 'facade': SMAC4HPO, 'scenario_dict': scenario_dict, 'rng': 42, 'smac_helper': self.smac_helper }, metrics=['accuracy'], random_seed=42, best_config_metric='accuracy', inner_cv=self.s_split, verbosity=0, output_settings=settings)
def setUp(self): super(InnerFoldTests, self).setUp() self.pipe = PhotonPipeline([ ("StandardScaler", PipelineElement("StandardScaler")), ("PCA", PipelineElement("PCA")), ("RidgeClassifier", PipelineElement("RidgeClassifier")), ]) self.config = { "PCA__n_components": 5, "RidgeClassifier__solver": "svd", "RidgeClassifier__random_state": 42, } self.outer_fold_id = "TestID" self.inner_cv = KFold(n_splits=4) self.X, self.y = load_breast_cancer(True) self.cross_validation = Hyperpipe.CrossValidation( self.inner_cv, None, True, 0.2, True, False) self.cross_validation.inner_folds = { self.outer_fold_id: { i: FoldInfo(i, i + 1, train, test) for i, (train, test) in enumerate(self.inner_cv.split(self.X, self.y)) } } self.optimization = Hyperpipe.Optimization( "grid_search", {}, ["accuracy", "recall", "specificity"], "accuracy", None)
def test_neuro_hyperpipe_parallelized_batched_caching(self): cache_path = self.cache_folder_path self.hyperpipe = Hyperpipe('complex_case', inner_cv=KFold(n_splits=5), outer_cv=KFold(n_splits=3), optimizer='grid_search', cache_folder=cache_path, metrics=['mean_squared_error'], best_config_metric='mean_squared_error', output_settings=OutputSettings( project_folder=self.tmp_folder_path)) nb = ParallelBranch("SubjectCaching", nr_of_processes=1) nb += PipelineElement.create("ResampleImages", StupidAdditionTransformer(), {'voxel_size': [3, 5, 10]}, batch_size=4) self.hyperpipe += nb self.hyperpipe += PipelineElement("StandardScaler", {}) self.hyperpipe += PipelineElement("PCA", {'n_components': [3, 4]}) self.hyperpipe += PipelineElement("SVR", {'kernel': ['rbf', 'linear']}) self.hyperpipe.fit(self.X, self.y) # assert cache is empty again nr_of_p_files = len( glob.glob(os.path.join(self.hyperpipe.cache_folder, "*.p"))) print(nr_of_p_files) self.assertTrue(nr_of_p_files == 0)
def test_best_config_metric(self): my_pipe_optimizer = Hyperpipe.Optimization('grid_search', {}, [], 'balanced_accuracy', None) self.assertTrue(my_pipe_optimizer.maximize_metric) my_pipe_optimizer = Hyperpipe.Optimization('grid_search', {}, [], 'mean_squared_error', None) self.assertFalse(my_pipe_optimizer.maximize_metric)
def test_recursive_disabling(self): list_of_elements_to_detect = self.setup_crazy_pipe() self.hyperpipe._pipe = Branch.prepare_photon_pipe( list_of_elements_to_detect) Hyperpipe.disable_multiprocessing_recursively(self.hyperpipe._pipe) self.assertTrue( [i.nr_of_processes == 1 for i in list_of_elements_to_detect])
def setUp(self): super(InnerFoldTests, self).setUp() self.pipe = PhotonPipeline([ ('StandardScaler', PipelineElement('StandardScaler')), ('PCA', PipelineElement('PCA')), ('RidgeClassifier', PipelineElement('RidgeClassifier')) ]) self.config = { 'PCA__n_components': 5, 'RidgeClassifier__solver': 'svd', 'RidgeClassifier__random_state': 42 } self.outer_fold_id = 'TestID' self.inner_cv = KFold(n_splits=4) self.X, self.y = load_breast_cancer(return_X_y=True) self.cross_validation = Hyperpipe.CrossValidation( self.inner_cv, None, True, 0.2, True, False, False, None) self.cross_validation.inner_folds = { self.outer_fold_id: { i: FoldInfo(i, i + 1, train, test) for i, (train, test) in enumerate(self.inner_cv.split(self.X, self.y)) } } self.optimization = Hyperpipe.Optimization( 'grid_search', {}, ['accuracy', 'recall', 'specificity'], 'accuracy', None)
def create_hyperpipe(self): self.hyperpipe = Hyperpipe('optimizer_test', output_settings=OutputSettings(project_folder='./tmp'), metrics=['accuracy'], best_config_metric='accuracy', inner_cv=KFold(n_splits=3), outer_cv=ShuffleSplit(n_splits=2), optimizer=self.optimizer_name)
def test_one_hyperpipe(learning_curves, learning_curves_cut): if learning_curves and learning_curves_cut is None: learning_curves_cut = FloatRange(0, 1, 'range', 0.2) output_settings = OutputSettings( project_folder=self.tmp_folder_path, save_output=False) test_hyperpipe = Hyperpipe( 'test_pipe', learning_curves=learning_curves, learning_curves_cut=learning_curves_cut, metrics=['accuracy', 'recall', 'specificity'], best_config_metric='accuracy', inner_cv=self.inner_cv, output_settings=output_settings) self.assertEqual(test_hyperpipe.cross_validation.learning_curves, learning_curves) if learning_curves: self.assertEqual( test_hyperpipe.cross_validation.learning_curves_cut, learning_curves_cut) else: self.assertIsNone( test_hyperpipe.cross_validation.learning_curves_cut) test_hyperpipe += PipelineElement('StandardScaler') test_hyperpipe += PipelineElement('PCA', {'n_components': [1, 2]}, random_state=42) test_hyperpipe += PipelineElement('SVC', { 'C': [0.1], 'kernel': ['linear'] }, random_state=42) test_hyperpipe.fit(self.X, self.y) config_results = test_hyperpipe.results_handler.results.outer_folds[ 0].tested_config_list config_num = len(config_results) for config_nr in range(config_num): for inner_fold_nr in range(self.inner_cv.n_splits): curves = config_results[config_nr].inner_folds[ inner_fold_nr].learning_curves if learning_curves: self.assertEqual(len(curves), len(learning_curves_cut.values)) for learning_point_nr in range( len(learning_curves_cut.values)): test_metrics = list( curves[learning_point_nr][1].keys()) train_metrics = list( curves[learning_point_nr][2].keys()) self.assertEqual( test_hyperpipe.optimization.metrics, test_metrics) self.assertEqual( test_hyperpipe.optimization.metrics, train_metrics) else: self.assertEqual(curves, [])
def test_shall_continue(self): X, y = load_boston(True) inner_fold_length = 7 # DESIGN YOUR PIPELINE my_pipe = Hyperpipe( name="performance_pipe", optimizer="random_search", optimizer_params={"limit_in_minutes": 2}, metrics=["mean_squared_error"], best_config_metric="mean_squared_error", # outer_cv=KFold(n_splits=2, shuffle=True), inner_cv=KFold(n_splits=inner_fold_length), eval_final_performance=True, performance_constraints=[self.constraint_object], ) my_pipe += PipelineElement("StandardScaler") my_pipe += PipelineElement( "RandomForestRegressor", hyperparameters={"n_estimators": IntegerRange(5, 50)}, ) # NOW TRAIN YOUR PIPELINE my_pipe.fit(X, y) # clip config results results = my_pipe.results.outer_folds[0].tested_config_list configs = [] for i in range(len(configs) - 1): configs.append([ x.validation.metrics["mean_squared_error"] for x in results[i].inner_folds ]) threshold = np.inf for val in configs[:10]: challenger = np.mean(val) if threshold > challenger: threshold = challenger originals_for_std = configs[:10] for i, val in enumerate(configs[10:]): std = np.mean([np.std(x) for x in originals_for_std]) for j, v in enumerate(val): if np.mean(val[:j + 1]) > threshold + std: self.assertEqual(v, val[-1]) continue if len(val) == inner_fold_length - 1 and np.mean( val) < threshold + std: threshold = np.mean(val) if len(val) > 1: originals_for_std.append(val)
def test_no_metrics(self): # make sure that no metrics means raising an error with self.assertRaises(ValueError): hyperpipe = Hyperpipe("hp_name", inner_cv=self.inner_cv_object) # make sure that if no best config metric is given, PHOTON raises a warning with self.assertRaises(Warning): hyperpipe = Hyperpipe("hp_name", inner_cv=self.inner_cv_object, metrics=["accuracy", "f1_score"])
def test_metrics_and_aggreation_eval_performance_false(self): self.hyperpipe = Hyperpipe('test_prediction_collection', inner_cv=KFold(n_splits=self.inner_fold_nr), metrics=['mean_absolute_error', 'mean_squared_error'], eval_final_performance=False, best_config_metric='mean_absolute_error', calculate_metrics_across_folds=True, output_settings=OutputSettings(project_folder=self.tmp_folder_path)) self.test_metrics_and_aggregations()
def setUp(self): """ Set default start settings for all tests. """ super(ResultsHandlerTest, self).setUp() self.files = [ "best_config_predictions.csv", "time_monitor.csv", "time_monitor_pie.png", "photon_result_file.p", "photon_summary.txt", "photon_best_model.photon", "optimum_pipe_feature_importances_backmapped.npz", "photon_code.py", "optimizer_history.png", ] self.output_settings = OutputSettings( project_folder=self.tmp_folder_path, save_output=True) self.ss_pipe_element = PipelineElement("StandardScaler") self.pca_pipe_element = PipelineElement("PCA", {"n_components": [1, 2]}, random_state=42) self.svc_pipe_element = PipelineElement( "SVC", { "C": [0.1], "kernel": ["linear"] }, # 'rbf', 'sigmoid'] random_state=42, ) self.inner_cv_object = KFold(n_splits=3) self.metrics = ["accuracy", "recall", "precision"] self.best_config_metric = "accuracy" self.hyperpipe = Hyperpipe( "god", inner_cv=self.inner_cv_object, metrics=self.metrics, best_config_metric=self.best_config_metric, outer_cv=KFold(n_splits=2), output_settings=self.output_settings, verbosity=1, ) self.hyperpipe += self.ss_pipe_element self.hyperpipe += self.pca_pipe_element self.hyperpipe.add(self.svc_pipe_element) dataset = load_breast_cancer() self.__X = dataset.data self.__y = dataset.target self.hyperpipe.fit(self.__X, self.__y)
def test_shall_continue(self): X, y = load_boston(return_X_y=True) inner_fold_length = 7 # DESIGN YOUR PIPELINE my_pipe = Hyperpipe( name='performance_pipe', optimizer='random_search', optimizer_params={'limit_in_minutes': 2}, metrics=['mean_squared_error'], best_config_metric='mean_squared_error', inner_cv=KFold(n_splits=inner_fold_length), eval_final_performance=True, output_settings=OutputSettings(project_folder='./tmp'), performance_constraints=[self.constraint_object]) my_pipe += PipelineElement('StandardScaler') my_pipe += PipelineElement( 'RandomForestRegressor', hyperparameters={'n_estimators': IntegerRange(5, 50)}) # NOW TRAIN YOUR PIPELINE my_pipe.fit(X, y) # clip config results results = my_pipe.results.outer_folds[0].tested_config_list configs = [] for i in range(len(configs) - 1): configs.append([ x.validation.metrics['mean_squared_error'] for x in results[i].inner_folds ]) threshold = np.inf for val in configs[:10]: challenger = np.mean(val) if threshold > challenger: threshold = challenger originals_for_std = configs[:10] for i, val in enumerate(configs[10:]): std = np.mean([np.std(x) for x in originals_for_std]) for j, v in enumerate(val): if np.mean(val[:j + 1]) > threshold + std: self.assertEqual(v, val[-1]) continue if len(val) == inner_fold_length - 1 and np.mean( val) < threshold + std: threshold = np.mean(val) if len(val) > 1: originals_for_std.append(val)
def setup_hyperpipe(self, output_settings=None): if output_settings is None: output_settings = OutputSettings( project_folder=self.tmp_folder_path) self.hyperpipe = Hyperpipe('god', inner_cv=self.inner_cv_object, metrics=self.metrics, best_config_metric=self.best_config_metric, output_settings=output_settings) self.hyperpipe += self.ss_pipe_element self.hyperpipe += self.pca_pipe_element self.hyperpipe.add(self.svc_pipe_element)
def test_inverse_transform(self): settings = OutputSettings( project_folder=self.tmp_folder_path, overwrite_results=True ) # DESIGN YOUR PIPELINE pipe = Hyperpipe( "Limbic_System", optimizer="grid_search", metrics=["mean_absolute_error"], best_config_metric="mean_absolute_error", outer_cv=ShuffleSplit(n_splits=1, test_size=0.2), inner_cv=ShuffleSplit(n_splits=1, test_size=0.2), verbosity=2, cache_folder=self.cache_folder_path, eval_final_performance=True, output_settings=settings, ) # PICK AN ATLAS atlas = PipelineElement( "BrainAtlas", rois=["Hippocampus_L", "Amygdala_L"], atlas_name="AAL", extract_mode="vec", batch_size=20, ) # EITHER ADD A NEURO BRANCH OR THE ATLAS ITSELF neuro_branch = NeuroBranch("NeuroBranch") neuro_branch += atlas pipe += neuro_branch pipe += PipelineElement("LinearSVR") pipe.fit(self.X, self.y) # GET IMPORTANCE SCORES handler = ResultsHandler(pipe.results) importance_scores_optimum_pipe = handler.results.best_config_feature_importances manual_img, _, _ = pipe.optimum_pipe.inverse_transform( importance_scores_optimum_pipe, None ) img = image.load_img( os.path.join( self.tmp_folder_path, "Limbic_System_results/optimum_pipe_feature_importances_backmapped.nii.gz", ) ) self.assertTrue(np.array_equal(manual_img.get_data(), img.get_data()))
def test_huge_combinations(self): hp = Hyperpipe('huge_combinations', inner_cv=KFold(n_splits=3), metrics=['accuracy'], best_config_metric='accuracy', output_settings=OutputSettings(project_folder=self.tmp_folder_path)) hp += PipelineElement("PCA", hyperparameters={'n_components': [5, 10]}) stack = Stack('ensemble') for i in range(20): stack += PipelineElement('SVC', hyperparameters={'C': FloatRange(0.001, 5), 'kernel': ["linear", "rbf", "sigmoid", "polynomial"]}) hp += stack hp += PipelineElement("SVC", hyperparameters={'kernel': ["linear", "rbf", "sigmoid"]}) X, y = load_breast_cancer(return_X_y=True) with self.assertRaises(Warning): hp.fit(X, y)
def setUp(self): super(ResultHandlerAndHelperTests, self).setUp() self.inner_fold_nr = 10 self.outer_fold_nr = 5 self.y_true = np.linspace(1, 100, 100) self.X = self.y_true self.hyperpipe = Hyperpipe('test_prediction_collection', inner_cv=KFold(n_splits=self.inner_fold_nr), outer_cv=KFold(n_splits=self.outer_fold_nr), metrics=['mean_absolute_error', 'mean_squared_error'], best_config_metric='mean_absolute_error', output_settings=OutputSettings(project_folder=self.tmp_folder_path), verbosity=0)
def test_class_with_data_preproc(self): """ Test for simple pipeline with data. """ X, y = load_breast_cancer(return_X_y=True) # DESIGN YOUR PIPELINE my_pipe = Hyperpipe( 'basic_svm_pipe', optimizer='grid_search', metrics=['accuracy', 'precision', 'recall', 'balanced_accuracy'], best_config_metric='accuracy', eval_final_performance=False, outer_cv=KFold(n_splits=2), inner_cv=KFold(n_splits=3), verbosity=1, random_seed=42) preprocessing = Preprocessing() preprocessing += PipelineElement("LabelEncoder") my_pipe += preprocessing # ADD ELEMENTS TO YOUR PIPELINE # first normalize all features my_pipe.add(PipelineElement('StandardScaler')) # then do feature selection using a PCA, my_pipe += PipelineElement( 'PCA', hyperparameters={'n_components': IntegerRange(10, 12)}, test_disabled=True) # engage and optimize the good old SVM for Classification my_pipe += PipelineElement( 'SVC', hyperparameters={'kernel': Categorical(['rbf', 'linear'])}, C=2, gamma='scale') # NOW TRAIN YOUR PIPELINE my_pipe.fit(X, y) json_transformer = JsonTransformer() pipe_json = json_transformer.create_json(my_pipe) a = elements_to_dict(my_pipe.copy_me()) my_pipe_reload = json_transformer.from_json(pipe_json) pipe_json_reload = pipe_json = json_transformer.create_json( my_pipe_reload) self.assertEqual(pipe_json, pipe_json_reload) my_pipe_reload.fit(X, y) self.assertDictEqual(my_pipe.best_config, my_pipe_reload.best_config) self.assertDictEqual(elements_to_dict(my_pipe.copy_me()), elements_to_dict(my_pipe_reload.copy_me()))
def setUp(self): """ Set default start settings for all tests. """ super(ResultsHandlerTest, self).setUp() self.files = [ 'best_config_predictions.csv', 'time_monitor.csv', 'time_monitor_pie.png', 'photon_result_file.p', 'photon_summary.txt', 'photon_best_model.photon', 'optimum_pipe_feature_importances_backmapped.npz', 'photon_code.py', 'optimizer_history.png' ] self.output_settings = OutputSettings( project_folder=self.tmp_folder_path, save_output=True) self.ss_pipe_element = PipelineElement('StandardScaler') self.pca_pipe_element = PipelineElement('PCA', {'n_components': [1, 2]}, random_state=42) self.svc_pipe_element = PipelineElement( 'SVC', { 'C': [0.1], 'kernel': ['linear'] }, # 'rbf', 'sigmoid'] random_state=42) self.inner_cv_object = KFold(n_splits=3) self.metrics = ["accuracy", 'recall', 'precision'] self.best_config_metric = "accuracy" self.hyperpipe = Hyperpipe('god', inner_cv=self.inner_cv_object, metrics=self.metrics, best_config_metric=self.best_config_metric, outer_cv=KFold(n_splits=2), output_settings=self.output_settings, verbosity=1) self.hyperpipe += self.ss_pipe_element self.hyperpipe += self.pca_pipe_element self.hyperpipe.add(self.svc_pipe_element) dataset = load_breast_cancer() self.__X = dataset.data self.__y = dataset.target self.hyperpipe.fit(self.__X, self.__y)
def create_hyperpipe(self): # this is needed here for the parallelisation from photonai.base import Hyperpipe, PipelineElement, OutputSettings from photonai.optimization import FloatRange, Categorical, IntegerRange from sklearn.model_selection import GroupKFold from sklearn.model_selection import KFold settings = OutputSettings(mongodb_connect_url='mongodb://localhost:27017/photon_results', project_folder=self.tmp_folder_path) my_pipe = Hyperpipe('permutation_test_1', optimizer='grid_search', metrics=['accuracy', 'precision', 'recall'], best_config_metric='accuracy', outer_cv=GroupKFold(n_splits=2), inner_cv=KFold(n_splits=2), calculate_metrics_across_folds=True, eval_final_performance=True, verbosity=1, output_settings=settings) # Add transformer elements my_pipe += PipelineElement("StandardScaler", hyperparameters={}, test_disabled=False, with_mean=True, with_std=True) my_pipe += PipelineElement("PCA", hyperparameters={'n_components': IntegerRange(3, 5)}, test_disabled=False) # Add estimator my_pipe += PipelineElement("SVC", hyperparameters={'kernel': ['linear', 'rbf']}, # C': FloatRange(0.1, 5), gamma='scale', max_iter=1000000) return my_pipe
def test_metrics_and_aggregations_no_outer_cv_but_eval_performance_true( self): self.hyperpipe = Hyperpipe( "test_prediction_collection", outer_cv=KFold(n_splits=self.outer_fold_nr), inner_cv=KFold(n_splits=self.inner_fold_nr), metrics=["mean_absolute_error", "mean_squared_error"], eval_final_performance=False, best_config_metric="mean_absolute_error", calculate_metrics_per_fold=True, calculate_metrics_across_folds=True, output_settings=OutputSettings( project_folder=self.tmp_folder_path), ) self.test_metrics_and_aggregations()
def test_get_optimum_config_outer_folds(self): my_pipe_optimizer = Hyperpipe.Optimization( "grid_search", {}, [], "balanced_accuracy", None ) outer_fold_list = list() for i in range(10): outer_fold = MDBOuterFold() outer_fold.best_config = MDBConfig() outer_fold.best_config.best_config_score = MDBInnerFold() outer_fold.best_config.best_config_score.validation = MDBScoreInformation() # again fold 5 wins if i == 5: outer_fold.best_config.best_config_score.validation.metrics = { "balanced_accuracy": 0.99 } else: outer_fold.best_config.best_config_score.validation.metrics = { "balanced_accuracy": 0.5 } outer_fold_list.append(outer_fold) best_config_outer_folds = my_pipe_optimizer.get_optimum_config_outer_folds( outer_fold_list ) self.assertEqual( best_config_outer_folds.best_config_score.validation.metrics[ "balanced_accuracy" ], 0.99, ) self.assertIs(best_config_outer_folds, outer_fold_list[5].best_config)