def test_save_optimum_pipe_custom_element(self):
tmp_path = os.path.join(self.tmp_folder_path, 'optimum_pipypipe')
settings = OutputSettings(project_folder=tmp_path,
overwrite_results=True)
my_pipe = Hyperpipe('hyperpipe',
optimizer='random_grid_search',
optimizer_params={'n_configurations': 1},
metrics=['accuracy', 'precision', 'recall'],
best_config_metric='f1_score',
outer_cv=KFold(n_splits=2),
inner_cv=KFold(n_splits=2),
verbosity=1,
output_settings=settings)
my_pipe += PipelineElement('KerasDnnClassifier', {},
epochs=1,
hidden_layer_sizes=[5])
my_pipe.fit(self.__X, self.__y)
model_path = os.path.join(my_pipe.output_settings.results_folder,
'photon_best_model.photon')
self.assertTrue(os.path.exists(model_path))
# check if load_optimum_pipe also works
# check if we have the meta information recovered
loaded_optimum_pipe = Hyperpipe.load_optimum_pipe(model_path)
self.assertIsNotNone(loaded_optimum_pipe._meta_information)
def test_register_element(self):
with self.assertRaises(ValueError):
self.registry.register('MyCustomEstimator', 'custom_estimator.CustomEstimator', 'WrongType')
self.registry.register('MyCustomEstimator', 'custom_estimator.CustomEstimator', 'Estimator')
self.registry.activate()
settings = OutputSettings(save_output=False, project_folder='./tmp/')
# DESIGN YOUR PIPELINE
pipe = Hyperpipe('custom_estimator_pipe',
optimizer='random_grid_search',
optimizer_params={'n_configurations': 2},
metrics=['accuracy', 'precision', 'recall', 'balanced_accuracy'],
best_config_metric='accuracy',
outer_cv=KFold(n_splits=2),
inner_cv=KFold(n_splits=2),
verbosity=1,
output_settings=settings)
pipe += PipelineElement('MyCustomEstimator')
pipe.fit(np.random.randn(30, 30), np.random.randint(0, 2, 30))
self.registry.delete('MyCustomEstimator')
os.remove(os.path.join(self.custom_folder, 'CustomElements.json'))
def test_huge_combinations(self):
hp = Hyperpipe(
"huge_combinations",
metrics=["accuracy"],
best_config_metric="accuracy",
output_settings=OutputSettings(
project_folder=self.tmp_folder_path),
)
hp += PipelineElement("PCA", hyperparameters={"n_components": [5, 10]})
stack = Stack("ensemble")
for i in range(20):
stack += PipelineElement(
"SVC",
hyperparameters={
"C": FloatRange(0.001, 5),
"kernel": ["linear", "rbf", "sigmoid", "polynomial"],
},
)
hp += stack
hp += PipelineElement(
"SVC", hyperparameters={"kernel": ["linear", "rbf", "sigmoid"]})
X, y = load_breast_cancer(True)
with self.assertRaises(Warning):
hp.fit(X, y)
def generate_hyperpipes(self):
if self.atlas_info_object.roi_names_runtime:
self.rois = self.atlas_info_object.roi_names_runtime
#
# self.outer_pipe = Hyperpipe(self.atlas_name + 'outer_pipe', optimizer='grid_search',
# metrics=['accuracy'], hyperparameter_specific_config_cv_object=
# ShuffleSplit(n_splits=1, test_size=0.2, random_state=3),
# hyperparameter_search_cv_object=
# ShuffleSplit(n_splits=1, test_size=0.2, random_state=3),
# eval_final_performance=True)
inner_pipe_list = {}
for i in range(len(self.rois)):
tmp_inner_pipe = Hyperpipe(self.atlas_name + '_' + str(self.rois[i]), optimizer='grid_search',
inner_cv=ShuffleSplit(n_splits=1, test_size=0.2, random_state=3),
eval_final_performance=False, verbose=logging.verbosity_level,
best_config_metric=self.best_config_metric, metrics=self.metrics)
# at first set a filter element
roi_filter_element = RoiFilterElement(i)
tmp_inner_pipe.filter_element = roi_filter_element
# secondly add all other items
for pipe_item in self.hyperpipe_elements:
tmp_inner_pipe += PipelineElement.create(pipe_item[0], pipe_item[1], **pipe_item[2])
inner_pipe_list[self.rois[i]] = tmp_inner_pipe
self.pipeline_fusion = Stack('multiple_source_pipes', inner_pipe_list.values(), voting=False)
def test_three_levels_of_feature_importances(self):
hyperpipe = Hyperpipe(
"fimps",
inner_cv=KFold(n_splits=4),
outer_cv=KFold(n_splits=3),
metrics=["mean_absolute_error", "mean_squared_error"],
best_config_metric="mean_squared_error",
output_settings=OutputSettings(
project_folder=self.tmp_folder_path),
)
hyperpipe += PipelineElement("StandardScaler")
hyperpipe += PipelineElement("DecisionTreeRegressor")
X, y = load_boston(True)
hyperpipe.fit(X, y)
exepcted_nr_of_feature_importances = X.shape[1]
self.assertTrue(
len(hyperpipe.results.best_config_feature_importances) ==
exepcted_nr_of_feature_importances)
for outer_fold in hyperpipe.results.outer_folds:
self.assertTrue(
len(outer_fold.best_config.best_config_score.
feature_importances) == exepcted_nr_of_feature_importances)
for inner_fold in outer_fold.best_config.inner_folds:
self.assertTrue(
len(inner_fold.feature_importances) ==
exepcted_nr_of_feature_importances)
def test_sanity(self):
# make sure that no metrics means raising an error
with self.assertRaises(ValueError):
hyperpipe = Hyperpipe("hp_name", inner_cv=self.inner_cv_object)
# make sure that if no best config metric is given, PHOTON raises a warning
with self.assertRaises(Warning):
hyperpipe = Hyperpipe("hp_name",
inner_cv=self.inner_cv_object,
metrics=["accuracy", "f1_score"])
with self.assertRaises(Warning):
hyperpipe = Hyperpipe("hp_name",
inner_cv=self.inner_cv_object,
best_config_metric=["accuracy", "f1_score"])
with self.assertRaises(NotImplementedError):
hyperpipe = Hyperpipe("hp_name",
inner_cv=self.inner_cv_object,
best_config_metric='accuracy',
metrics=["accuracy"],
calculate_metrics_across_folds=False,
calculate_metrics_per_fold=False)
with self.assertRaises(AttributeError):
hyperpipe = Hyperpipe("hp_name",
best_config_metric='accuracy',
metrics=["accuracy"])
data = np.random.random((500, 50))
with self.assertRaises(ValueError):
targets = np.random.randint(0, 1, (500, 2))
self.hyperpipe.fit(data, targets)
def test_register_element(self):
with self.assertRaises(ValueError):
self.registry.register("MyCustomEstimator",
"custom_estimator.CustomEstimator",
"WrongType")
self.registry.register("MyCustomEstimator",
"custom_estimator.CustomEstimator", "Estimator")
self.registry.activate()
settings = OutputSettings(save_output=False, project_folder="./tmp/")
# DESIGN YOUR PIPELINE
pipe = Hyperpipe(
"custom_estimator_pipe",
optimizer="random_grid_search",
optimizer_params={"n_configurations": 2},
metrics=["accuracy", "precision", "recall", "balanced_accuracy"],
best_config_metric="accuracy",
outer_cv=KFold(n_splits=2),
inner_cv=KFold(n_splits=2),
verbosity=1,
output_settings=settings,
)
pipe += PipelineElement("MyCustomEstimator")
pipe.fit(np.random.randn(30, 30), np.random.randint(0, 2, 30))
self.registry.delete("MyCustomEstimator")
os.remove(os.path.join(self.custom_folder, "CustomElements.json"))
def setUp(self):
self.time_limit = 60 * 2
settings = OutputSettings(project_folder="./tmp/")
self.smac_helper = {"data": None, "initial_runs": None}
# DESIGN YOUR PIPELINE
self.pipe = Hyperpipe(
"basic_svm_pipe", # the name of your pipeline
optimizer="smac", # which optimizer PHOTON shall use
optimizer_params={
"wallclock_limit": self.time_limit,
"smac_helper": self.smac_helper,
"run_limit": 20,
},
metrics=["accuracy"],
# the performance metrics of your interest
best_config_metric="accuracy",
inner_cv=KFold(
n_splits=3
), # test each configuration ten times respectively,
verbosity=0,
output_settings=settings,
)
def setUp(self):
self.s_split = ShuffleSplit(n_splits=3,
test_size=0.2,
random_state=42)
self.time_limit = 20
settings = OutputSettings(project_folder='./tmp/')
self.smac_helper = {"data": None, "initial_runs": None}
# Scenario object
scenario_dict = {
"run_obj": "quality",
"deterministic": "true",
"wallclock_limit": self.time_limit
}
# DESIGN YOUR PIPELINE
self.pipe = Hyperpipe('basic_svm_pipe',
optimizer='smac',
optimizer_params={
'facade': SMAC4HPO,
'scenario_dict': scenario_dict,
'rng': 42,
'smac_helper': self.smac_helper
},
metrics=['accuracy'],
random_seed=42,
best_config_metric='accuracy',
inner_cv=self.s_split,
verbosity=0,
output_settings=settings)
def setUp(self):
super(InnerFoldTests, self).setUp()
self.pipe = PhotonPipeline([
("StandardScaler", PipelineElement("StandardScaler")),
("PCA", PipelineElement("PCA")),
("RidgeClassifier", PipelineElement("RidgeClassifier")),
])
self.config = {
"PCA__n_components": 5,
"RidgeClassifier__solver": "svd",
"RidgeClassifier__random_state": 42,
}
self.outer_fold_id = "TestID"
self.inner_cv = KFold(n_splits=4)
self.X, self.y = load_breast_cancer(True)
self.cross_validation = Hyperpipe.CrossValidation(
self.inner_cv, None, True, 0.2, True, False)
self.cross_validation.inner_folds = {
self.outer_fold_id: {
i: FoldInfo(i, i + 1, train, test)
for i, (train,
test) in enumerate(self.inner_cv.split(self.X, self.y))
}
}
self.optimization = Hyperpipe.Optimization(
"grid_search", {}, ["accuracy", "recall", "specificity"],
"accuracy", None)
def test_neuro_hyperpipe_parallelized_batched_caching(self):
cache_path = self.cache_folder_path
self.hyperpipe = Hyperpipe('complex_case',
inner_cv=KFold(n_splits=5),
outer_cv=KFold(n_splits=3),
optimizer='grid_search',
cache_folder=cache_path,
metrics=['mean_squared_error'],
best_config_metric='mean_squared_error',
output_settings=OutputSettings(
project_folder=self.tmp_folder_path))
nb = ParallelBranch("SubjectCaching", nr_of_processes=1)
nb += PipelineElement.create("ResampleImages",
StupidAdditionTransformer(),
{'voxel_size': [3, 5, 10]},
batch_size=4)
self.hyperpipe += nb
self.hyperpipe += PipelineElement("StandardScaler", {})
self.hyperpipe += PipelineElement("PCA", {'n_components': [3, 4]})
self.hyperpipe += PipelineElement("SVR", {'kernel': ['rbf', 'linear']})
self.hyperpipe.fit(self.X, self.y)
# assert cache is empty again
nr_of_p_files = len(
glob.glob(os.path.join(self.hyperpipe.cache_folder, "*.p")))
print(nr_of_p_files)
self.assertTrue(nr_of_p_files == 0)
def test_best_config_metric(self):
my_pipe_optimizer = Hyperpipe.Optimization('grid_search', {}, [],
'balanced_accuracy', None)
self.assertTrue(my_pipe_optimizer.maximize_metric)
my_pipe_optimizer = Hyperpipe.Optimization('grid_search', {}, [],
'mean_squared_error', None)
self.assertFalse(my_pipe_optimizer.maximize_metric)
def test_recursive_disabling(self):
list_of_elements_to_detect = self.setup_crazy_pipe()
self.hyperpipe._pipe = Branch.prepare_photon_pipe(
list_of_elements_to_detect)
Hyperpipe.disable_multiprocessing_recursively(self.hyperpipe._pipe)
self.assertTrue(
[i.nr_of_processes == 1 for i in list_of_elements_to_detect])
def setUp(self):
super(InnerFoldTests, self).setUp()
self.pipe = PhotonPipeline([
('StandardScaler', PipelineElement('StandardScaler')),
('PCA', PipelineElement('PCA')),
('RidgeClassifier', PipelineElement('RidgeClassifier'))
])
self.config = {
'PCA__n_components': 5,
'RidgeClassifier__solver': 'svd',
'RidgeClassifier__random_state': 42
}
self.outer_fold_id = 'TestID'
self.inner_cv = KFold(n_splits=4)
self.X, self.y = load_breast_cancer(return_X_y=True)
self.cross_validation = Hyperpipe.CrossValidation(
self.inner_cv, None, True, 0.2, True, False, False, None)
self.cross_validation.inner_folds = {
self.outer_fold_id: {
i: FoldInfo(i, i + 1, train, test)
for i, (train,
test) in enumerate(self.inner_cv.split(self.X, self.y))
}
}
self.optimization = Hyperpipe.Optimization(
'grid_search', {}, ['accuracy', 'recall', 'specificity'],
'accuracy', None)
def create_hyperpipe(self):
self.hyperpipe = Hyperpipe('optimizer_test',
output_settings=OutputSettings(project_folder='./tmp'),
metrics=['accuracy'],
best_config_metric='accuracy',
inner_cv=KFold(n_splits=3),
outer_cv=ShuffleSplit(n_splits=2),
optimizer=self.optimizer_name)
def test_one_hyperpipe(learning_curves, learning_curves_cut):
if learning_curves and learning_curves_cut is None:
learning_curves_cut = FloatRange(0, 1, 'range', 0.2)
output_settings = OutputSettings(
project_folder=self.tmp_folder_path, save_output=False)
test_hyperpipe = Hyperpipe(
'test_pipe',
learning_curves=learning_curves,
learning_curves_cut=learning_curves_cut,
metrics=['accuracy', 'recall', 'specificity'],
best_config_metric='accuracy',
inner_cv=self.inner_cv,
output_settings=output_settings)
self.assertEqual(test_hyperpipe.cross_validation.learning_curves,
learning_curves)
if learning_curves:
self.assertEqual(
test_hyperpipe.cross_validation.learning_curves_cut,
learning_curves_cut)
else:
self.assertIsNone(
test_hyperpipe.cross_validation.learning_curves_cut)
test_hyperpipe += PipelineElement('StandardScaler')
test_hyperpipe += PipelineElement('PCA', {'n_components': [1, 2]},
random_state=42)
test_hyperpipe += PipelineElement('SVC', {
'C': [0.1],
'kernel': ['linear']
},
random_state=42)
test_hyperpipe.fit(self.X, self.y)
config_results = test_hyperpipe.results_handler.results.outer_folds[
0].tested_config_list
config_num = len(config_results)
for config_nr in range(config_num):
for inner_fold_nr in range(self.inner_cv.n_splits):
curves = config_results[config_nr].inner_folds[
inner_fold_nr].learning_curves
if learning_curves:
self.assertEqual(len(curves),
len(learning_curves_cut.values))
for learning_point_nr in range(
len(learning_curves_cut.values)):
test_metrics = list(
curves[learning_point_nr][1].keys())
train_metrics = list(
curves[learning_point_nr][2].keys())
self.assertEqual(
test_hyperpipe.optimization.metrics,
test_metrics)
self.assertEqual(
test_hyperpipe.optimization.metrics,
train_metrics)
else:
self.assertEqual(curves, [])
def test_shall_continue(self):
X, y = load_boston(True)
inner_fold_length = 7
# DESIGN YOUR PIPELINE
my_pipe = Hyperpipe(
name="performance_pipe",
optimizer="random_search",
optimizer_params={"limit_in_minutes": 2},
metrics=["mean_squared_error"],
best_config_metric="mean_squared_error",
# outer_cv=KFold(n_splits=2, shuffle=True),
inner_cv=KFold(n_splits=inner_fold_length),
eval_final_performance=True,
performance_constraints=[self.constraint_object],
)
my_pipe += PipelineElement("StandardScaler")
my_pipe += PipelineElement(
"RandomForestRegressor",
hyperparameters={"n_estimators": IntegerRange(5, 50)},
)
# NOW TRAIN YOUR PIPELINE
my_pipe.fit(X, y)
# clip config results
results = my_pipe.results.outer_folds[0].tested_config_list
configs = []
for i in range(len(configs) - 1):
configs.append([
x.validation.metrics["mean_squared_error"]
for x in results[i].inner_folds
])
threshold = np.inf
for val in configs[:10]:
challenger = np.mean(val)
if threshold > challenger:
threshold = challenger
originals_for_std = configs[:10]
for i, val in enumerate(configs[10:]):
std = np.mean([np.std(x) for x in originals_for_std])
for j, v in enumerate(val):
if np.mean(val[:j + 1]) > threshold + std:
self.assertEqual(v, val[-1])
continue
if len(val) == inner_fold_length - 1 and np.mean(
val) < threshold + std:
threshold = np.mean(val)
if len(val) > 1:
originals_for_std.append(val)
def test_no_metrics(self):
# make sure that no metrics means raising an error
with self.assertRaises(ValueError):
hyperpipe = Hyperpipe("hp_name", inner_cv=self.inner_cv_object)
# make sure that if no best config metric is given, PHOTON raises a warning
with self.assertRaises(Warning):
hyperpipe = Hyperpipe("hp_name",
inner_cv=self.inner_cv_object,
metrics=["accuracy", "f1_score"])
def test_metrics_and_aggreation_eval_performance_false(self):
self.hyperpipe = Hyperpipe('test_prediction_collection',
inner_cv=KFold(n_splits=self.inner_fold_nr),
metrics=['mean_absolute_error', 'mean_squared_error'],
eval_final_performance=False,
best_config_metric='mean_absolute_error',
calculate_metrics_across_folds=True,
output_settings=OutputSettings(project_folder=self.tmp_folder_path))
self.test_metrics_and_aggregations()
def setUp(self):
"""
Set default start settings for all tests.
"""
super(ResultsHandlerTest, self).setUp()
self.files = [
"best_config_predictions.csv",
"time_monitor.csv",
"time_monitor_pie.png",
"photon_result_file.p",
"photon_summary.txt",
"photon_best_model.photon",
"optimum_pipe_feature_importances_backmapped.npz",
"photon_code.py",
"optimizer_history.png",
]
self.output_settings = OutputSettings(
project_folder=self.tmp_folder_path, save_output=True)
self.ss_pipe_element = PipelineElement("StandardScaler")
self.pca_pipe_element = PipelineElement("PCA",
{"n_components": [1, 2]},
random_state=42)
self.svc_pipe_element = PipelineElement(
"SVC",
{
"C": [0.1],
"kernel": ["linear"]
}, # 'rbf', 'sigmoid']
random_state=42,
)
self.inner_cv_object = KFold(n_splits=3)
self.metrics = ["accuracy", "recall", "precision"]
self.best_config_metric = "accuracy"
self.hyperpipe = Hyperpipe(
"god",
inner_cv=self.inner_cv_object,
metrics=self.metrics,
best_config_metric=self.best_config_metric,
outer_cv=KFold(n_splits=2),
output_settings=self.output_settings,
verbosity=1,
)
self.hyperpipe += self.ss_pipe_element
self.hyperpipe += self.pca_pipe_element
self.hyperpipe.add(self.svc_pipe_element)
dataset = load_breast_cancer()
self.__X = dataset.data
self.__y = dataset.target
self.hyperpipe.fit(self.__X, self.__y)
def test_shall_continue(self):
X, y = load_boston(return_X_y=True)
inner_fold_length = 7
# DESIGN YOUR PIPELINE
my_pipe = Hyperpipe(
name='performance_pipe',
optimizer='random_search',
optimizer_params={'limit_in_minutes': 2},
metrics=['mean_squared_error'],
best_config_metric='mean_squared_error',
inner_cv=KFold(n_splits=inner_fold_length),
eval_final_performance=True,
output_settings=OutputSettings(project_folder='./tmp'),
performance_constraints=[self.constraint_object])
my_pipe += PipelineElement('StandardScaler')
my_pipe += PipelineElement(
'RandomForestRegressor',
hyperparameters={'n_estimators': IntegerRange(5, 50)})
# NOW TRAIN YOUR PIPELINE
my_pipe.fit(X, y)
# clip config results
results = my_pipe.results.outer_folds[0].tested_config_list
configs = []
for i in range(len(configs) - 1):
configs.append([
x.validation.metrics['mean_squared_error']
for x in results[i].inner_folds
])
threshold = np.inf
for val in configs[:10]:
challenger = np.mean(val)
if threshold > challenger:
threshold = challenger
originals_for_std = configs[:10]
for i, val in enumerate(configs[10:]):
std = np.mean([np.std(x) for x in originals_for_std])
for j, v in enumerate(val):
if np.mean(val[:j + 1]) > threshold + std:
self.assertEqual(v, val[-1])
continue
if len(val) == inner_fold_length - 1 and np.mean(
val) < threshold + std:
threshold = np.mean(val)
if len(val) > 1:
originals_for_std.append(val)
def setup_hyperpipe(self, output_settings=None):
if output_settings is None:
output_settings = OutputSettings(
project_folder=self.tmp_folder_path)
self.hyperpipe = Hyperpipe('god',
inner_cv=self.inner_cv_object,
metrics=self.metrics,
best_config_metric=self.best_config_metric,
output_settings=output_settings)
self.hyperpipe += self.ss_pipe_element
self.hyperpipe += self.pca_pipe_element
self.hyperpipe.add(self.svc_pipe_element)
def test_inverse_transform(self):
settings = OutputSettings(
project_folder=self.tmp_folder_path, overwrite_results=True
)
# DESIGN YOUR PIPELINE
pipe = Hyperpipe(
"Limbic_System",
optimizer="grid_search",
metrics=["mean_absolute_error"],
best_config_metric="mean_absolute_error",
outer_cv=ShuffleSplit(n_splits=1, test_size=0.2),
inner_cv=ShuffleSplit(n_splits=1, test_size=0.2),
verbosity=2,
cache_folder=self.cache_folder_path,
eval_final_performance=True,
output_settings=settings,
)
# PICK AN ATLAS
atlas = PipelineElement(
"BrainAtlas",
rois=["Hippocampus_L", "Amygdala_L"],
atlas_name="AAL",
extract_mode="vec",
batch_size=20,
)
# EITHER ADD A NEURO BRANCH OR THE ATLAS ITSELF
neuro_branch = NeuroBranch("NeuroBranch")
neuro_branch += atlas
pipe += neuro_branch
pipe += PipelineElement("LinearSVR")
pipe.fit(self.X, self.y)
# GET IMPORTANCE SCORES
handler = ResultsHandler(pipe.results)
importance_scores_optimum_pipe = handler.results.best_config_feature_importances
manual_img, _, _ = pipe.optimum_pipe.inverse_transform(
importance_scores_optimum_pipe, None
)
img = image.load_img(
os.path.join(
self.tmp_folder_path,
"Limbic_System_results/optimum_pipe_feature_importances_backmapped.nii.gz",
)
)
self.assertTrue(np.array_equal(manual_img.get_data(), img.get_data()))
def test_huge_combinations(self):
hp = Hyperpipe('huge_combinations', inner_cv=KFold(n_splits=3), metrics=['accuracy'], best_config_metric='accuracy',
output_settings=OutputSettings(project_folder=self.tmp_folder_path))
hp += PipelineElement("PCA", hyperparameters={'n_components': [5, 10]})
stack = Stack('ensemble')
for i in range(20):
stack += PipelineElement('SVC', hyperparameters={'C': FloatRange(0.001, 5),
'kernel': ["linear", "rbf", "sigmoid", "polynomial"]})
hp += stack
hp += PipelineElement("SVC", hyperparameters={'kernel': ["linear", "rbf", "sigmoid"]})
X, y = load_breast_cancer(return_X_y=True)
with self.assertRaises(Warning):
hp.fit(X, y)
def setUp(self):
super(ResultHandlerAndHelperTests, self).setUp()
self.inner_fold_nr = 10
self.outer_fold_nr = 5
self.y_true = np.linspace(1, 100, 100)
self.X = self.y_true
self.hyperpipe = Hyperpipe('test_prediction_collection',
inner_cv=KFold(n_splits=self.inner_fold_nr),
outer_cv=KFold(n_splits=self.outer_fold_nr),
metrics=['mean_absolute_error', 'mean_squared_error'],
best_config_metric='mean_absolute_error',
output_settings=OutputSettings(project_folder=self.tmp_folder_path),
verbosity=0)
def test_class_with_data_preproc(self):
"""
Test for simple pipeline with data.
"""
X, y = load_breast_cancer(return_X_y=True)
# DESIGN YOUR PIPELINE
my_pipe = Hyperpipe(
'basic_svm_pipe',
optimizer='grid_search',
metrics=['accuracy', 'precision', 'recall', 'balanced_accuracy'],
best_config_metric='accuracy',
eval_final_performance=False,
outer_cv=KFold(n_splits=2),
inner_cv=KFold(n_splits=3),
verbosity=1,
random_seed=42)
preprocessing = Preprocessing()
preprocessing += PipelineElement("LabelEncoder")
my_pipe += preprocessing
# ADD ELEMENTS TO YOUR PIPELINE
# first normalize all features
my_pipe.add(PipelineElement('StandardScaler'))
# then do feature selection using a PCA,
my_pipe += PipelineElement(
'PCA',
hyperparameters={'n_components': IntegerRange(10, 12)},
test_disabled=True)
# engage and optimize the good old SVM for Classification
my_pipe += PipelineElement(
'SVC',
hyperparameters={'kernel': Categorical(['rbf', 'linear'])},
C=2,
gamma='scale')
# NOW TRAIN YOUR PIPELINE
my_pipe.fit(X, y)
json_transformer = JsonTransformer()
pipe_json = json_transformer.create_json(my_pipe)
a = elements_to_dict(my_pipe.copy_me())
my_pipe_reload = json_transformer.from_json(pipe_json)
pipe_json_reload = pipe_json = json_transformer.create_json(
my_pipe_reload)
self.assertEqual(pipe_json, pipe_json_reload)
my_pipe_reload.fit(X, y)
self.assertDictEqual(my_pipe.best_config, my_pipe_reload.best_config)
self.assertDictEqual(elements_to_dict(my_pipe.copy_me()),
elements_to_dict(my_pipe_reload.copy_me()))
def setUp(self):
"""
Set default start settings for all tests.
"""
super(ResultsHandlerTest, self).setUp()
self.files = [
'best_config_predictions.csv', 'time_monitor.csv',
'time_monitor_pie.png', 'photon_result_file.p',
'photon_summary.txt', 'photon_best_model.photon',
'optimum_pipe_feature_importances_backmapped.npz',
'photon_code.py', 'optimizer_history.png'
]
self.output_settings = OutputSettings(
project_folder=self.tmp_folder_path, save_output=True)
self.ss_pipe_element = PipelineElement('StandardScaler')
self.pca_pipe_element = PipelineElement('PCA',
{'n_components': [1, 2]},
random_state=42)
self.svc_pipe_element = PipelineElement(
'SVC',
{
'C': [0.1],
'kernel': ['linear']
}, # 'rbf', 'sigmoid']
random_state=42)
self.inner_cv_object = KFold(n_splits=3)
self.metrics = ["accuracy", 'recall', 'precision']
self.best_config_metric = "accuracy"
self.hyperpipe = Hyperpipe('god',
inner_cv=self.inner_cv_object,
metrics=self.metrics,
best_config_metric=self.best_config_metric,
outer_cv=KFold(n_splits=2),
output_settings=self.output_settings,
verbosity=1)
self.hyperpipe += self.ss_pipe_element
self.hyperpipe += self.pca_pipe_element
self.hyperpipe.add(self.svc_pipe_element)
dataset = load_breast_cancer()
self.__X = dataset.data
self.__y = dataset.target
self.hyperpipe.fit(self.__X, self.__y)
def create_hyperpipe(self):
# this is needed here for the parallelisation
from photonai.base import Hyperpipe, PipelineElement, OutputSettings
from photonai.optimization import FloatRange, Categorical, IntegerRange
from sklearn.model_selection import GroupKFold
from sklearn.model_selection import KFold
settings = OutputSettings(mongodb_connect_url='mongodb://localhost:27017/photon_results',
project_folder=self.tmp_folder_path)
my_pipe = Hyperpipe('permutation_test_1',
optimizer='grid_search',
metrics=['accuracy', 'precision', 'recall'],
best_config_metric='accuracy',
outer_cv=GroupKFold(n_splits=2),
inner_cv=KFold(n_splits=2),
calculate_metrics_across_folds=True,
eval_final_performance=True,
verbosity=1,
output_settings=settings)
# Add transformer elements
my_pipe += PipelineElement("StandardScaler", hyperparameters={},
test_disabled=False, with_mean=True, with_std=True)
my_pipe += PipelineElement("PCA", hyperparameters={'n_components': IntegerRange(3, 5)},
test_disabled=False)
# Add estimator
my_pipe += PipelineElement("SVC", hyperparameters={'kernel': ['linear', 'rbf']}, # C': FloatRange(0.1, 5),
gamma='scale', max_iter=1000000)
return my_pipe
def test_metrics_and_aggregations_no_outer_cv_but_eval_performance_true(
self):
self.hyperpipe = Hyperpipe(
"test_prediction_collection",
outer_cv=KFold(n_splits=self.outer_fold_nr),
inner_cv=KFold(n_splits=self.inner_fold_nr),
metrics=["mean_absolute_error", "mean_squared_error"],
eval_final_performance=False,
best_config_metric="mean_absolute_error",
calculate_metrics_per_fold=True,
calculate_metrics_across_folds=True,
output_settings=OutputSettings(
project_folder=self.tmp_folder_path),
)
self.test_metrics_and_aggregations()
def test_get_optimum_config_outer_folds(self):
my_pipe_optimizer = Hyperpipe.Optimization(
"grid_search", {}, [], "balanced_accuracy", None
)
outer_fold_list = list()
for i in range(10):
outer_fold = MDBOuterFold()
outer_fold.best_config = MDBConfig()
outer_fold.best_config.best_config_score = MDBInnerFold()
outer_fold.best_config.best_config_score.validation = MDBScoreInformation()
# again fold 5 wins
if i == 5:
outer_fold.best_config.best_config_score.validation.metrics = {
"balanced_accuracy": 0.99
}
else:
outer_fold.best_config.best_config_score.validation.metrics = {
"balanced_accuracy": 0.5
}
outer_fold_list.append(outer_fold)
best_config_outer_folds = my_pipe_optimizer.get_optimum_config_outer_folds(
outer_fold_list
)
self.assertEqual(
best_config_outer_folds.best_config_score.validation.metrics[
"balanced_accuracy"
],
0.99,
)
self.assertIs(best_config_outer_folds, outer_fold_list[5].best_config)
