def test_save_feature_importances(self):
test_pipe = InnerFoldManager(self.pipe.copy_me, self.config,
self.optimization, self.cross_validation,
self.outer_fold_id)
# we expect the feature importances to be of length 5 because the input is through the PCA reduced to 5 dimensions
output_config = test_pipe.fit(self.X, self.y)
for inner_fold in output_config.inner_folds:
self.assertEqual(len(inner_fold.feature_importances[0]), 5)
def test_fit_against_sklearn(self):
test_pipe = InnerFoldManager(
self.pipe.copy_me,
self.config,
self.optimization,
self.cross_validation,
self.outer_fold_id,
)
photon_results_config_item = test_pipe.fit(self.X, self.y)
self.assertIsNotNone(photon_results_config_item.computation_start_time)
self.assertIsNotNone(photon_results_config_item.computation_end_time)
# now sklearn.
sklearn_pipe = Pipeline([
("StandardScaler", StandardScaler()),
("PCA", PCA()),
("RidgeClassifier", RidgeClassifier()),
])
sklearn_pipe.set_params(**self.config)
for fold_obj in self.cross_validation.inner_folds[
self.outer_fold_id].values():
train_X, test_X = (
self.X[fold_obj.train_indices],
self.X[fold_obj.test_indices],
)
train_y, test_y = (
self.y[fold_obj.train_indices],
self.y[fold_obj.test_indices],
)
sklearn_pipe.fit(train_X, train_y)
sklearn_predictions = sklearn_pipe.predict(test_X)
sklearn_feature_importances = sklearn_pipe.named_steps[
"RidgeClassifier"].coef_
photon_test_results = photon_results_config_item.inner_folds[
fold_obj.fold_nr - 1].validation
self.assertTrue(
np.array_equal(sklearn_predictions,
photon_test_results.y_pred))
for fi, sklearn_feature_importance_score in enumerate(
sklearn_feature_importances[0]):
self.assertAlmostEqual(
sklearn_feature_importance_score,
photon_results_config_item.inner_folds[
fold_obj.fold_nr - 1].feature_importances[0][fi],
)
accuracy = accuracy_score(test_y, sklearn_predictions)
self.assertEqual(photon_test_results.metrics["accuracy"], accuracy)
recall = recall_score(test_y, sklearn_predictions)
self.assertEqual(photon_test_results.metrics["recall"], recall)
def _fit_dummy(self):
if self.dummy_estimator is not None:
logger.info("Running Dummy Estimator...")
try:
if isinstance(self._validation_X, np.ndarray):
if len(self._validation_X.shape) > 2:
logger.info(
"Skipping dummy estimator because of too many dimensions"
)
self.result_object.dummy_results = None
return
dummy_y = np.reshape(self._validation_y, (-1, 1))
self.dummy_estimator.fit(dummy_y, self._validation_y)
train_scores = InnerFoldManager.score(
self.dummy_estimator,
self._validation_X,
self._validation_y,
metrics=self.optimization_info.metrics,
)
# fill result tree with fold information
inner_fold = MDBInnerFold()
inner_fold.training = train_scores
if self.cross_validaton_info.eval_final_performance:
test_scores = InnerFoldManager.score(
self.dummy_estimator,
self._test_X,
self._test_y,
metrics=self.optimization_info.metrics,
)
print_metrics("DUMMY", test_scores.metrics)
inner_fold.validation = test_scores
self.result_object.dummy_results = inner_fold
# performaceConstraints: DummyEstimator
if self.constraint_objects is not None:
dummy_constraint_objs = [
opt
for opt in self.constraint_objects
if isinstance(opt, DummyPerformance)
]
if dummy_constraint_objs:
for dummy_constraint_obj in dummy_constraint_objs:
dummy_constraint_obj.set_dummy_performance(
self.result_object.dummy_results
)
return inner_fold
except Exception as e:
logger.error(e)
logger.info("Skipping dummy because of error..")
return None
else:
logger.info("Skipping dummy ..")
def test_save_predictions(self):
# assert that we have the predictions stored
test_pipe = InnerFoldManager(self.pipe.copy_me, self.config,
self.optimization, self.cross_validation,
self.outer_fold_id)
# in case we want to have metrics calculated across false, we need to temporarily store the predictions
test_pipe.optimization_infos.calculate_metrics_across_folds = True
config_item = test_pipe.fit(self.X, self.y)
for inner_fold in config_item.inner_folds:
self.assertEqual(len(inner_fold.training.y_pred),
inner_fold.number_samples_training)
self.assertEqual(len(inner_fold.validation.y_pred),
inner_fold.number_samples_validation)
def test_performance_constraints(self):
# test if the constraints are considered
# A: for a single constraint
test_pipe = InnerFoldManager(
self.pipe.copy_me,
self.config,
self.optimization,
self.cross_validation,
self.outer_fold_id,
optimization_constraints=MinimumPerformance(
"accuracy", 0.95, "first"),
)
photon_results_config_item = test_pipe.fit(self.X, self.y)
# the first fold has an accuracy of 0.874 so we expect the test_pipe to stop calculating after the first fold
# which means it has only one outer fold and
self.assertTrue(len(photon_results_config_item.inner_folds) == 1)
# B: for a list of constraints, accuracy should pass (0.874 in first fold > accuracy threshold)
# but specificity should stop the computation (0.78 in first fold < specificity threshold)
test_pipe = InnerFoldManager(
self.pipe.copy_me,
self.config,
self.optimization,
self.cross_validation,
self.outer_fold_id,
optimization_constraints=[
MinimumPerformance("accuracy", 0.85, "first"),
MinimumPerformance("specificity", 0.8, "first"),
],
)
photon_results_config_item = test_pipe.fit(self.X, self.y)
self.assertTrue(len(photon_results_config_item.inner_folds) == 1)
# C: for a list of constraints, all should pass
test_pipe = InnerFoldManager(
self.pipe.copy_me,
self.config,
self.optimization,
self.cross_validation,
self.outer_fold_id,
optimization_constraints=[
MinimumPerformance("accuracy", 0.75, "all"),
MinimumPerformance("specificity", 0.75, "all"),
],
)
photon_results_config_item = test_pipe.fit(self.X, self.y)
self.assertTrue(len(photon_results_config_item.inner_folds) == 4)
def test_raise_error(self):
# case A: raise_error = False -> we expect continuation of the computation
test_pipe = InnerFoldManager(self.pipe.copy_me,
self.config,
self.optimization,
self.cross_validation,
self.outer_fold_id,
raise_error=False)
# computing with inequal number of features and targets should result in an error
test_pipe.fit(self.X, self.y[:10])
# case B:
test_pipe.raise_error = True
with self.assertRaises(IndexError):
test_pipe.fit(self.X, self.y[:10])
def test_process_fit_results(self):
test_pipe = InnerFoldManager(
self.pipe.copy_me,
self.config,
self.optimization,
self.cross_validation,
self.outer_fold_id,
)
test_pipe.cross_validation_infos.calculate_metrics_across_folds = True
test_pipe.cross_validation_infos.calculate_metrics_per_fold = False
across_folds_config_item = test_pipe.fit(self.X, self.y)
test_pipe.cross_validation_infos.calculate_metrics_across_folds = False
test_pipe.cross_validation_infos.calculate_metrics_per_fold = True
per_fold_config_item = test_pipe.fit(self.X, self.y)
test_pipe.cross_validation_infos.calculate_metrics_across_folds = True
test_pipe.cross_validation_infos.calculate_metrics_per_fold = True
across_and_per_folds_config_item = test_pipe.fit(self.X, self.y)
def assert_fold_operations(expected_operations, returned_metric_list):
# assert that we have raw and std and mean
expected_returns = list()
for metric in self.optimization.metrics:
for operation in expected_operations:
expected_returns.append(metric + "__" + str(operation))
returned_formatted_metric_list = [
m.metric_name + "__" + str(m.operation)
for m in returned_metric_list
]
self.assertTrue(
set(expected_returns) == set(returned_formatted_metric_list))
# if we have both, then we have mean and std over the folds + three raw across folds
num_of_metrics = len(test_pipe.optimization_infos.metrics)
self.assertTrue(
len(across_and_per_folds_config_item.metrics_train) == 2 *
num_of_metrics + num_of_metrics)
self.assertTrue(
len(across_and_per_folds_config_item.metrics_test) == 2 *
num_of_metrics + num_of_metrics)
assert_fold_operations(
[FoldOperations.RAW, FoldOperations.MEAN, FoldOperations.STD],
across_and_per_folds_config_item.metrics_train,
)
assert_fold_operations(
[FoldOperations.RAW, FoldOperations.MEAN, FoldOperations.STD],
across_and_per_folds_config_item.metrics_test,
)
# if we have across folds only, then it should be 3, one for each metrics
self.assertTrue(
len(across_folds_config_item.metrics_train) == num_of_metrics)
self.assertTrue(
len(across_folds_config_item.metrics_test) == num_of_metrics)
assert_fold_operations([FoldOperations.RAW],
across_folds_config_item.metrics_train)
assert_fold_operations([FoldOperations.RAW],
across_folds_config_item.metrics_test)
# if we have per fold only, then it should be 6, one for mean and std for each of the three metrics
self.assertTrue(
len(per_fold_config_item.metrics_train) == 2 * num_of_metrics)
self.assertTrue(
len(per_fold_config_item.metrics_test) == 2 * num_of_metrics)
assert_fold_operations(
[FoldOperations.MEAN, FoldOperations.STD],
per_fold_config_item.metrics_train,
)
assert_fold_operations([FoldOperations.MEAN, FoldOperations.STD],
per_fold_config_item.metrics_test)
def objective_function(self, current_config):
if current_config is None:
return
logger.clean_info(
'---------------------------------------------------------------------------------------------------------------'
)
self.tested_config_counter += 1
if hasattr(self.optimizer, 'ask_for_pipe'):
pipe_ctor = self.optimizer.ask_for_pipe()
else:
pipe_ctor = self.copy_pipe_fnc
# self.__distribute_cv_info_to_hyperpipe_children(reset=True, config_counter=tested_config_counter)
hp = InnerFoldManager(pipe_ctor,
current_config,
self.optimization_info,
self.cross_validaton_info,
self.outer_fold_id,
self.constraint_objects,
cache_folder=self.cache_folder,
cache_updater=self.cache_updater)
# Test the configuration cross validated by inner_cv object
current_config_mdb = hp.fit(self._validation_X, self._validation_y,
**self._validation_kwargs)
current_config_mdb.config_nr = self.tested_config_counter
if not current_config_mdb.config_failed:
metric_train = MDBHelper.get_metric(
current_config_mdb, self.fold_operation,
self.optimization_info.best_config_metric)
metric_test = MDBHelper.get_metric(
current_config_mdb,
self.fold_operation,
self.optimization_info.best_config_metric,
train=False)
if metric_train is None or metric_test is None:
raise Exception(
"Config did not fail, but did not get any metrics either....!!?"
)
config_performance = (metric_train, metric_test)
if self.best_metric_yet is None:
self.best_metric_yet = config_performance
self.current_best_config = current_config_mdb
else:
# check if we have the next superstar around that exceeds any old performance
if self.optimization_info.maximize_metric:
if metric_test > self.best_metric_yet[1]:
self.best_metric_yet = config_performance
self.current_best_config.save_memory()
self.current_best_config = current_config_mdb
else:
current_config_mdb.save_memory()
else:
if metric_test < self.best_metric_yet[1]:
self.best_metric_yet = config_performance
self.current_best_config.save_memory()
self.current_best_config = current_config_mdb
else:
current_config_mdb.save_memory()
# Print Result for config
computation_duration = current_config_mdb.computation_end_time - current_config_mdb.computation_start_time
logger.info('Computed configuration ' +
str(self.tested_config_counter) + "/" +
self.max_nr_of_configs + " in " +
str(computation_duration))
logger.info("Performance: " +
self.optimization_info.best_config_metric +
" - Train: " + "%.4f" % config_performance[0] +
", Validation: " + "%.4f" % config_performance[1])
logger.info("Best Performance So Far: " +
self.optimization_info.best_config_metric +
" - Train: " + "%.4f" % self.best_metric_yet[0] +
", Validation: " + "%.4f" % self.best_metric_yet[1])
else:
config_performance = (-1, -1)
# Print Result for config
logger.debug('...failed:')
logger.error(current_config_mdb.config_error)
# add config to result tree
self.result_object.tested_config_list.append(current_config_mdb)
# 3. inform optimizer about performance
logger.debug(
"Telling hyperparameter optimizer about recent performance.")
if isinstance(self.optimizer, PhotonSlaveOptimizer):
self.optimizer.tell(current_config, config_performance)
logger.debug("Asking hyperparameter optimizer for new config.")
if self.optimization_info.maximize_metric:
return 1 - config_performance[1]
else:
return config_performance[1]
def fit(self, X, y=None, **kwargs):
logger.photon_system_log('')
logger.photon_system_log(
'***************************************************************************************************************'
)
logger.photon_system_log('Outer Cross validation Fold {}'.format(
self.cross_validaton_info.outer_folds[self.outer_fold_id].fold_nr))
logger.photon_system_log(
'***************************************************************************************************************'
)
self._prepare_data(X, y, **kwargs)
self._fit_dummy()
self._generate_inner_folds()
self._prepare_optimization()
outer_fold_fit_start_time = datetime.datetime.now()
self.best_metric_yet = None
self.tested_config_counter = 0
# distribute number of folds to encapsulated child hyperpipes
# self.__distribute_cv_info_to_hyperpipe_children(num_of_folds=num_folds,
# outer_fold_counter=outer_fold_counter)
if self.cross_validaton_info.calculate_metrics_per_fold:
self.fold_operation = FoldOperations.MEAN
else:
self.fold_operation = FoldOperations.RAW
self.max_nr_of_configs = ''
if hasattr(self.optimizer, 'n_configurations'):
self.max_nr_of_configs = str(self.optimizer.n_configurations)
if isinstance(self.optimizer, PhotonMasterOptimizer):
self.optimizer.optimize()
else:
# do the optimizing
for current_config in self.optimizer.ask:
self.objective_function(current_config)
logger.clean_info(
'---------------------------------------------------------------------------------------------------------------'
)
logger.info(
'Hyperparameter Optimization finished. Now finding best configuration .... '
)
print(self.tested_config_counter)
# now go on with the best config found
if self.tested_config_counter > 0:
best_config_outer_fold = self.optimization_info.get_optimum_config(
self.result_object.tested_config_list, self.fold_operation)
if not best_config_outer_fold:
raise Exception("No best config was found!")
# ... and create optimal pipeline
optimum_pipe = self.copy_pipe_fnc()
if self.cache_updater is not None:
self.cache_updater(optimum_pipe, self.cache_folder,
"fixed_fold_id")
optimum_pipe.caching = False
# set self to best config
optimum_pipe.set_params(**best_config_outer_fold.config_dict)
# Todo: set all children to best config and inform to NOT optimize again, ONLY fit
# for child_name, child_config in best_config_outer_fold_mdb.children_config_dict.items():
# if child_config:
# # in case we have a pipeline stacking we need to identify the particular subhyperpipe
# splitted_name = child_name.split('__')
# if len(splitted_name) > 1:
# stacking_element = self.optimum_pipe.named_steps[splitted_name[0]]
# pipe_element = stacking_element.elements[splitted_name[1]]
# else:
# pipe_element = self.optimum_pipe.named_steps[child_name]
# pipe_element.set_params(**child_config)
# pipe_element.is_final_fit = True
# self.__distribute_cv_info_to_hyperpipe_children(reset=True)
logger.debug(
'Fitting model with best configuration of outer fold...')
optimum_pipe.fit(self._validation_X, self._validation_y,
**self._validation_kwargs)
self.result_object.best_config = best_config_outer_fold
# save test performance
best_config_performance_mdb = MDBInnerFold()
best_config_performance_mdb.fold_nr = -99
best_config_performance_mdb.number_samples_training = self._validation_y.shape[
0]
best_config_performance_mdb.number_samples_validation = self._test_y.shape[
0]
best_config_performance_mdb.feature_importances = optimum_pipe.feature_importances_
if self.cross_validaton_info.eval_final_performance:
# Todo: generate mean and std over outer folds as well. move this items to the top
logger.info(
'Calculating best model performance on test set...')
logger.debug('...scoring test data')
test_score_mdb = InnerFoldManager.score(
optimum_pipe,
self._test_X,
self._test_y,
indices=self.cross_validaton_info.outer_folds[
self.outer_fold_id].test_indices,
metrics=self.optimization_info.metrics,
**self._test_kwargs)
logger.debug('... scoring training data')
train_score_mdb = InnerFoldManager.score(
optimum_pipe,
self._validation_X,
self._validation_y,
indices=self.cross_validaton_info.outer_folds[
self.outer_fold_id].train_indices,
metrics=self.optimization_info.metrics,
training=True,
**self._validation_kwargs)
best_config_performance_mdb.training = train_score_mdb
best_config_performance_mdb.validation = test_score_mdb
print_double_metrics(train_score_mdb.metrics,
test_score_mdb.metrics)
else:
def _copy_inner_fold_means(metric_dict):
# We copy all mean values from validation to the best config
# training
train_item_metrics = {}
for m in metric_dict:
if m.operation == str(self.fold_operation):
train_item_metrics[m.metric_name] = m.value
train_item = MDBScoreInformation()
train_item.metrics_copied_from_inner = True
train_item.metrics = train_item_metrics
return train_item
# training
best_config_performance_mdb.training = _copy_inner_fold_means(
best_config_outer_fold.metrics_train)
# validation
best_config_performance_mdb.validation = _copy_inner_fold_means(
best_config_outer_fold.metrics_test)
# write best config performance to best config item
self.result_object.best_config.best_config_score = best_config_performance_mdb
logger.info('Computations in outer fold {} took {} minutes.'.format(
self.cross_validaton_info.outer_folds[self.outer_fold_id].fold_nr,
(datetime.datetime.now() -
outer_fold_fit_start_time).total_seconds() / 60))
def fit(self, X, y=None, **kwargs):
logger.photon_system_log("")
logger.photon_system_log(
"********************************************************"
)
logger.photon_system_log(
"Outer Cross validation Fold {}".format(
self.cross_validaton_info.outer_folds[self.outer_fold_id].fold_nr
)
)
logger.photon_system_log(
"********************************************************"
)
self._prepare_data(X, y, **kwargs)
self._fit_dummy()
self._generate_inner_folds()
self._prepare_optimization()
outer_fold_fit_start_time = datetime.datetime.now()
best_metric_yet = None
tested_config_counter = 0
# distribute number of folds to encapsulated child hyperpipes
# self.__distribute_cv_info_to_hyperpipe_children(num_of_folds=num_folds,
# outer_fold_counter=outer_fold_counter)
if self.cross_validaton_info.calculate_metrics_per_fold:
fold_operation = FoldOperations.MEAN
else:
fold_operation = FoldOperations.RAW
max_nr_of_configs = ""
if hasattr(self.optimizer, "n_configurations"):
max_nr_of_configs = str(self.optimizer.n_configurations)
# do the optimizing1
for current_config in self.optimizer.ask:
if current_config is None:
continue
logger.clean_info(
"---------------------------------------------------------------------------------------------------------------"
)
tested_config_counter += 1
if hasattr(self.optimizer, "ask_for_pipe"):
pipe_ctor = self.optimizer.ask_for_pipe()
else:
pipe_ctor = self.copy_pipe_fnc
# self.__distribute_cv_info_to_hyperpipe_children(reset=True, config_counter=tested_config_counter)
hp = InnerFoldManager(
pipe_ctor,
current_config,
self.optimization_info,
self.cross_validaton_info,
self.outer_fold_id,
self.constraint_objects,
cache_folder=self.cache_folder,
cache_updater=self.cache_updater,
)
# Test the configuration cross validated by inner_cv object
current_config_mdb = hp.fit(
self._validation_X, self._validation_y, **self._validation_kwargs
)
current_config_mdb.config_nr = tested_config_counter
if not current_config_mdb.config_failed:
metric_train = MDBHelper.get_metric(
current_config_mdb,
fold_operation,
self.optimization_info.best_config_metric,
)
metric_test = MDBHelper.get_metric(
current_config_mdb,
fold_operation,
self.optimization_info.best_config_metric,
train=False,
)
if metric_train is None or metric_test is None:
raise Exception(
"Config did not fail, but did not get any metrics either....!!?"
)
config_performance = (metric_train, metric_test)
if best_metric_yet is None:
best_metric_yet = config_performance
self.current_best_config = current_config_mdb
else:
# check if we have the next superstar around that exceeds any old performance
if self.optimization_info.maximize_metric:
if metric_test > best_metric_yet[1]:
best_metric_yet = config_performance
self.current_best_config.save_memory()
self.current_best_config = current_config_mdb
else:
current_config_mdb.save_memory()
else:
if metric_test < best_metric_yet[1]:
best_metric_yet = config_performance
self.current_best_config.save_memory()
self.current_best_config = current_config_mdb
else:
current_config_mdb.save_memory()
# Print Result for config
computation_duration = (
current_config_mdb.computation_end_time
- current_config_mdb.computation_start_time
)
logger.info(
"Computed configuration "
+ str(tested_config_counter)
+ "/"
+ max_nr_of_configs
+ " in "
+ str(computation_duration)
)
logger.info(
"Performance: "
+ self.optimization_info.best_config_metric
+ " - Train: "
+ "%.4f" % config_performance[0]
+ ", Validation: "
+ "%.4f" % config_performance[1]
)
logger.info(
"Best Performance So Far: "
+ self.optimization_info.best_config_metric
+ " - Train: "
+ "%.4f" % best_metric_yet[0]
+ ", Validation: "
+ "%.4f" % best_metric_yet[1]
)
else:
config_performance = (-1, -1)
# Print Result for config
logger.debug("...failed:")
logger.error(current_config_mdb.config_error)
# add config to result tree
self.result_object.tested_config_list.append(current_config_mdb)
# 3. inform optimizer about performance
logger.debug("Telling hyperparameter optimizer about recent performance.")
self.optimizer.tell(current_config, config_performance)
logger.debug("Asking hyperparameter optimizer for new config.")
logger.clean_info(
"---------------------------------------------------------------------------------------------------------------"
)
logger.info(
"Hyperparameter Optimization finished. Now finding best configuration .... "
)
# now go on with the best config found
if tested_config_counter > 0:
best_config_outer_fold = self.optimization_info.get_optimum_config(
self.result_object.tested_config_list, fold_operation
)
if not best_config_outer_fold:
raise Exception("No best config was found!")
# ... and create optimal pipeline
optimum_pipe = self.copy_pipe_fnc()
if self.cache_updater is not None:
self.cache_updater(optimum_pipe, self.cache_folder, "fixed_fold_id")
optimum_pipe.caching = False
# set self to best config
optimum_pipe.set_params(**best_config_outer_fold.config_dict)
# Todo: set all children to best config and inform to NOT optimize again, ONLY fit
# for child_name, child_config in best_config_outer_fold_mdb.children_config_dict.items():
# if child_config:
# # in case we have a pipeline stacking we need to identify the particular subhyperpipe
# splitted_name = child_name.split('__')
# if len(splitted_name) > 1:
# stacking_element = self.optimum_pipe.named_steps[splitted_name[0]]
# pipe_element = stacking_element.elements[splitted_name[1]]
# else:
# pipe_element = self.optimum_pipe.named_steps[child_name]
# pipe_element.set_params(**child_config)
# pipe_element.is_final_fit = True
# self.__distribute_cv_info_to_hyperpipe_children(reset=True)
logger.debug("Fitting model with best configuration of outer fold...")
optimum_pipe.fit(
self._validation_X, self._validation_y, **self._validation_kwargs
)
self.result_object.best_config = best_config_outer_fold
# save test performance
best_config_performance_mdb = MDBInnerFold()
best_config_performance_mdb.fold_nr = -99
best_config_performance_mdb.number_samples_training = self._validation_y.shape[
0
]
best_config_performance_mdb.number_samples_validation = self._test_y.shape[
0
]
best_config_performance_mdb.feature_importances = (
optimum_pipe.feature_importances_
)
if self.cross_validaton_info.eval_final_performance:
# Todo: generate mean and std over outer folds as well. move this items to the top
logger.info("Calculating best model performance on test set...")
logger.debug("...scoring test data")
test_score_mdb = InnerFoldManager.score(
optimum_pipe,
self._test_X,
self._test_y,
indices=self.cross_validaton_info.outer_folds[
self.outer_fold_id
].test_indices,
metrics=self.optimization_info.metrics,
**self._test_kwargs
)
logger.debug("... scoring training data")
train_score_mdb = InnerFoldManager.score(
optimum_pipe,
self._validation_X,
self._validation_y,
indices=self.cross_validaton_info.outer_folds[
self.outer_fold_id
].train_indices,
metrics=self.optimization_info.metrics,
training=True,
**self._validation_kwargs
)
best_config_performance_mdb.training = train_score_mdb
best_config_performance_mdb.validation = test_score_mdb
print_double_metrics(train_score_mdb.metrics, test_score_mdb.metrics)
else:
def _copy_inner_fold_means(metric_dict):
# We copy all mean values from validation to the best config
# training
train_item_metrics = {}
for m in metric_dict:
if m.operation == str(fold_operation):
train_item_metrics[m.metric_name] = m.value
train_item = MDBScoreInformation()
train_item.metrics_copied_from_inner = True
train_item.metrics = train_item_metrics
return train_item
# training
best_config_performance_mdb.training = _copy_inner_fold_means(
best_config_outer_fold.metrics_train
)
# validation
best_config_performance_mdb.validation = _copy_inner_fold_means(
best_config_outer_fold.metrics_test
)
# write best config performance to best config item
self.result_object.best_config.best_config_score = (
best_config_performance_mdb
)
logger.info(
"Computations in outer fold {} took {} minutes.".format(
self.cross_validaton_info.outer_folds[self.outer_fold_id].fold_nr,
(datetime.datetime.now() - outer_fold_fit_start_time).total_seconds()
/ 60,
)
)