def _calculate_results(
permutation_id,
save_to_db=True,
mongodb_path="mongodb://trap-umbriel:27017/photon_results"):
logger.info("Calculating permutation test results")
try:
mother_permutation = PermutationTest.find_reference(
mongodb_path, permutation_id)
# mother_permutation = MDBHyperpipe.objects.raw({'permutation_id': PermutationTest.get_mother_permutation_id(permutation_id),
# 'computation_completed': True}).first()
except DoesNotExist:
return None
else:
all_permutations = list(
MDBHyperpipe.objects.raw({
'permutation_id': permutation_id,
'computation_completed': True
}).project({'metrics_test': 1}))
# all_permutations = MDBHyperpipe.objects.raw({'permutation_id': permutation_id,
# 'computation_completed': True}).only('metrics_test')
number_of_permutations = len(all_permutations)
if number_of_permutations == 0:
number_of_permutations = 1
true_performances = dict([(m.metric_name, m.value)
for m in mother_permutation.metrics_test
if m.operation == "FoldOperations.MEAN"])
perm_performances = dict()
metric_list = list(
set([m.metric_name for m in mother_permutation.metrics_test]))
metrics = PermutationTest.manage_metrics(
metric_list, None,
mother_permutation.hyperpipe_info.best_config_metric)
for _, metric in metrics.items():
perm_performances[metric["name"]] = [
m.value for i in all_permutations for m in i.metrics_test
if m.metric_name == metric["name"]
and m.operation == "FoldOperations.MEAN"
]
# Calculate p-value
p = PermutationTest.calculate_p(
true_performance=true_performances,
perm_performances=perm_performances,
metrics=metrics,
n_perms=number_of_permutations)
p_text = dict()
for _, metric in metrics.items():
if p[metric['name']] == 0:
p_text[metric['name']] = "p < {}".format(
str(1 / number_of_permutations))
else:
p_text[metric['name']] = "p = {}".format(p[metric['name']])
# Print results
logger.clean_info("""
Done with permutations...
Results Permutation test
===============================================
""")
for _, metric in metrics.items():
logger.clean_info("""
Metric: {}
True Performance: {}
p Value: {}
""".format(metric['name'], true_performances[metric['name']],
p_text[metric['name']]))
if save_to_db:
# Write results to results object
if mother_permutation.permutation_test is None:
perm_results = MDBPermutationResults(
n_perms=number_of_permutations)
else:
perm_results = mother_permutation.permutation_test
perm_results.n_perms_done = number_of_permutations
results_all_metrics = list()
for _, metric in metrics.items():
perm_metrics = MDBPermutationMetrics(
metric_name=metric['name'],
p_value=p[metric['name']],
metric_value=true_performances[metric['name']])
perm_metrics.values_permutations = perm_performances[
metric['name']]
results_all_metrics.append(perm_metrics)
perm_results.metrics = results_all_metrics
mother_permutation.permutation_test = perm_results
mother_permutation.save()
if mother_permutation.permutation_test is not None:
n_perms = mother_permutation.permutation_test.n_perms
else:
# we guess?
n_perms = 1000
result = PermutationTest.PermutationResult(true_performances,
perm_performances, p,
number_of_permutations,
n_perms)
return result
def fit(self, X, y, **kwargs):
"""
Iterates over cross-validation folds and trains the pipeline, then uses it for predictions.
Calculates metrics per fold and averages them over fold.
:param X: Training and test data
:param y: Training and test targets
:returns: configuration class for result tree that monitors training and test performance
"""
# needed for testing Timeboxed Random Grid Search
# time.sleep(35)
config_item = MDBConfig()
config_item.config_dict = self.params
config_item.inner_folds = []
config_item.metrics_test = []
config_item.metrics_train = []
config_item.computation_start_time = datetime.datetime.now()
try:
# do inner cv
for idx, (inner_fold_id, inner_fold) in enumerate(
self.cross_validation_infos.inner_folds[
self.outer_fold_id].items()):
train, test = inner_fold.train_indices, inner_fold.test_indices
# split kwargs according to cross validation
train_X, train_y, kwargs_cv_train = PhotonDataHelper.split_data(
X, y, kwargs, indices=train)
test_X, test_y, kwargs_cv_test = PhotonDataHelper.split_data(
X, y, kwargs, indices=test)
new_pipe = self.pipe()
if self.cache_folder is not None and self.cache_updater is not None:
self.cache_updater(new_pipe, self.cache_folder,
inner_fold_id)
if not config_item.human_readable_config:
config_item.human_readable_config = PhotonPrintHelper.config_to_human_readable_dict(
new_pipe, self.params)
logger.clean_info(
json.dumps(config_item.human_readable_config,
indent=4,
sort_keys=True))
job_data = InnerFoldManager.InnerCVJob(
pipe=new_pipe,
config=dict(self.params),
metrics=self.optimization_infos.metrics,
callbacks=self.optimization_constraints,
train_data=InnerFoldManager.JobData(
train_X, train_y, train, kwargs_cv_train),
test_data=InnerFoldManager.JobData(test_X, test_y, test,
kwargs_cv_test),
)
# only for unparallel processing
# inform children in which inner fold we are
# self.pipe.distribute_cv_info_to_hyperpipe_children(inner_fold_counter=fold_cnt)
# self.mother_inner_fold_handle(fold_cnt)
# --> write that output in InnerFoldManager!
# logger.debug(config_item.human_readable_config)
fold_nr = idx + 1
logger.debug("calculating inner fold " + str(fold_nr) + "...")
curr_test_fold, curr_train_fold = InnerFoldManager.fit_and_score(
job_data)
logger.debug("Performance inner fold " + str(fold_nr))
print_double_metrics(
curr_train_fold.metrics,
curr_test_fold.metrics,
photon_system_log=False,
)
durations = job_data.pipe.time_monitor
self.update_config_item_with_inner_fold(
config_item=config_item,
fold_cnt=fold_nr,
curr_train_fold=curr_train_fold,
curr_test_fold=curr_test_fold,
time_monitor=durations,
feature_importances=new_pipe.feature_importances_,
)
if isinstance(self.optimization_constraints, list):
break_cv = 0
for cf in self.optimization_constraints:
if not cf.shall_continue(config_item):
logger.info(
"Skipped further cross validation after fold "
+ str(fold_nr) +
" due to performance constraints in " +
cf.metric)
break_cv += 1
break
if break_cv > 0:
break
elif self.optimization_constraints is not None:
if not self.optimization_constraints.shall_continue(
config_item):
logger.info(
"Skipped further cross validation after fold " +
str(fold_nr) +
" due to performance constraints in " + cf.metric)
break
InnerFoldManager.process_fit_results(
config_item,
self.cross_validation_infos.calculate_metrics_across_folds,
self.cross_validation_infos.calculate_metrics_per_fold,
self.optimization_infos.metrics,
)
except Exception as e:
if self.raise_error:
raise e
logger.error(e)
logger.error(traceback.format_exc())
traceback.print_exc()
if not isinstance(e, Warning):
config_item.config_failed = True
config_item.config_error = str(e)
warnings.warn("One test iteration of pipeline failed with error")
logger.debug("...done with")
logger.debug(
json.dumps(config_item.human_readable_config,
indent=4,
sort_keys=True))
config_item.computation_end_time = datetime.datetime.now()
return config_item
def fit(self, X, y=None, **kwargs):
logger.photon_system_log('')
logger.photon_system_log(
'***************************************************************************************************************'
)
logger.photon_system_log('Outer Cross validation Fold {}'.format(
self.cross_validaton_info.outer_folds[self.outer_fold_id].fold_nr))
logger.photon_system_log(
'***************************************************************************************************************'
)
self._prepare_data(X, y, **kwargs)
self._fit_dummy()
self._generate_inner_folds()
self._prepare_optimization()
outer_fold_fit_start_time = datetime.datetime.now()
self.best_metric_yet = None
self.tested_config_counter = 0
# distribute number of folds to encapsulated child hyperpipes
# self.__distribute_cv_info_to_hyperpipe_children(num_of_folds=num_folds,
# outer_fold_counter=outer_fold_counter)
if self.cross_validaton_info.calculate_metrics_per_fold:
self.fold_operation = FoldOperations.MEAN
else:
self.fold_operation = FoldOperations.RAW
self.max_nr_of_configs = ''
if hasattr(self.optimizer, 'n_configurations'):
self.max_nr_of_configs = str(self.optimizer.n_configurations)
if isinstance(self.optimizer, PhotonMasterOptimizer):
self.optimizer.optimize()
else:
# do the optimizing
for current_config in self.optimizer.ask:
self.objective_function(current_config)
logger.clean_info(
'---------------------------------------------------------------------------------------------------------------'
)
logger.info(
'Hyperparameter Optimization finished. Now finding best configuration .... '
)
print(self.tested_config_counter)
# now go on with the best config found
if self.tested_config_counter > 0:
best_config_outer_fold = self.optimization_info.get_optimum_config(
self.result_object.tested_config_list, self.fold_operation)
if not best_config_outer_fold:
raise Exception("No best config was found!")
# ... and create optimal pipeline
optimum_pipe = self.copy_pipe_fnc()
if self.cache_updater is not None:
self.cache_updater(optimum_pipe, self.cache_folder,
"fixed_fold_id")
optimum_pipe.caching = False
# set self to best config
optimum_pipe.set_params(**best_config_outer_fold.config_dict)
# Todo: set all children to best config and inform to NOT optimize again, ONLY fit
# for child_name, child_config in best_config_outer_fold_mdb.children_config_dict.items():
# if child_config:
# # in case we have a pipeline stacking we need to identify the particular subhyperpipe
# splitted_name = child_name.split('__')
# if len(splitted_name) > 1:
# stacking_element = self.optimum_pipe.named_steps[splitted_name[0]]
# pipe_element = stacking_element.elements[splitted_name[1]]
# else:
# pipe_element = self.optimum_pipe.named_steps[child_name]
# pipe_element.set_params(**child_config)
# pipe_element.is_final_fit = True
# self.__distribute_cv_info_to_hyperpipe_children(reset=True)
logger.debug(
'Fitting model with best configuration of outer fold...')
optimum_pipe.fit(self._validation_X, self._validation_y,
**self._validation_kwargs)
self.result_object.best_config = best_config_outer_fold
# save test performance
best_config_performance_mdb = MDBInnerFold()
best_config_performance_mdb.fold_nr = -99
best_config_performance_mdb.number_samples_training = self._validation_y.shape[
0]
best_config_performance_mdb.number_samples_validation = self._test_y.shape[
0]
best_config_performance_mdb.feature_importances = optimum_pipe.feature_importances_
if self.cross_validaton_info.eval_final_performance:
# Todo: generate mean and std over outer folds as well. move this items to the top
logger.info(
'Calculating best model performance on test set...')
logger.debug('...scoring test data')
test_score_mdb = InnerFoldManager.score(
optimum_pipe,
self._test_X,
self._test_y,
indices=self.cross_validaton_info.outer_folds[
self.outer_fold_id].test_indices,
metrics=self.optimization_info.metrics,
**self._test_kwargs)
logger.debug('... scoring training data')
train_score_mdb = InnerFoldManager.score(
optimum_pipe,
self._validation_X,
self._validation_y,
indices=self.cross_validaton_info.outer_folds[
self.outer_fold_id].train_indices,
metrics=self.optimization_info.metrics,
training=True,
**self._validation_kwargs)
best_config_performance_mdb.training = train_score_mdb
best_config_performance_mdb.validation = test_score_mdb
print_double_metrics(train_score_mdb.metrics,
test_score_mdb.metrics)
else:
def _copy_inner_fold_means(metric_dict):
# We copy all mean values from validation to the best config
# training
train_item_metrics = {}
for m in metric_dict:
if m.operation == str(self.fold_operation):
train_item_metrics[m.metric_name] = m.value
train_item = MDBScoreInformation()
train_item.metrics_copied_from_inner = True
train_item.metrics = train_item_metrics
return train_item
# training
best_config_performance_mdb.training = _copy_inner_fold_means(
best_config_outer_fold.metrics_train)
# validation
best_config_performance_mdb.validation = _copy_inner_fold_means(
best_config_outer_fold.metrics_test)
# write best config performance to best config item
self.result_object.best_config.best_config_score = best_config_performance_mdb
logger.info('Computations in outer fold {} took {} minutes.'.format(
self.cross_validaton_info.outer_folds[self.outer_fold_id].fold_nr,
(datetime.datetime.now() -
outer_fold_fit_start_time).total_seconds() / 60))
def objective_function(self, current_config):
if current_config is None:
return
logger.clean_info(
'---------------------------------------------------------------------------------------------------------------'
)
self.tested_config_counter += 1
if hasattr(self.optimizer, 'ask_for_pipe'):
pipe_ctor = self.optimizer.ask_for_pipe()
else:
pipe_ctor = self.copy_pipe_fnc
# self.__distribute_cv_info_to_hyperpipe_children(reset=True, config_counter=tested_config_counter)
hp = InnerFoldManager(pipe_ctor,
current_config,
self.optimization_info,
self.cross_validaton_info,
self.outer_fold_id,
self.constraint_objects,
cache_folder=self.cache_folder,
cache_updater=self.cache_updater)
# Test the configuration cross validated by inner_cv object
current_config_mdb = hp.fit(self._validation_X, self._validation_y,
**self._validation_kwargs)
current_config_mdb.config_nr = self.tested_config_counter
if not current_config_mdb.config_failed:
metric_train = MDBHelper.get_metric(
current_config_mdb, self.fold_operation,
self.optimization_info.best_config_metric)
metric_test = MDBHelper.get_metric(
current_config_mdb,
self.fold_operation,
self.optimization_info.best_config_metric,
train=False)
if metric_train is None or metric_test is None:
raise Exception(
"Config did not fail, but did not get any metrics either....!!?"
)
config_performance = (metric_train, metric_test)
if self.best_metric_yet is None:
self.best_metric_yet = config_performance
self.current_best_config = current_config_mdb
else:
# check if we have the next superstar around that exceeds any old performance
if self.optimization_info.maximize_metric:
if metric_test > self.best_metric_yet[1]:
self.best_metric_yet = config_performance
self.current_best_config.save_memory()
self.current_best_config = current_config_mdb
else:
current_config_mdb.save_memory()
else:
if metric_test < self.best_metric_yet[1]:
self.best_metric_yet = config_performance
self.current_best_config.save_memory()
self.current_best_config = current_config_mdb
else:
current_config_mdb.save_memory()
# Print Result for config
computation_duration = current_config_mdb.computation_end_time - current_config_mdb.computation_start_time
logger.info('Computed configuration ' +
str(self.tested_config_counter) + "/" +
self.max_nr_of_configs + " in " +
str(computation_duration))
logger.info("Performance: " +
self.optimization_info.best_config_metric +
" - Train: " + "%.4f" % config_performance[0] +
", Validation: " + "%.4f" % config_performance[1])
logger.info("Best Performance So Far: " +
self.optimization_info.best_config_metric +
" - Train: " + "%.4f" % self.best_metric_yet[0] +
", Validation: " + "%.4f" % self.best_metric_yet[1])
else:
config_performance = (-1, -1)
# Print Result for config
logger.debug('...failed:')
logger.error(current_config_mdb.config_error)
# add config to result tree
self.result_object.tested_config_list.append(current_config_mdb)
# 3. inform optimizer about performance
logger.debug(
"Telling hyperparameter optimizer about recent performance.")
if isinstance(self.optimizer, PhotonSlaveOptimizer):
self.optimizer.tell(current_config, config_performance)
logger.debug("Asking hyperparameter optimizer for new config.")
if self.optimization_info.maximize_metric:
return 1 - config_performance[1]
else:
return config_performance[1]
def fit(self, X, y=None, **kwargs):
logger.photon_system_log("")
logger.photon_system_log(
"********************************************************"
)
logger.photon_system_log(
"Outer Cross validation Fold {}".format(
self.cross_validaton_info.outer_folds[self.outer_fold_id].fold_nr
)
)
logger.photon_system_log(
"********************************************************"
)
self._prepare_data(X, y, **kwargs)
self._fit_dummy()
self._generate_inner_folds()
self._prepare_optimization()
outer_fold_fit_start_time = datetime.datetime.now()
best_metric_yet = None
tested_config_counter = 0
# distribute number of folds to encapsulated child hyperpipes
# self.__distribute_cv_info_to_hyperpipe_children(num_of_folds=num_folds,
# outer_fold_counter=outer_fold_counter)
if self.cross_validaton_info.calculate_metrics_per_fold:
fold_operation = FoldOperations.MEAN
else:
fold_operation = FoldOperations.RAW
max_nr_of_configs = ""
if hasattr(self.optimizer, "n_configurations"):
max_nr_of_configs = str(self.optimizer.n_configurations)
# do the optimizing1
for current_config in self.optimizer.ask:
if current_config is None:
continue
logger.clean_info(
"---------------------------------------------------------------------------------------------------------------"
)
tested_config_counter += 1
if hasattr(self.optimizer, "ask_for_pipe"):
pipe_ctor = self.optimizer.ask_for_pipe()
else:
pipe_ctor = self.copy_pipe_fnc
# self.__distribute_cv_info_to_hyperpipe_children(reset=True, config_counter=tested_config_counter)
hp = InnerFoldManager(
pipe_ctor,
current_config,
self.optimization_info,
self.cross_validaton_info,
self.outer_fold_id,
self.constraint_objects,
cache_folder=self.cache_folder,
cache_updater=self.cache_updater,
)
# Test the configuration cross validated by inner_cv object
current_config_mdb = hp.fit(
self._validation_X, self._validation_y, **self._validation_kwargs
)
current_config_mdb.config_nr = tested_config_counter
if not current_config_mdb.config_failed:
metric_train = MDBHelper.get_metric(
current_config_mdb,
fold_operation,
self.optimization_info.best_config_metric,
)
metric_test = MDBHelper.get_metric(
current_config_mdb,
fold_operation,
self.optimization_info.best_config_metric,
train=False,
)
if metric_train is None or metric_test is None:
raise Exception(
"Config did not fail, but did not get any metrics either....!!?"
)
config_performance = (metric_train, metric_test)
if best_metric_yet is None:
best_metric_yet = config_performance
self.current_best_config = current_config_mdb
else:
# check if we have the next superstar around that exceeds any old performance
if self.optimization_info.maximize_metric:
if metric_test > best_metric_yet[1]:
best_metric_yet = config_performance
self.current_best_config.save_memory()
self.current_best_config = current_config_mdb
else:
current_config_mdb.save_memory()
else:
if metric_test < best_metric_yet[1]:
best_metric_yet = config_performance
self.current_best_config.save_memory()
self.current_best_config = current_config_mdb
else:
current_config_mdb.save_memory()
# Print Result for config
computation_duration = (
current_config_mdb.computation_end_time
- current_config_mdb.computation_start_time
)
logger.info(
"Computed configuration "
+ str(tested_config_counter)
+ "/"
+ max_nr_of_configs
+ " in "
+ str(computation_duration)
)
logger.info(
"Performance: "
+ self.optimization_info.best_config_metric
+ " - Train: "
+ "%.4f" % config_performance[0]
+ ", Validation: "
+ "%.4f" % config_performance[1]
)
logger.info(
"Best Performance So Far: "
+ self.optimization_info.best_config_metric
+ " - Train: "
+ "%.4f" % best_metric_yet[0]
+ ", Validation: "
+ "%.4f" % best_metric_yet[1]
)
else:
config_performance = (-1, -1)
# Print Result for config
logger.debug("...failed:")
logger.error(current_config_mdb.config_error)
# add config to result tree
self.result_object.tested_config_list.append(current_config_mdb)
# 3. inform optimizer about performance
logger.debug("Telling hyperparameter optimizer about recent performance.")
self.optimizer.tell(current_config, config_performance)
logger.debug("Asking hyperparameter optimizer for new config.")
logger.clean_info(
"---------------------------------------------------------------------------------------------------------------"
)
logger.info(
"Hyperparameter Optimization finished. Now finding best configuration .... "
)
# now go on with the best config found
if tested_config_counter > 0:
best_config_outer_fold = self.optimization_info.get_optimum_config(
self.result_object.tested_config_list, fold_operation
)
if not best_config_outer_fold:
raise Exception("No best config was found!")
# ... and create optimal pipeline
optimum_pipe = self.copy_pipe_fnc()
if self.cache_updater is not None:
self.cache_updater(optimum_pipe, self.cache_folder, "fixed_fold_id")
optimum_pipe.caching = False
# set self to best config
optimum_pipe.set_params(**best_config_outer_fold.config_dict)
# Todo: set all children to best config and inform to NOT optimize again, ONLY fit
# for child_name, child_config in best_config_outer_fold_mdb.children_config_dict.items():
# if child_config:
# # in case we have a pipeline stacking we need to identify the particular subhyperpipe
# splitted_name = child_name.split('__')
# if len(splitted_name) > 1:
# stacking_element = self.optimum_pipe.named_steps[splitted_name[0]]
# pipe_element = stacking_element.elements[splitted_name[1]]
# else:
# pipe_element = self.optimum_pipe.named_steps[child_name]
# pipe_element.set_params(**child_config)
# pipe_element.is_final_fit = True
# self.__distribute_cv_info_to_hyperpipe_children(reset=True)
logger.debug("Fitting model with best configuration of outer fold...")
optimum_pipe.fit(
self._validation_X, self._validation_y, **self._validation_kwargs
)
self.result_object.best_config = best_config_outer_fold
# save test performance
best_config_performance_mdb = MDBInnerFold()
best_config_performance_mdb.fold_nr = -99
best_config_performance_mdb.number_samples_training = self._validation_y.shape[
0
]
best_config_performance_mdb.number_samples_validation = self._test_y.shape[
0
]
best_config_performance_mdb.feature_importances = (
optimum_pipe.feature_importances_
)
if self.cross_validaton_info.eval_final_performance:
# Todo: generate mean and std over outer folds as well. move this items to the top
logger.info("Calculating best model performance on test set...")
logger.debug("...scoring test data")
test_score_mdb = InnerFoldManager.score(
optimum_pipe,
self._test_X,
self._test_y,
indices=self.cross_validaton_info.outer_folds[
self.outer_fold_id
].test_indices,
metrics=self.optimization_info.metrics,
**self._test_kwargs
)
logger.debug("... scoring training data")
train_score_mdb = InnerFoldManager.score(
optimum_pipe,
self._validation_X,
self._validation_y,
indices=self.cross_validaton_info.outer_folds[
self.outer_fold_id
].train_indices,
metrics=self.optimization_info.metrics,
training=True,
**self._validation_kwargs
)
best_config_performance_mdb.training = train_score_mdb
best_config_performance_mdb.validation = test_score_mdb
print_double_metrics(train_score_mdb.metrics, test_score_mdb.metrics)
else:
def _copy_inner_fold_means(metric_dict):
# We copy all mean values from validation to the best config
# training
train_item_metrics = {}
for m in metric_dict:
if m.operation == str(fold_operation):
train_item_metrics[m.metric_name] = m.value
train_item = MDBScoreInformation()
train_item.metrics_copied_from_inner = True
train_item.metrics = train_item_metrics
return train_item
# training
best_config_performance_mdb.training = _copy_inner_fold_means(
best_config_outer_fold.metrics_train
)
# validation
best_config_performance_mdb.validation = _copy_inner_fold_means(
best_config_outer_fold.metrics_test
)
# write best config performance to best config item
self.result_object.best_config.best_config_score = (
best_config_performance_mdb
)
logger.info(
"Computations in outer fold {} took {} minutes.".format(
self.cross_validaton_info.outer_folds[self.outer_fold_id].fold_nr,
(datetime.datetime.now() - outer_fold_fit_start_time).total_seconds()
/ 60,
)
)