def pickup(self):
print('Entering ensemble pickup for dataset:', self.data_set_name)
outer_cv = 0
outer_folds = self._get_outer_folds(outer_cv)
for training_outer_index, testing_index in outer_folds.split(
get_input_variables(self.samples).values,
get_target_variable(self.samples).values):
print('\tIndex of outer fold:', outer_cv)
training_outer, testing = pd.DataFrame(
self.samples.values[training_outer_index]), pd.DataFrame(
self.samples.values[testing_index])
algorithm = self.best_result[outer_cv]['best_overall_algorithm']
configuration = self.best_result[outer_cv][
'best_overall_configuration']
self._run_ensembles(outer_cv, algorithm.get_corresponding_algo(),
configuration, training_outer, testing,
self.metric)
outer_cv += 1
benchmark_to_pickle(self)
print('Leaving ensemble pickup for dataset:', self.data_set_name)
def __init__(self,
data_set_name,
metric=RootMeanSquaredError,
models=None,
ensembles=None,
benchmark_id=None):
"""Initializes benchmark environment."""
self.benchmark_id = benchmark_id
self.data_set_name = data_set_name
# Creates file name as combination of data set name and and date.
self.file_name = self.data_set_name + "_" + self.benchmark_id + "__" + _now.strftime(
"%Y_%m_%d__%H_%M_%S")
# Loads samples into object.
# self.samples = [load_samples(data_set_name, index) for index in range(10)]
# self.samples = [load_samples_no_val(data_set_name, index) for index in range(10)] # changed from 30 , change back at the end
self.samples = load_standardized_samples(data_set_name)
self.metric = metric
self.ensembles = ensembles
self.models = models
# If data set is classification problem, remove regression models. Else, vice versa.
if is_classification(
self.samples): # original self.samples[0][0] new self.samples
self.classification = True
if 'mlpr_lbfgs' in self.models.keys():
del self.models['mlpr_lbfgs']
if 'mlpr_adam' in self.models.keys():
del self.models['mlpr_adam']
if 'mlpr_sgd' in self.models.keys():
del self.models['mlpr_sgd']
else:
self.classification = False
if 'mlpc_lbfgs' in self.models.keys():
del self.models['mlpc_lbfgs']
if 'mlpc_adam' in self.models.keys():
del self.models['mlpc_adam']
if 'mlpc_sgd' in self.models.keys():
del self.models['mlpc_sgd']
# if models = MLP, remove Random Independent Weighting
if self.ensembles != None:
if 'mlpc_lbfgs' in self.models.keys(
) or 'mlpr_lbfgs' in self.models.keys():
if 'riw' in self.ensembles.keys():
del self.ensembles['riw']
# Create results dictionary with models under study.
self.results = {
k: [None for i in range(_OUTER_FOLDS)]
for k in self.models.keys()
}
if self.ensembles != None:
self.results_ensemble = {
ensemble: [None for i in range(_OUTER_FOLDS)]
for ensemble in self.ensembles.keys()
}
self.best_result = [None for i in range(_OUTER_FOLDS)]
# Serialize benchmark environment.
benchmark_to_pickle(self)
def run(self):
"""Runs benchmark study, where it evaluates every algorithms on every sample set."""
i = 0
for training, validation, testing in self.samples:
for key, value in self.models.items():
# If evaluation for key, iteration pair already exists, skip this pair.
if not self.results[key][i]:
self.results[key][i] = self._evaluate_algorithm(
algorithm=value['algorithms'],
configurations=value['configurations'],
training_set=training,
validation_set=validation,
testing_set=testing,
metric=self.metric)
# Serialize benchmark.
benchmark_to_pickle(self)
i += 1
def run_nested_cv(self):
""" runs benchmark study on a nested cross-validation environment """
#=======================================================================
# print('self.learning_metric =', self.learning_metric)
# print('self.selection_metric =', self.selection_metric)
#=======================================================================
""" N configuration for each method, trained on all data, selected from the same data """
print('Entering training benchmark for dataset:', self.dataset_name)
training_data = pd.DataFrame(self.samples.values)
for outer_cv in range(_OUTER_FOLDS):
print('\n\tIndex of outer fold:', outer_cv)
for key in self.models.keys():
print('\t\tAlgorithm with key:', key)
if not self.results[key][outer_cv]:
if self.classification:
best_training_value = float('-Inf')
else:
best_training_value = float('-Inf') if self.selection_metric.greater_is_better else float('Inf')
training_value_list = list()
for configuration in range(self.models[key]['max_combinations']):
print('\n\t\t\tIndex of algorithm configuration:', len(training_value_list))
if(len(self.models[key]['algorithms'])) > 1:
option = randint(0, 2)
algorithm = self.models[key]['algorithms'][option]
config = self.models[key]['configuration_method'](option)
else:
algorithm = self.models[key]['algorithms'][0]
#===================================================
# if (key == 'mlpc_sgd' or key == 'mlpc_adam' or key == 'mlpr_sgd' or key == 'mlpr_adam'):
#===================================================
if key.startswith('mlp'):
# version from 01-22
# config = self.models[key]['configuration_method'](self.get_dataset_size(training_outer))
# version from 01-25
batch_size = int(training_data.shape[0])
# batch_size = int(training_outer.shape[0] / _INNER_FOLDS) * 2
config = self.models[key]['configuration_method'](batch_size)
else:
config = self.models[key]['configuration_method']()
if key.startswith('mlp'):
config['max_iter'] = DEFAULT_NUMBER_OF_ITERATIONS
else:
config['stopping_criterion'] = MaxGenerationsCriterion(DEFAULT_NUMBER_OF_ITERATIONS)
results = self._evaluate_algorithm(algorithm=algorithm, configurations=config,
training_set=training_data, validation_set=None, testing_set=training_data, metric=self.learning_metric)
if self.classification:
training_value = results['training_accuracy']
else:
training_value = results['training_value']
if self.classification:
print("\t\t\tAUROC training: %.3f" % (training_value))
else:
print("\t\t\tRMSE training: %.3f" % (training_value))
if self.classification:
if training_value > best_training_value:
#===============================================
# print('\n\t\t\t\t\tClassification: %.3f is better than %.3f\n' % (training_value, best_training_value))
#===============================================
best_algorithm = algorithm
best_key = key
best_configuration = config
best_training_value = training_value
self.results[key][outer_cv] = results
self.results[key][outer_cv]['best_configuration'] = best_configuration
self.results[key][outer_cv]['avg_inner_validation_error'] = best_training_value
self.results[key][outer_cv]['avg_inner_training_error'] = best_training_value
best_overall_algorithm = best_algorithm
best_overall_configuration = best_configuration
best_overall_key = best_key
self.best_result[outer_cv] = self.results[key][outer_cv]
self.best_result[outer_cv]['best_overall_algorithm'] = best_overall_algorithm
self.best_result[outer_cv]['best_overall_configuration'] = best_overall_configuration
self.best_result[outer_cv]['best_overall_key'] = best_overall_key
#===================================================
# else:
# print('\n\t\t\t\t\tClassification: %.3f is worse (!) than %.3f\n' % (training_value, best_training_value))
#===================================================
else:
if is_better(training_value, best_training_value, self.selection_metric):
#===============================================
# print('\n\t\t\t\t\tRegression: %.3f is better than %.3f\n' % (training_value, best_training_value))
#===============================================
best_algorithm = algorithm
best_key = key
best_configuration = config
best_training_value = training_value
self.results[key][outer_cv] = results
self.results[key][outer_cv]['best_configuration'] = best_configuration
self.results[key][outer_cv]['avg_inner_validation_error'] = best_training_value
self.results[key][outer_cv]['avg_inner_training_error'] = best_training_value
best_overall_algorithm = best_algorithm
best_overall_configuration = best_configuration
best_overall_key = best_key
self.best_result[outer_cv] = self.results[key][outer_cv]
self.best_result[outer_cv]['best_overall_algorithm'] = best_overall_algorithm
self.best_result[outer_cv]['best_overall_configuration'] = best_overall_configuration
self.best_result[outer_cv]['best_overall_key'] = best_overall_key
#===================================================
# else:
# print('\n\t\t\t\t\tRegression: %.3f is worse (!) than %.3f\n' % (training_value, best_training_value))
#===================================================
training_value_list.append((configuration, training_value))
if self.classification:
print("\n\t\tAUROC training: %.3f" % (self.results[key][outer_cv]['training_accuracy']))
else:
print("\n\t\tRMSE training: %.3f" % (self.results[key][outer_cv]['training_value']))
# Serialize benchmark
benchmark_to_pickle(self)
print('Leaving training benchmark for dataset:', self.dataset_name)
def __init__(self,
dataset_name,
learning_metric=None,
selection_metric=None,
models=None,
ensembles=None,
benchmark_id=None,
file_path=None):
"""Initializes benchmark environment."""
self.benchmark_id = benchmark_id
self.dataset_name = dataset_name
# Creates file name as combination of dataset name and and date
self.file_name = self.dataset_name + "_" + self.benchmark_id + "__" + _now.strftime(
"%Y_%m_%d__%H_%M_%S")
# Loads samples into object
self.samples = load_standardized_samples(dataset_name, file_path)
self.ensembles = ensembles
self.models = models
# todo make it more general (small fix can be done later ...)
if self.dataset_name == 'data_batch_1':
print(self.samples.keys())
b = self.samples[b'labels']
self.samples = pd.DataFrame(self.samples[b'data'])
self.samples = self.samples / 255
self.samples[3072] = b
self.samples = self.samples.head(
1000) # change it to get all data not sample
print(self.samples.shape)
# If dataset is classification problem, remove regression models. Else, vice versa.
if is_binary(
self.samples): # original self.samples[0][0] new self.samples
self.classification = True
self.binary = True
if learning_metric != None:
self.learning_metric = learning_metric
else:
self.learning_metric = RootMeanSquaredError
if selection_metric != None:
self.selection_metric = selection_metric
else:
self.selection_metric = AUROC
if 'mlpr_lbfgs' in self.models.keys():
del self.models['mlpr_lbfgs']
if 'mlpr_adam' in self.models.keys():
del self.models['mlpr_adam']
if 'mlpr_sgd' in self.models.keys():
del self.models['mlpr_sgd']
elif is_classification(self.samples):
self.classification = True
self.binary = False
if learning_metric != None:
self.learning_metric = learning_metric
else:
self.learning_metric = RootMeanSquaredError
if selection_metric != None:
self.selection_metric = selection_metric
else:
self.selection_metric = AUROC
else:
self.classification = False
if learning_metric != None:
self.learning_metric = learning_metric
else:
self.learning_metric = RootMeanSquaredError
if selection_metric != None:
self.selection_metric = selection_metric
else:
self.selection_metric = RootMeanSquaredError
if 'mlpc_lbfgs' in self.models.keys():
del self.models['mlpc_lbfgs']
if 'mlpc_adam' in self.models.keys():
del self.models['mlpc_adam']
if 'mlpc_sgd' in self.models.keys():
del self.models['mlpc_sgd']
# if models = MLP, remove Random Independent Weighting
if self.ensembles != None:
if 'mlpc_lbfgs' in self.models.keys(
) or 'mlpr_lbfgs' in self.models.keys():
if 'riw' in self.ensembles.keys():
del self.ensembles['riw']
# Create results dictionary with models under study.
self.results = {
k: [None for i in range(_OUTER_FOLDS)]
for k in self.models.keys()
}
if self.ensembles != None:
self.results_ensemble = {
ensemble: [None for i in range(_OUTER_FOLDS)]
for ensemble in self.ensembles.keys()
}
self.best_result = [None for i in range(_OUTER_FOLDS)]
# Serialize benchmark environment.
benchmark_to_pickle(self)
def run_nested_cv(self):
""" runs benchmark study on a nested cross-validation environment """
#=======================================================================
# print('self.learning_metric =', self.learning_metric)
# print('self.selection_metric =', self.selection_metric)
#=======================================================================
print('Entering run_nested_cv for dataset:', self.dataset_name)
outer_cv = 0
outer_folds = self._get_outer_folds(outer_cv)
for training_outer_index, testing_index in outer_folds.split(
get_input_variables(self.samples).values,
get_target_variable(self.samples).values):
print('\n\tIndex of outer fold:', outer_cv)
training_outer, testing = pd.DataFrame(
self.samples.values[training_outer_index]), pd.DataFrame(
self.samples.values[testing_index])
if self.classification:
best_overall_validation_value = float('-Inf')
else:
best_overall_validation_value = float(
'-Inf'
) if self.selection_metric.greater_is_better else float('Inf')
for key in self.models.keys():
print('\t\tAlgorithm with key:', key)
if not self.results[key][outer_cv]:
if self.classification:
best_validation_value = float('-Inf')
else:
best_validation_value = float(
'-Inf'
) if self.selection_metric.greater_is_better else float(
'Inf')
validation_value_list = list()
for configuration in range(
self.models[key]['max_combinations']):
print('\n\t\t\tIndex of algorithm configuration:',
len(validation_value_list))
if (len(self.models[key]['algorithms'])) > 1:
option = randint(0, 2)
algorithm = self.models[key]['algorithms'][option]
config = self.models[key]['configuration_method'](
option)
else:
algorithm = self.models[key]['algorithms'][0]
#===================================================
# if (key == 'mlpc_sgd' or key == 'mlpc_adam' or key == 'mlpr_sgd' or key == 'mlpr_adam'):
#===================================================
if key.startswith('mlp'):
# version from 01-22
# config = self.models[key]['configuration_method'](self.get_dataset_size(training_outer))
# version from 01-25
batch_size = int(training_outer.shape[0] /
_INNER_FOLDS)
# batch_size = int(training_outer.shape[0] / _INNER_FOLDS) * 2
config = self.models[key][
'configuration_method'](batch_size)
else:
config = self.models[key][
'configuration_method']()
inner_folds = self._get_inner_folds(outer_cv)
tmp_valid_training_values_list = list()
for training_inner_index, validation_index in inner_folds.split(
get_input_variables(training_outer).values,
get_target_variable(training_outer).values):
print('\t\t\t\tIndex of inner fold:',
len(tmp_valid_training_values_list))
training_inner, validation = pd.DataFrame(
training_outer.values[training_inner_index]
), pd.DataFrame(
training_outer.values[validation_index])
results = self._evaluate_algorithm(
algorithm=algorithm,
configurations=config,
training_set=training_inner,
validation_set=None,
testing_set=validation,
metric=self.learning_metric)
# print('results[testing_value] =', results['testing_value'], ', results[training_value] =', results['training_value'])
if self.classification:
tmp_valid_training_values_list.append(
(results['testing_accuracy'],
results['training_accuracy']))
else:
tmp_valid_training_values_list.append(
(results['testing_value'],
results['training_value']))
# Calculate average validation value and check if the current value is better than the best one
average_validation_value = mean(
tmp_valid_training_values_list, axis=0)[0]
average_training_value = mean(
tmp_valid_training_values_list, axis=0)[1]
if self.classification:
print(
"\t\t\tAverage AUROC training vs. validation: %.3f vs. %.3f"
% (average_training_value,
average_validation_value))
else:
print(
"\t\t\tAverage RMSE training vs. validation: %.3f vs. %.3f"
% (average_training_value,
average_validation_value))
if self.classification:
if average_validation_value > best_validation_value:
#===============================================
# print('\n\t\t\t\t\tClassification: %.3f is better than %.3f\n' % (average_validation_value, best_validation_value))
#===============================================
best_algorithm = algorithm
best_key = key
best_configuration = config
best_validation_value = average_validation_value
best_training_value = average_training_value
#===================================================
# else:
# print('\n\t\t\t\t\tClassification: %.3f is worse (!) than %.3f\n' % (average_validation_value, best_validation_value))
#===================================================
else:
if is_better(average_validation_value,
best_validation_value,
self.selection_metric):
#===============================================
# print('\n\t\t\t\t\tRegression: %.3f is better than %.3f\n' % (average_validation_value, best_validation_value))
#===============================================
best_algorithm = algorithm
best_key = key
best_configuration = config
best_validation_value = average_validation_value
best_training_value = average_training_value
#===================================================
# else:
# print('\n\t\t\t\t\tRegression: %.3f is worse (!) than %.3f\n' % (average_validation_value, best_validation_value))
#===================================================
# Add configuration and validation error to validation error list.
validation_value_list.append(
(configuration, average_validation_value))
""" all allowed configurations assessed of a given variant/algorithm/method (key) """
print(
'\n\t\tEvaluating best configuration in outer fold with index',
outer_cv)
self.results[key][outer_cv] = self._evaluate_algorithm(
algorithm=best_algorithm,
configurations=best_configuration,
training_set=training_outer,
validation_set=None,
testing_set=testing,
metric=self.learning_metric)
self.results[key][outer_cv][
'best_configuration'] = best_configuration
self.results[key][outer_cv][
'avg_inner_validation_error'] = best_validation_value
self.results[key][outer_cv][
'avg_inner_training_error'] = best_training_value
if self.classification:
self.results[key][outer_cv][
'avg_inner_validation_accuracy'] = best_validation_value
self.results[key][outer_cv][
'avg_inner_training_accuracy'] = best_training_value
if self.classification:
print(
"\n\t\tAUROC training vs. test: %.3f vs. %.3f" %
(self.results[key][outer_cv]['training_accuracy'],
self.results[key][outer_cv]['testing_accuracy']))
#=======================================================
# print("\n\t\tAlgorithm %s, AUROC training vs. test: %.3f vs. %.3f" % (key, self.results[key][outer_cv]['training_accuracy'], self.results[key][outer_cv]['testing_accuracy']))
#=======================================================
else:
print("\n\t\tRMSE training vs. test: %.3f vs. %.3f" %
(self.results[key][outer_cv]['training_value'],
self.results[key][outer_cv]['testing_value']))
#=======================================================
# print("\n\t\tAlgorithm %s, RMSE training vs. test: %.3f vs. %.3f" % (key, self.results[key][outer_cv]['training_value'], self.results[key][outer_cv]['testing_value']))
#=======================================================
best_overall_algorithm = best_algorithm
best_overall_configuration = best_configuration
best_overall_key = best_key
self.best_result[outer_cv] = self.results[key][outer_cv]
self.best_result[outer_cv][
'best_overall_algorithm'] = best_overall_algorithm
self.best_result[outer_cv][
'best_overall_configuration'] = best_overall_configuration
self.best_result[outer_cv][
'best_overall_key'] = best_overall_key
# # Serialize benchmark
# benchmark_to_pickle(self)
outer_cv += 1
# Serialize benchmark
benchmark_to_pickle(self)
print('Leaving run_nested_cv for dataset:', self.dataset_name)
def run_nested_cv(self):
""" runs benchmark study on a nested cross-validation environment """
print('Entering run_nested_cv for dataset:', self.data_set_name)
outer_cv = 0
outer_folds = self._get_outer_folds(outer_cv)
for training_outer_index, testing_index in outer_folds.split(
get_input_variables(self.samples).values,
get_target_variable(self.samples).values):
print('\tIndex of outer fold:', outer_cv)
training_outer, testing = pd.DataFrame(
self.samples.values[training_outer_index]), pd.DataFrame(
self.samples.values[testing_index])
if self.classification:
best_overall_validation_value = float('-Inf')
else:
best_overall_validation_value = float(
'-Inf') if self.metric.greater_is_better else float('Inf')
for key in self.models.keys():
print('\t\tAlgorithm with key:', key)
if not self.results[key][outer_cv]:
if self.classification:
best_validation_value = float('-Inf')
else:
best_validation_value = float(
'-Inf'
) if self.metric.greater_is_better else float('Inf')
validation_value_list = list()
for configuration in range(
self.models[key]['max_combinations']):
print('\t\t\tIndex of algorithm configuration:',
len(validation_value_list))
if (len(self.models[key]['algorithms'])) > 1:
option = randint(0, 2)
algorithm = self.models[key]['algorithms'][option]
config = self.models[key]['configuration_method'](
option)
else:
algorithm = self.models[key]['algorithms'][0]
if (key == 'mlpc_sgd' or key == 'mlpc_adam'
or key == 'mlpr_sgd'
or key == 'mlpr_adam'):
# version from 01-22
# config = self.models[key]['configuration_method'](self.get_data_set_size(training_outer))
# version from 01-25
batch_size = int(training_outer.shape[0] /
_INNER_FOLDS)
# batch_size = int(training_outer.shape[0] / _INNER_FOLDS) * 2
config = self.models[key][
'configuration_method'](batch_size)
else:
config = self.models[key][
'configuration_method']()
inner_folds = self._get_inner_folds(outer_cv)
tmp_valid_training_values_list = list()
for training_inner_index, validation_index in inner_folds.split(
get_input_variables(training_outer).values,
get_target_variable(training_outer).values):
print('\t\t\t\tIndex of inner fold:',
len(tmp_valid_training_values_list))
training_inner, validation = pd.DataFrame(
training_outer.values[training_inner_index]
), pd.DataFrame(
training_outer.values[validation_index])
results = self._evaluate_algorithm(
algorithm=algorithm,
configurations=config,
training_set=training_inner,
validation_set=None,
testing_set=validation,
metric=self.metric)
# print('results[testing_value] =', results['testing_value'], ', results[training_value] =', results['training_value'])
if self.classification:
tmp_valid_training_values_list.append(
(results['testing_accuracy'],
results['training_accuracy']))
else:
tmp_valid_training_values_list.append(
(results['testing_value'],
results['training_value']))
# Calculate average validation value and check if the current value is better than the best one
average_validation_value = mean(
tmp_valid_training_values_list, axis=0)[0]
average_training_value = mean(
tmp_valid_training_values_list, axis=0)[1]
if self.classification:
if average_validation_value > best_validation_value:
best_algorithm = algorithm
best_key = key
best_configuration = config
best_validation_value = average_validation_value
best_training_value = average_training_value
else:
if is_better(average_validation_value,
best_validation_value, self.metric):
best_algorithm = algorithm
best_key = key
best_configuration = config
best_validation_value = average_validation_value
best_training_value = average_training_value
# Add configuration and validation error to validation error list.
validation_value_list.append(
(configuration, average_validation_value))
self.results[key][outer_cv] = self._evaluate_algorithm(
algorithm=best_algorithm,
configurations=best_configuration,
training_set=training_outer,
validation_set=None,
testing_set=testing,
metric=self.metric)
self.results[key][outer_cv][
'best_configuration'] = best_configuration
self.results[key][outer_cv][
'avg_inner_validation_error'] = best_validation_value
self.results[key][outer_cv][
'avg_inner_training_error'] = best_training_value
if self.classification:
self.results[key][outer_cv][
'avg_inner_validation_accuracy'] = best_validation_value
self.results[key][outer_cv][
'avg_inner_training_accuracy'] = best_training_value
# # Serialize benchmark
# benchmark_to_pickle(self)
if self.classification:
if best_validation_value > best_overall_validation_value:
best_overall_key = best_key
best_overall_algorithm = best_algorithm
best_overall_configuration = best_configuration
best_overall_validation_value = best_validation_value
else:
if is_better(best_validation_value,
best_overall_validation_value,
self.metric):
best_overall_key = best_key
best_overall_algorithm = best_algorithm
best_overall_configuration = best_configuration
best_overall_validation_value = best_validation_value
print(
'\tBest overall configuration found for outer fold with index',
outer_cv)
self.best_result[outer_cv] = self._evaluate_algorithm(
algorithm=best_overall_algorithm,
configurations=best_overall_configuration,
training_set=training_outer,
validation_set=None,
testing_set=testing,
metric=self.metric)
self.best_result[outer_cv][
'best_overall_algorithm'] = best_overall_algorithm
self.best_result[outer_cv][
'best_overall_configuration'] = best_overall_configuration
self.best_result[outer_cv]['best_overall_key'] = best_overall_key
if self.ensembles != None:
print('\tCreating ensembles')
self._run_ensembles(
outer_cv, best_overall_algorithm.get_corresponding_algo(),
best_overall_configuration, training_outer, testing,
self.metric)
else:
print('\tNo ensembles to create')
outer_cv += 1
# Serialize benchmark
benchmark_to_pickle(self)
print('Leaving run_nested_cv for dataset:', self.data_set_name)
def __init__(self, dataset_name, learning_metric=None, selection_metric=None, models=None, ensembles=None, benchmark_id=None):
"""Initializes benchmark environment."""
self.benchmark_id = benchmark_id
self.data_set_name = dataset_name
# Creates file name as combination of dataset name and and date # this is the benchmark name
self.file_name = 'c_' + self.data_set_name + "_" + self.benchmark_id + "__" + _now.strftime("%Y_%m_%d__%H_%M_%S")
self.samples = load_pmlb_samples(dataset_name)
self.ensembles = ensembles
self.models = models
# If dataset is classification problem, remove regression models. Else, vice versa.
if is_classification(self.samples): # original self.samples[0][0] new self.samples
self.classification = True
if learning_metric != None:
self.learning_metric = learning_metric
else:
self.learning_metric = RootMeanSquaredError
if selection_metric != None:
self.selection_metric = selection_metric
else:
self.selection_metric = AUROC
if 'mlpr_lbfgs' in self.models.keys():
del self.models['mlpr_lbfgs']
if 'mlpr_adam' in self.models.keys():
del self.models['mlpr_adam']
if 'mlpr_sgd' in self.models.keys():
del self.models['mlpr_sgd']
else:
self.classification = False
if learning_metric != None:
self.learning_metric = learning_metric
else:
self.learning_metric = RootMeanSquaredError
if selection_metric != None:
self.selection_metric = selection_metric
else:
self.selection_metric = RootMeanSquaredError
if 'mlpc_lbfgs' in self.models.keys():
del self.models['mlpc_lbfgs']
if 'mlpc_adam' in self.models.keys():
del self.models['mlpc_adam']
if 'mlpc_sgd' in self.models.keys():
del self.models['mlpc_sgd']
# if models = MLP, remove Random Independent Weighting
if self.ensembles != None:
if 'mlpc_lbfgs' in self.models.keys() or 'mlpr_lbfgs' in self.models.keys():
if 'riw' in self.ensembles.keys():
del self.ensembles['riw']
# Create results dictionary with models under study.
self.results = {k: [None for i in range(_OUTER_FOLDS)] for k in self.models.keys()}
if self.ensembles != None:
self.results_ensemble = {ensemble: [None for i in range(_OUTER_FOLDS)] for ensemble in self.ensembles.keys()}
self.best_result = [None for i in range(_OUTER_FOLDS)]
# Serialize benchmark environment.
benchmark_to_pickle(self)