def _get_metalearning_configurations(meta_base, basename, metric,
configuration_space, task,
initial_configurations_via_metalearning,
is_sparse, watcher, logger):
task_name = 'InitialConfigurations'
watcher.start_task(task_name)
try:
metalearning_configurations = suggest_via_metalearning(
meta_base, basename, metric, task, is_sparse == 1,
initial_configurations_via_metalearning)
except Exception as e:
logger.error("Error getting metalearning configurations!")
logger.error(str(e))
logger.error(traceback.format_exc())
metalearning_configurations = []
watcher.stop_task(task_name)
return metalearning_configurations
def _get_metalearning_configurations(meta_base, basename, metric,
configuration_space,
task,
initial_configurations_via_metalearning,
is_sparse,
watcher, logger):
task_name = 'InitialConfigurations'
watcher.start_task(task_name)
try:
metalearning_configurations = suggest_via_metalearning(
meta_base, basename, metric,
task,
is_sparse == 1,
initial_configurations_via_metalearning
)
except Exception as e:
logger.error("Error getting metalearning configurations!")
logger.error(str(e))
logger.error(traceback.format_exc())
metalearning_configurations = []
watcher.stop_task(task_name)
return metalearning_configurations
def test_metalearning(self):
dataset_name_classification = 'digits'
initial_challengers_classification = {
ACC_METRIC:
"--initial-challengers \" "
"-balancing:strategy 'weighting' "
"-classifier:__choice__ 'proj_logit'",
AUC_METRIC:
"--initial-challengers \" "
"-balancing:strategy 'weighting' "
"-classifier:__choice__ 'liblinear_svc'",
BAC_METRIC:
"--initial-challengers \" "
"-balancing:strategy 'weighting' "
"-classifier:__choice__ 'proj_logit'",
F1_METRIC:
"--initial-challengers \" "
"-balancing:strategy 'weighting' "
"-classifier:__choice__ 'proj_logit'",
PAC_METRIC:
"--initial-challengers \" "
"-balancing:strategy 'none' "
"-classifier:__choice__ 'random_forest'"
}
dataset_name_regression = 'diabetes'
initial_challengers_regression = {
A_METRIC:
"--initial-challengers \" "
"-imputation:strategy 'mean' "
"-one_hot_encoding:minimum_fraction '0.01' "
"-one_hot_encoding:use_minimum_fraction 'True' "
"-preprocessor:__choice__ 'no_preprocessing' "
"-regressor:__choice__ 'random_forest'",
R2_METRIC:
"--initial-challengers \" "
"-imputation:strategy 'mean' "
"-one_hot_encoding:minimum_fraction '0.01' "
"-one_hot_encoding:use_minimum_fraction 'True' "
"-preprocessor:__choice__ 'no_preprocessing' "
"-regressor:__choice__ 'random_forest'",
}
for dataset_name, task, initial_challengers in [
(dataset_name_regression, REGRESSION,
initial_challengers_regression),
(dataset_name_classification, MULTICLASS_CLASSIFICATION,
initial_challengers_classification)
]:
for metric in initial_challengers:
configuration_space = get_configuration_space(
{
'metric': metric,
'task': task,
'is_sparse': False
},
include_preprocessors=['no_preprocessing'])
X_train, Y_train, X_test, Y_test = get_dataset(dataset_name)
categorical = [False] * X_train.shape[1]
meta_features_label = calc_meta_features(
X_train, Y_train, categorical, dataset_name, task)
meta_features_encoded_label = calc_meta_features_encoded(
X_train, Y_train, categorical, dataset_name, task)
initial_configuration_strings_for_smac = \
suggest_via_metalearning(
meta_features_label,
meta_features_encoded_label,
configuration_space, dataset_name, metric,
task, False, 1, None)
print(METRIC_TO_STRING[metric])
print(initial_configuration_strings_for_smac[0])
self.assertTrue(
initial_configuration_strings_for_smac[0].startswith(
initial_challengers[metric]))
def test_metalearning(self):
dataset_name_classification = 'digits'
initial_challengers_classification = {
ACC_METRIC: "--initial-challengers \" "
"-balancing:strategy 'weighting' "
"-classifier:__choice__ 'proj_logit'",
AUC_METRIC: "--initial-challengers \" "
"-balancing:strategy 'weighting' "
"-classifier:__choice__ 'liblinear_svc'",
BAC_METRIC: "--initial-challengers \" "
"-balancing:strategy 'weighting' "
"-classifier:__choice__ 'proj_logit'",
F1_METRIC: "--initial-challengers \" "
"-balancing:strategy 'weighting' "
"-classifier:__choice__ 'proj_logit'",
PAC_METRIC: "--initial-challengers \" "
"-balancing:strategy 'none' "
"-classifier:__choice__ 'random_forest'"
}
dataset_name_regression = 'diabetes'
initial_challengers_regression = {
A_METRIC: "--initial-challengers \" "
"-imputation:strategy 'mean' "
"-one_hot_encoding:minimum_fraction '0.01' "
"-one_hot_encoding:use_minimum_fraction 'True' "
"-preprocessor:__choice__ 'no_preprocessing' "
"-regressor:__choice__ 'random_forest'",
R2_METRIC: "--initial-challengers \" "
"-imputation:strategy 'mean' "
"-one_hot_encoding:minimum_fraction '0.01' "
"-one_hot_encoding:use_minimum_fraction 'True' "
"-preprocessor:__choice__ 'no_preprocessing' "
"-regressor:__choice__ 'random_forest'",
}
for dataset_name, task, initial_challengers in [
(dataset_name_regression, REGRESSION,
initial_challengers_regression),
(dataset_name_classification, MULTICLASS_CLASSIFICATION,
initial_challengers_classification)
]:
for metric in initial_challengers:
configuration_space = get_configuration_space(
{
'metric': metric,
'task': task,
'is_sparse': False
},
include_preprocessors=['no_preprocessing'])
X_train, Y_train, X_test, Y_test = get_dataset(dataset_name)
categorical = [False] * X_train.shape[1]
meta_features_label = calc_meta_features(
X_train, Y_train, categorical, dataset_name, task)
meta_features_encoded_label = calc_meta_features_encoded(
X_train, Y_train, categorical, dataset_name, task)
initial_configuration_strings_for_smac = \
suggest_via_metalearning(
meta_features_label,
meta_features_encoded_label,
configuration_space, dataset_name, metric,
task, False, 1, None)
print(metric)
print(initial_configuration_strings_for_smac[0])
self.assertTrue(initial_configuration_strings_for_smac[
0].startswith(initial_challengers[metric]))