-GPyOpt 
-HyperOpt 
-HpBandSter 
-ConfigSpace

- SVM
- MLP
- CnnVanilla
- Resnet

- grid_search
- random_search      # 2 possible variants : (GP, GP_MCMC), 2 possible acquisition functions : (EI, MPI)
- gaussian_process
- tpe
- annealing
- hyperband
- BOHB

# Append path of module to sys and import module
module_path = os.path.dirname(os.getcwd())
sys.path.append(module_path)

# We generate data for our tests and global variables for all tests
x_train, t_train, x_test, t_test = load_breast_cancer_dataset(random_state=42)
dataset = 'Breast_Cancer_Wisconsin'
train_size = len(x_train)
nb_cross_validation = 4
nb_evals = 250

# We initialize an MLP with default hyper-parameters and 4 hidden layers of 20 neurons to classify
# our data and test its performance on both training and test data sets
mlp = MLP(hidden_layers_number=4, layers_size=20, max_iter=1000)
mlp.fit(x_train, t_train)

# We set the experiment title and save the path to save the results
experiment_title = 'BreastCancerClassification'
results_path = os.path.join(os.path.dirname(module_path), 'Results')

# We initialize a tuner with the standard GP method and set our search space
GP_tuner = HPtuner(mlp, 'gaussian_process')
GP_tuner.set_search_space({'alpha': ContinuousDomain(-8, 0, log_scaled=True),
                          'learning_rate_init': ContinuousDomain(-8, 0, log_scaled=True),
                          'batch_size': DiscreteDomain(list(linspace(50, 500, 10, dtype=int))),
                          'hidden_layers_number': DiscreteDomain(range(1, 21)),
                          'layers_size': DiscreteDomain(range(20, 101))})

# We execute the tuning using default parameters for GP
# ('GP' as method type, 5 initial points to evaluate before the beginning and 'EI' acquisition)
GP_results = GP_tuner.tune(x_train, t_train, n_evals=nb_evals, nb_cross_validation=nb_cross_validation)

# We save the results
GP_results.save_all_results(results_path, experiment_title, dataset,
                            train_size, mlp.score(x_test, t_test))