def test_reverse_format(parameters, results, transforms): X, y = GPyOpt._prepare_data_for_bayes_opt(parameters, results, transforms) reversed_X = GPyOpt._reverse_to_sherpa_format(X, transforms, parameters) assert reversed_X[0] == { 'dropout': 0.1, 'lr': 1e-3, 'activation': 'tanh', 'num_hidden': 111, 'batch_size': 10 } assert reversed_X[1] == { 'dropout': 0.4, 'lr': 1e-5, 'activation': 'relu', 'num_hidden': 222, 'batch_size': 100 } assert reversed_X[2] == { 'dropout': 0.33, 'lr': 1e-2, 'activation': 'sigmoid', 'num_hidden': 288, 'batch_size': 1000 }
def test_bayesopt_batch(parameters, results, transforms): gpyopt = GPyOpt(max_concurrent=10) domain = gpyopt._initialize_domain(parameters, transforms) X, y = GPyOpt._prepare_data_for_bayes_opt(parameters, results, transforms) batch = gpyopt._generate_bayesopt_batch(domain, X, y, lower_is_better=True) assert batch.shape == (10, 5)
def test_types_are_correct(parameters, results): gpyopt = GPyOpt(max_concurrent=1) suggestion = gpyopt.get_suggestion(parameters, results, True) assert isinstance(suggestion['dropout'], float) assert isinstance(suggestion['lr'], float) assert isinstance(suggestion['num_hidden'], int) assert isinstance(suggestion['activation'], str)
def test_processing_of_initial_data_points(parameters): data_points_df = pandas.DataFrame( collections.OrderedDict([('dropout', [0.5, 0.1, 0.3]), ('lr', [1e-4, 1e-3, 1e-7]), ('activation', ['relu', 'tanh', 'sigmoid']), ('num_hidden', [101, 202, 299])])) data_points = GPyOpt._process_initial_data_points(data_points_df, parameters) assert data_points == [{ 'activation': 'relu', 'dropout': 0.5, 'lr': 0.0001, 'num_hidden': 101 }, { 'activation': 'tanh', 'dropout': 0.1, 'lr': 0.001, 'num_hidden': 202 }, { 'activation': 'sigmoid', 'dropout': 0.3, 'lr': 1e-07, 'num_hidden': 299 }] with pytest.raises(ValueError): data_points_missing = GPyOpt._process_initial_data_points( data_points_df.drop("activation", axis=1), parameters)
def test_overall(): gpyopt = GPyOpt(max_concurrent=1) parameters, results, lower_is_better = sherpa.algorithms.get_sample_results_and_params( ) for i in range(51): suggestion = gpyopt.get_suggestion( parameters, results.loc[results['Trial-ID'] < i, :], lower_is_better) print(suggestion)
def test_get_best_pred(lower_is_better, expected_best): results = pandas.DataFrame({ 'x': numpy.linspace(0, 1, 10), 'Objective': numpy.linspace(0, 1, 10), 'Status': ['COMPLETED'] * 10 }) params = [sherpa.Continuous('x', [0, 1])] algorithm = GPyOpt(num_initial_data_points=2) algorithm.get_suggestion(results=results, parameters=params, lower_is_better=lower_is_better) best_params = algorithm.get_best_pred(results=results, parameters=params, lower_is_better=lower_is_better) assert best_params['x'] == expected_best
def test_1d(): def obj_func(x): # Global maximum of 4 is at x=4 return 4. * numpy.exp(-(x - 4.)**2 / 10.) * numpy.cos(1.5 * (x - 4.))**2 parameters = [sherpa.Continuous('x1', [0., 7.])] bayesian_optimization = GPyOpt(max_concurrent=1, max_num_trials=50, model_type='GP', acquisition_type='EI') study = sherpa.Study(algorithm=bayesian_optimization, parameters=parameters, lower_is_better=False, disable_dashboard=True) for trial in study: print("Trial {}:\t{}".format(trial.id, trial.parameters)) fval = obj_func(trial.parameters['x1']) print("Function Value: {}".format(fval)) study.add_observation(trial=trial, iteration=1, objective=fval) study.finalize(trial, status='COMPLETED') rval = study.get_best_result() print(rval) assert numpy.isclose(rval['Objective'], 4., atol=0.2)
def test_bayes_opt(): def f(x, sd=1): y = (x - 3) ** 2 + 10. if sd == 0: return y else: return y + numpy.random.normal(loc=0., scale=sd, size=numpy.array(x).shape) parameters = [sherpa.Continuous('x', [1, 6])] alg = GPyOpt(max_num_trials=10) gs = SequentialTesting(algorithm=alg, K=10, n=(3, 6, 9), P=0.5) study = sherpa.Study(algorithm=gs, parameters=parameters, lower_is_better=True, disable_dashboard=True) for trial in study: study.add_observation(trial, iteration=1, objective=f(trial.parameters['x'])) study.finalize(trial) print(study.results)
def test_noisy_parabola(): def f(x, sd=1): y = (x - 3)**2 + 10. if sd == 0: return y else: return y + numpy.random.normal( loc=0., scale=sd, size=numpy.array(x).shape) parameters = [sherpa.Continuous('x1', [0., 7.])] bayesian_optimization = GPyOpt(max_concurrent=1, max_num_trials=5, model_type='GP', acquisition_type='EI') rep = Repeat(algorithm=bayesian_optimization, num_times=3, agg=True) study = sherpa.Study(algorithm=rep, parameters=parameters, lower_is_better=True, disable_dashboard=True) for trial in study: # print("Trial {}:\t{}".format(trial.id, trial.parameters)) fval = f(trial.parameters['x1'], sd=1) # print("Function Value: {}".format(fval)) study.add_observation(trial=trial, iteration=1, objective=fval) study.finalize(trial, status='COMPLETED') # rval = study.get_best_result() # print(rval) print(study.results.query("Status=='COMPLETED'"))
def test_1d_minimize(): def obj_func(x): # Global maximum of 4 is at x=4 return -4. * numpy.exp(-(x - 4.)**2 / 10.) * numpy.cos(1.5 * (x - 4.))**2 parameters = [sherpa.Continuous('x1', [0., 7.])] bayesian_optimization = GPyOpt(max_concurrent=1, max_num_trials=12, model_type='GP', acquisition_type='EI', initial_data_points=[{ 'x1': 2 }, { 'x1': 5 }], num_initial_data_points=2) study = sherpa.Study(algorithm=bayesian_optimization, parameters=parameters, lower_is_better=True, disable_dashboard=True) for trial in study: print("Trial {}:\t{}".format(trial.id, trial.parameters)) fval = obj_func(trial.parameters['x1']) print("Function Value: {}".format(fval)) study.add_observation(trial=trial, iteration=1, objective=fval) study.finalize(trial, status='COMPLETED') rval = study.get_best_result() print(rval) # bounds = [{'name': 'x', 'type': 'continuous', 'domain': (0, 7)}] # Xinit = numpy.array([2, 5]).reshape(-1, 1) # yinit = obj_func(Xinit) # myBopt = gpyopt_package.methods.BayesianOptimization(f=obj_func, # # function to optimize # domain=bounds, # # box-constraints of the problem # acquisition_type='EI', # X=Xinit, # y=yinit, # initial_design_numdata=0, # initial_design_type='random', # evaluator_type='local_penalization', # batch_size=1, # maximize=True, # exact_feval=False) # # Run the optimization # max_iter = 10 # evaluation budget # max_time = 60 # time budget # eps = 10e-6 # Minimum allows distance between the las two observations # # myBopt.run_optimization(max_iter, max_time, eps) # print(myBopt.get_evaluations()) assert numpy.isclose(rval['x1'], 4., atol=0.1)
def test_prepare_data_for_bayes_opt(parameters, results): X, y, y_var = GPyOpt._prepare_data_for_bayes_opt(parameters, results) assert numpy.array_equal( X, numpy.array([[0.1, -3., 1, 111], [0.4, -5., 0, 222], [0.33, -2., 2, 288]])) assert numpy.array_equal(y, numpy.array([[0.1], [0.055], [0.15]]))
def test_transformation_to_gpyopt_domain_log_continuous(parameters): domain = GPyOpt._initialize_domain(parameters) for p, d in zip(parameters, domain): assert d['name'] == p.name assert d['type'] == 'continuous' assert d['domain'] == tuple( [numpy.log10(p.range[0]), numpy.log10(p.range[1])])
def test_bayesopt_batch(parameters, results): gpyopt = GPyOpt(max_concurrent=10) gpyopt.domain = gpyopt._initialize_domain(parameters) gpyopt.lower_is_better = True X, y, y_var = GPyOpt._prepare_data_for_bayes_opt(parameters, results) domain = gpyopt._initialize_domain(parameters) batch = gpyopt._generate_bayesopt_batch(X, y, lower_is_better=True, domain=domain) assert batch.shape == (10, 4)
def test_reverse_format(parameters, results): X, y, y_var = GPyOpt._prepare_data_for_bayes_opt(parameters, results) reversed_X = GPyOpt._reverse_to_sherpa_format(X, parameters) assert reversed_X[0] == { 'dropout': 0.1, 'lr': 1e-3, 'activation': 'tanh', 'num_hidden': 111 } assert reversed_X[1] == { 'dropout': 0.4, 'lr': 1e-5, 'activation': 'relu', 'num_hidden': 222 } assert reversed_X[2] == { 'dropout': 0.33, 'lr': 1e-2, 'activation': 'sigmoid', 'num_hidden': 288 }
def test_mixed_dtype(): algorithm = GPyOpt(max_num_trials=4) parameters = [ sherpa.Choice('param_int', [0, 1]), sherpa.Choice('param_float', [0.1, 1.1]), ] study = sherpa.Study( parameters=parameters, algorithm=algorithm, lower_is_better=True, disable_dashboard=True, ) for trial in study: study.add_observation(trial, iteration=0, objective=0) study.finalize(trial) assert type(trial.parameters['param_int']) == int assert type(trial.parameters['param_float']) == float
def test_transformation_to_gpyopt_domain_with_multiple_parameters(parameters): domain = GPyOpt._initialize_domain(parameters) assert { 'name': 'dropout', 'type': 'continuous', 'domain': (0., 0.5) } == domain[0] assert { 'name': 'lr', 'type': 'continuous', 'domain': (-7, -1) } == domain[1] assert domain[2]['name'] == 'activation' assert domain[2]['type'] == 'categorical' assert numpy.array_equal(domain[2]['domain'], numpy.array([0, 1, 2])) assert { 'name': 'num_hidden', 'type': 'discrete', 'domain': tuple(range(100, 301)) } == domain[3]
def test_3d(): def obj_func(x, y, z): assert isinstance(x, float) assert isinstance(y, str) assert isinstance(z, int) # Global maximum of 4 is at x=4 return -4. * numpy.exp(-(x - 4.)**2 / 10.) * numpy.cos( 1.5 * (x - 4.))**2 - int(y) * z parameters = [ sherpa.Continuous('x', [0., 7.]), sherpa.Choice('y', ["-1", "0", "1"]), sherpa.Discrete('z', [1, 5]) ] bayesian_optimization = GPyOpt(max_concurrent=1, max_num_trials=100, model_type='GP', acquisition_type='EI') study = sherpa.Study(algorithm=bayesian_optimization, parameters=parameters, lower_is_better=True, disable_dashboard=True) for trial in study: print("Trial {}:\t{}".format(trial.id, trial.parameters)) fval = obj_func(**trial.parameters) print("Function Value: {}".format(fval)) study.add_observation(trial=trial, iteration=1, objective=fval) study.finalize(trial, status='COMPLETED') rval = study.get_best_result() print(rval) assert numpy.isclose(rval['x'], 4., atol=0.1) assert rval['y'] == 1 assert rval['z'] == 5
def test_domain(parameters, transforms): domain = GPyOpt._initialize_domain(parameters, transforms) assert { 'name': 'dropout', 'type': 'continuous', 'domain': (0., 0.5) } in domain assert {'name': 'lr', 'type': 'continuous', 'domain': (-7, -1)} in domain assert { 'name': 'activation', 'type': 'discrete', 'domain': (0, 1, 2) } in domain assert { 'name': 'num_hidden', 'type': 'continuous', 'domain': (100, 300) } in domain assert { 'name': 'batch_size', 'type': 'continuous', 'domain': (0, 3) } in domain
def test_infer_num_initial_data_points_too_few_specified(parameters): with pytest.warns( UserWarning, match="num_initial_data_points < number of parameters found"): assert GPyOpt._infer_num_initial_data_points(3, parameters) == 5
def test_infer_num_initial_data_points(parameters): assert GPyOpt._infer_num_initial_data_points('infer', parameters) == 5 assert GPyOpt._infer_num_initial_data_points(3, parameters) == 5 assert GPyOpt._infer_num_initial_data_points(7, parameters) == 7
def test_3d(): def obj_func(x, y, z): # Global maximum of 4 is at x=4 return -4. * numpy.exp(-(x - 4.)**2 / 10.) * numpy.cos( 1.5 * (x - 4.))**2 - y * z parameters = [ sherpa.Continuous('x', [0., 7.]), sherpa.Choice('y', [-1, 0, 1]), sherpa.Discrete('z', [1, 5]) ] bayesian_optimization = GPyOpt(max_concurrent=1, max_num_trials=100, model_type='GP', acquisition_type='EI') study = sherpa.Study(algorithm=bayesian_optimization, parameters=parameters, lower_is_better=True, disable_dashboard=True) for trial in study: print("Trial {}:\t{}".format(trial.id, trial.parameters)) fval = obj_func(**trial.parameters) print("Function Value: {}".format(fval)) study.add_observation(trial=trial, iteration=1, objective=fval) study.finalize(trial, status='COMPLETED') rval = study.get_best_result() print(rval) assert numpy.isclose(rval['x'], 4., atol=0.1) assert rval['y'] == 1 assert rval['z'] == 5 # def test_noisy_parabola(): # def f(x, sd=1): # y = (x - 3) ** 2 + 10. # if sd == 0: # return y # else: # return y + numpy.random.normal(loc=0., scale=sd, # size=numpy.array(x).shape) # # parameters = [sherpa.Continuous('x1', [0., 7.])] # # bayesian_optimization = GPyOpt(max_concurrent=1, # max_num_trials=20, # model_type='GP', # acquisition_type='EI') # rep = Repeat(algorithm=bayesian_optimization, # num_times=5) # study = sherpa.Study(algorithm=rep, # parameters=parameters, # lower_is_better=True, # disable_dashboard=True) # # for trial in study: # print("Trial {}:\t{}".format(trial.id, trial.parameters)) # # fval = f(trial.parameters['x1'], sd=1) # print("Function Value: {}".format(fval)) # study.add_observation(trial=trial, # iteration=1, # objective=fval) # study.finalize(trial, status='COMPLETED') # rval = study.get_best_result() # print(rval) # # assert numpy.sqrt((rval['Objective'] - 3.)**2) < 0.2 # # if __name__ == '__main__': # test_noisy_parabola()
def test_transformation_to_gpyopt_domain_discrete(parameters): domain = GPyOpt._initialize_domain(parameters) for p, d in zip(parameters, domain): assert d['name'] == p.name assert d['type'] == 'discrete' assert d['domain'] == tuple(range(p.range[0], p.range[1] + 1))
def test_transformation_to_gpyopt_domain_log_discrete(): parameters = [sherpa.Discrete('a', [1, 100], 'log')] with pytest.warns(UserWarning, match='does not support log-scale'): GPyOpt._initialize_domain(parameters)
def transforms(parameters): return GPyOpt._initialize_transforms(parameters)
def test_infer_num_initial_data_points(num_initial_data_points, expected, parameters): assert GPyOpt._infer_num_initial_data_points(num_initial_data_points, parameters) == expected