def evaluate(mth, run_i, seed): print(mth, run_i, seed, '===== start =====', flush=True) def objective_function(config): y = problem.evaluate_config(config) return y bo = SMBO(objective_function, cs, num_constraints=num_constraints, surrogate_type=surrogate_type, # default: gp acq_optimizer_type=acq_optimizer_type, # default: random_scipy initial_runs=initial_runs, # default: 3 init_strategy=init_strategy, # default: random_explore_first max_runs=max_runs + initial_runs, time_limit_per_trial=time_limit_per_trial, task_id=task_id, random_state=seed) # bo.run() config_list = [] perf_list = [] time_list = [] global_start_time = time.time() for i in range(max_runs): config, trial_state, constraints, objs = bo.iterate() global_time = time.time() - global_start_time origin_perf = objs[0] if any(c > 0 for c in constraints): perf = 9999999.0 else: perf = origin_perf print(seed, i, perf, config, constraints, trial_state, 'time=', global_time) config_list.append(config) perf_list.append(perf) time_list.append(global_time) return config_list, perf_list, time_list
def evaluate(mth, run_i, seed): print(mth, run_i, seed, '===== start =====', flush=True) def objective_function(config): y = problem.evaluate_config(config) res = dict() res['config'] = config res['objs'] = (y,) res['constraints'] = None return res bo = SMBO(objective_function, cs, surrogate_type=surrogate_type, # default: gp acq_optimizer_type=acq_optimizer_type, # default: random_scipy initial_runs=initial_runs, # default: 3 init_strategy=init_strategy, # default: random_explore_first max_runs=max_runs, time_limit_per_trial=time_limit_per_trial, task_id=task_id, random_state=seed) # bo.run() time_list = [] global_start_time = time.time() for i in range(max_runs): config, trial_state, _, objs = bo.iterate() global_time = time.time() - global_start_time print(seed, i, objs, config, trial_state, 'time=', global_time) time_list.append(global_time) config_list = bo.get_history().configurations perf_list = bo.get_history().perfs return config_list, perf_list, time_list
def evaluate(mth, run_i, seed): print(mth, run_i, seed, '===== start =====', flush=True) def objective_function(config): y = problem.evaluate_config(config) res = dict() # res['config'] = config res['objs'] = (y, ) # res['constraints'] = None return res task_id = '%s_%s_%d' % (mth, problem_str, seed) bo = SMBO( objective_function, cs, advisor_type=advisor_type, # choices: default, tpe surrogate_type=surrogate_type, # choices: gp, gp_mcmc, prf, lightgbm acq_optimizer_type=acq_optimizer_type, # default: local_random initial_runs=initial_runs, # default: 3 init_strategy=init_strategy, # default: random_explore_first max_runs=max_runs, time_limit_per_trial=time_limit_per_trial, task_id=task_id, random_state=seed) if advisor_type == 'tpe': bo.config_advisor.num_samples = tpe_num_samples bo.run() config_list = bo.get_history().configurations perf_list = bo.get_history().perfs time_list = bo.get_history().update_times return config_list, perf_list, time_list
def evaluate(mth, run_i, seed): print(mth, run_i, seed, '===== start =====', flush=True) def objective_function(config): y = problem.evaluate_config(config) res = dict() res['config'] = config res['objs'] = (y, ) res['constraints'] = None return res bo = SMBO(objective_function, cs, sample_strategy='random', init_strategy='random', max_runs=max_runs, time_limit_per_trial=time_limit_per_trial, task_id=task_id, random_state=seed) # bo.run() config_list = [] perf_list = [] time_list = [] global_start_time = time.time() for i in range(max_runs): config, trial_state, objs, trial_info = bo.iterate() global_time = time.time() - global_start_time print(seed, i, objs, config, trial_state, trial_info, 'time=', global_time) config_list.append(config) perf_list.append(objs[0]) time_list.append(global_time) return config_list, perf_list, time_list
def evaluate(mth, run_i, seed): print(mth, run_i, seed, '===== start =====', flush=True) def objective_function(config): res = problem.evaluate_config(config) res['config'] = config res['objs'] = np.asarray(res['objs']).tolist() res['constraints'] = np.asarray(res['constraints']).tolist() return res bo = SMBO( objective_function, cs, num_objs=problem.num_objs, num_constraints=problem.num_constraints, surrogate_type=surrogate_type, # default: gp acq_type=acq_type, # default: ehvic acq_optimizer_type=acq_optimizer_type, # default: random_scipy initial_runs=initial_runs, # default: 2 * (problem.dim + 1) init_strategy=init_strategy, # default: sobol max_runs=max_runs, ref_point=problem.ref_point, time_limit_per_trial=time_limit_per_trial, task_id=task_id, random_state=seed) # bo.run() hv_diffs = [] config_list = [] perf_list = [] time_list = [] global_start_time = time.time() for i in range(max_runs): config, trial_state, constraints, origin_objs = bo.iterate() global_time = time.time() - global_start_time if any(c > 0 for c in constraints): objs = [9999999.0] * problem.num_objs else: objs = origin_objs print(seed, i, origin_objs, objs, constraints, config, trial_state, 'time=', global_time) config_list.append(config) perf_list.append(objs) time_list.append(global_time) hv = Hypervolume(problem.ref_point).compute(perf_list) hv_diff = problem.max_hv - hv hv_diffs.append(hv_diff) print(seed, i, 'hypervolume =', hv) print(seed, i, 'hv diff =', hv_diff) pf = np.asarray(bo.get_history().get_pareto_front()) # plot for debugging if plot_mode == 1: Y_init = None plot_pf(problem, problem_str, mth, pf, Y_init) return hv_diffs, pf, config_list, perf_list, time_list
def iterate(self): config_space, hist_list = self.get_configspace() # print(self._hp_cnt, config_space) # print(self._hp_cnt, hist_list) # Set the number of initial number. if len(hist_list) > 0: init_num = 0 else: init_num = 3 # Set the number of iterations. # eta = 3 # if self._hp_cnt > 0: # iter_num = eta ** (self._hp_cnt + 1) - eta ** self._hp_cnt # if eta ** (self._hp_cnt + 1) > self.max_run: # iter_num = self.max_run - eta ** self._hp_cnt # else: # iter_num = eta iter_num = self.step_size smbo = SMBO(self.evaluate_wrapper, config_space, advisor_type=self.strategy, max_runs=iter_num, init_num=init_num, task_id='smbo%d' % self._hp_cnt, random_state=self.random_state) # Set the history trials. for _config_dict, _perf in hist_list: config = deactivate_inactive_hyperparameters( configuration_space=config_space, configuration=_config_dict) _observation = Observation(config, SUCCESS, None, (_perf, ), None) smbo.config_advisor.history_container.update_observation( _observation) smbo.run() # Save the runhistory. self.history_dict = OrderedDict() for _config, perf in zip( smbo.config_advisor.history_container.configurations, smbo.config_advisor.history_container.perfs): self.history_dict[_config] = perf self._hp_cnt += self._delta if self._hp_cnt > self.hp_size: self._hp_cnt = self.hp_size
def evaluate(mth, run_i, seed): print(mth, run_i, seed, '===== start =====', flush=True) def objective_function(config): y = problem.evaluate_config(config) return y bo = SMBO(objective_function, cs, num_constraints=num_constraints, sample_strategy='random', init_strategy='random', max_runs=max_runs, time_limit_per_trial=time_limit_per_trial, task_id=task_id, random_state=seed) # bo.run() config_list = [] perf_list = [] time_list = [] global_start_time = time.time() for i in range(max_runs): config, trial_state, objs, trial_info = bo.iterate() global_time = time.time() - global_start_time print(seed, i, objs, config, trial_state, trial_info, 'time=', global_time) config_list.append(config) perf_list.append(objs[0]) time_list.append(global_time) _perf_list = [] for i, c in enumerate(bo.config_advisor.constraint_perfs[0]): if c > 0: _perf_list.append(9999999) else: _perf_list.append(perf_list[i]) return config_list, _perf_list, time_list
def evaluate(problem, seed): def objective_function(config): y = problem.evaluate_config(config) res = dict() res['config'] = config res['objs'] = (y, ) res['constraints'] = None return res from openbox.optimizer.generic_smbo import SMBO bo = SMBO( objective_function, cs, surrogate_type=surrogate_type, # default: prf acq_optimizer_type=acq_optimizer_type, # default: local_random initial_runs=initial_runs, # default: 3 init_strategy=init_strategy, # default: random_explore_first max_runs=max_runs, time_limit_per_trial=time_limit_per_trial, task_id=task_id, random_state=seed) # bo.run() config_list = [] perf_list = [] time_list = [] global_start_time = time.time() for i in range(max_runs): config, trial_state, _, objs = bo.iterate() global_time = time.time() - global_start_time print(seed, i, objs, config, trial_state, 'time=', global_time) config_list.append(config) perf_list.append(objs[0]) time_list.append(global_time) if global_time >= runtime_limit: break return config_list, perf_list, time_list
from openbox.utils.tuning import get_config_space, get_objective_function from openbox.optimizer.generic_smbo import SMBO # prepare your data X, y = load_digits(return_X_y=True) x_train, x_val, y_train, y_val = train_test_split(X, y, test_size=0.2, stratify=y, random_state=1) # get config_space and objective_function config_space = get_config_space('xgboost') objective_function = get_objective_function('xgboost', x_train, x_val, y_train, y_val) # run bo = SMBO(objective_function, config_space, max_runs=100, time_limit_per_trial=180, task_id='tuning_xgboost') history = bo.run() print(history) history.plot_convergence() plt.show() # history.visualize_jupyter()
from openbox.benchmark.objective_functions.synthetic import * from openbox.optimizer.generic_smbo import SMBO problem = Bukin() bo = SMBO(problem.evaluate, problem.config_space, surrogate_type='gp', initial_runs=10, max_runs=60, task_id='bo') bo.run() c_problem = Ackley(constrained=True) cbo = SMBO(c_problem.evaluate, c_problem.config_space, num_constraints=2, surrogate_type='gp', initial_runs=10, max_runs=110, task_id='cbo') cbo.run() cbor = SMBO(c_problem.evaluate, c_problem.config_space, num_constraints=2, sample_strategy='random', initial_runs=10, max_runs=110, task_id='c_random', random_state=trial_id) cbor.run()
6)**2 + 10 * (1 - 1 / (8 * np.pi)) * np.cos(px1) + 10 f2 = (1 - np.exp(-1 / (2 * x2))) * (2300 * x1**3 + 1900 * x1**2 + 2092 * x1 + 60) / (100 * x1**3 + 500 * x1**2 + 4 * x1 + 20) res['objs'] = [f1, f2] res['constraints'] = [] return res bc_params = {'float': {'x1': (0, 1, 0.5), 'x2': (0, 1, 0.5)}} bc_cs = ConfigurationSpace() bc_cs.add_hyperparameters([ UniformFloatHyperparameter(e, *bc_params['float'][e]) for e in bc_params['float'] ]) bc_max_hv = 59.36011874867746 bc_ref_point = [18., 6.] bo = SMBO(branin_currin, bc_cs, task_id='parego', num_objs=2, acq_type='parego', ref_point=bc_ref_point, max_runs=100) bo.run() hvs = bo.get_history().hv_data log_hv_diff = np.log10(bc_max_hv - np.asarray(hvs))
mishra_params = {'float': {'x0': (-10, 0, -5), 'x1': (-6.5, 0, -3.25)}} mishra_cs = ConfigurationSpace() mishra_cs.add_hyperparameters([ UniformFloatHyperparameter(name, *para) for name, para in mishra_params['float'].items() ]) mishra_optimal_value = -106.7645367 obj_func = mishra cs = mishra_cs bo = SMBO(obj_func, cs, num_constraints=1, num_objs=1, acq_optimizer_type='random_scipy', max_runs=50, task_id='soc') bo.run() history = bo.get_history() print(history) history.plot_convergence(true_minimum=mishra_optimal_value) import matplotlib.pyplot as plt plt.show() #plt.savefig('logs/plot_convergence_mishra.png')
def predict(self, X): if self.estimator is None: raise NotImplementedError() return self.estimator.predict(X) check_datasets(dataset_list, data_dir) cs = get_cs() for dataset in dataset_list: _x, _y = load_data(dataset, data_dir) eval = partial(eval_func, x=_x, y=_y) print('=' * 10, 'SMBO') bo = SMBO(eval, cs, max_runs=run_count, time_limit_per_trial=60, logging_dir='logs', task_id='test_lgb') bo.run() inc_value = bo.get_incumbent() print('SMBO', '='*30) print(inc_value) print('=' * 10, 'Sync Parallel SMBO') bo = pSMBO(eval, cs, max_runs=run_count, time_limit_per_trial=60, logging_dir='logs', parallel_strategy='sync', batch_size=4) bo.run() inc_value = bo.get_incumbent() print('Sync Parallel SMBO', '='*30) print(inc_value) print('=' * 10, 'Async Parallel SMBO') bo = pSMBO(eval, cs, max_runs=run_count, time_limit_per_trial=60, logging_dir='logs',
[7.77777778e-01, 7.77777778e-01], [5.55555556e-01, 0], [0, 5.55555556e-01], [1.00000000e+00, 4.44444444e-01], [4.44444444e-01, 1.00000000e+00], [8.88888889e-01, 1.11111111e-01], [1.11111111e-01, 8.88888889e-01], ]) # use latin hypercube X_init = [Configuration(cs, vector=X_init[i]) for i in range(X_init.shape[0])] bo = SMBO( multi_objective_func, cs, num_objs=num_objs, max_runs=max_runs, # surrogate_type='gp_rbf', # use default acq_type=mth, # initial_configurations=X_init, initial_runs=10, time_limit_per_trial=60, task_id='mo', random_state=seed) bo.config_advisor.optimizer.random_chooser.prob = rand_prob # set rand_prob, default 0 bo.config_advisor.acquisition_function.sample_num = sample_num # set sample_num #bo.config_advisor.acquisition_function.random_state = seed # set random_state bo.config_advisor.optimizer.num_mc = 1000 # MESMO optimizer only bo.config_advisor.optimizer.num_opt = 100 # MESMO optimizer only print(mth, '===== start =====') # bo.run() hv_diffs = [] for i in range(max_runs): config, trial_state, objs, trial_info = bo.iterate()
from openbox.optimizer.generic_smbo import SMBO from openbox.benchmark.objective_functions.synthetic import Ackley num_inputs = 10 acq_optimizer_type = 'random_scipy' seed = 1 prob = Ackley(dim=num_inputs, constrained=False) initial_runs = 2 * (num_inputs + 1) max_runs = 250 bo = SMBO(prob.evaluate, prob.config_space, task_id='turbo', advisor_type='mcadvisor', num_objs=prob.num_objs, num_constraints=prob.num_constraints, acq_type='mcei', acq_optimizer_type=acq_optimizer_type, use_trust_region=True, surrogate_type='gp', max_runs=max_runs, initial_runs=initial_runs, init_strategy='latin_hypercube', random_state=seed) bo.run() values = list(bo.get_history().data.values()) plt.plot(values) plt.show()
t1 + t2 + t3, ] result['constraints'] = [ np.sum((X + 5)**2) - 25, ] return result if __name__ == "__main__": params = {'float': {'x0': (-10, 0, -5), 'x1': (-6.5, 0, -3.25)}} cs = ConfigurationSpace() cs.add_hyperparameters([ UniformFloatHyperparameter(name, *para) for name, para in params['float'].items() ]) bo = SMBO(mishra, cs, num_constraints=1, num_objs=1, acq_optimizer_type='random_scipy', max_runs=50, time_limit_per_trial=10, task_id='soc') history = bo.run() print(history) history.plot_convergence(true_minimum=-106.7645367) plt.show()
import numpy as np from openbox.optimizer.generic_smbo import SMBO from openbox.utils.config_space import ConfigurationSpace, UniformFloatHyperparameter def townsend(config): X = np.array(list(config.get_dictionary().values())) res = dict() res['objs'] = (-(np.cos((X[0]-0.1)*X[1])**2 + X[0] * np.sin(3*X[0]+X[1])), ) res['constraints'] = (-(-np.cos(1.5*X[0]+np.pi)*np.cos(1.5*X[1])+np.sin(1.5*X[0]+np.pi)*np.sin(1.5*X[1])), ) return res townsend_params = { 'float': { 'x1': (-2.25, 2.5, 0), 'x2': (-2.5, 1.75, 0) } } townsend_cs = ConfigurationSpace() townsend_cs.add_hyperparameters([UniformFloatHyperparameter(e, *townsend_params['float'][e]) for e in townsend_params['float']]) bo = SMBO(townsend, townsend_cs, advisor_type='mcadvisor', acq_type='mceic', num_constraints=1, max_runs=60, task_id='mceic') bo.run()
config_space = ConfigurationSpace() x1 = UniformFloatHyperparameter("x1", 0, 1) x2 = UniformFloatHyperparameter("x2", 0, 1) config_space.add_hyperparameters([x1, x2]) # provide reference point if using EHVI method ref_point = [18.0, 6.0] # run bo = SMBO(BraninCurrin, config_space, num_objs=2, num_constraints=0, max_runs=50, surrogate_type='gp', acq_type='ehvi', acq_optimizer_type='random_scipy', initial_runs=6, init_strategy='sobol', ref_point=ref_point, time_limit_per_trial=10, task_id='mo', random_state=1) bo.run() # plot pareto front pareto_front = np.asarray(bo.get_history().get_pareto_front()) if pareto_front.shape[-1] in (2, 3): if pareto_front.shape[-1] == 2: plt.scatter(pareto_front[:, 0], pareto_front[:, 1]) plt.xlabel('Objective 1') plt.ylabel('Objective 2')
x2 = xs['x2'] a = 1. b = 5.1 / (4. * np.pi**2) c = 5. / np.pi r = 6. s = 10. t = 1. / (8. * np.pi) ret = a * (x2 - b * x1**2 + c * x1 - r)**2 + s * (1 - t) * np.cos(x1) + s return {'objs': (ret, )} cs = ConfigurationSpace() x1 = UniformFloatHyperparameter("x1", -5, 10, default_value=0) x2 = UniformFloatHyperparameter("x2", 0, 15, default_value=0) cs.add_hyperparameters([x1, x2]) bo = SMBO(branin, cs, advisor_type='default', max_runs=50, time_limit_per_trial=3, task_id='hp1') bo.run() inc_value = bo.get_incumbent() print('BO', '=' * 30) print(inc_value) # Execute the following command: # Step 1: `python test/visualization/visualize_smbo.py` # Step 2: `tensorboard --logdir logs/hp1`
import os import sys import numpy as np import matplotlib.pyplot as plt sys.path.insert(0, os.getcwd()) from openbox.optimizer.generic_smbo import SMBO from openbox.benchmark.objective_functions.synthetic import BraninCurrin prob = BraninCurrin(constrained=True) bo = SMBO(prob.evaluate, prob.config_space, advisor_type='mcadvisor', task_id='mccmo', num_objs=prob.num_objs, num_constraints=prob.num_constraints, acq_type='mcparegoc', ref_point=prob.ref_point, max_runs=100, random_state=2) bo.run() hvs = bo.get_history().hv_data log_hv_diff = np.log10(prob.max_hv - np.asarray(hvs)) plt.plot(log_hv_diff) plt.show()
bc_params = {'float': {'x1': (0, 1, 0.5), 'x2': (0, 1, 0.5)}} bc_cs = ConfigurationSpace() bc_cs.add_hyperparameters([ UniformFloatHyperparameter(e, *bc_params['float'][e]) for e in bc_params['float'] ]) bc_max_hv = 59.36011874867746 bc_ref_point = [18., 6.] bo = SMBO(branin_currin, bc_cs, advisor_type='mcadvisor', task_id='mcparego', num_objs=2, acq_type='mcparego', ref_point=bc_ref_point, max_runs=100, random_state=2) bo.run() hvs = bo.get_history().hv_data log_hv_diff = np.log10(bc_max_hv - np.asarray(hvs)) import matplotlib.pyplot as plt plt.plot(log_hv_diff) # plt.savefig('plt.pdf') plt.show()
num_inputs = 5 num_objs = 3 acq_optimizer_type = 'random_scipy' seed = 1 prob = VehicleSafety() initial_runs = 2 * (num_inputs + 1) max_runs = 100 + initial_runs bo = SMBO(prob.evaluate, prob.config_space, task_id='ehvi', num_objs=prob.num_objs, num_constraints=prob.num_constraints, acq_type='ehvi', acq_optimizer_type=acq_optimizer_type, surrogate_type='gp', ref_point=prob.ref_point, max_runs=max_runs, initial_runs=initial_runs, init_strategy='sobol', random_state=seed) bo.run() # plot pareto front pareto_front = np.asarray(bo.get_history().get_pareto_front()) if pareto_front.shape[-1] in (2, 3): if pareto_front.shape[-1] == 2: plt.scatter(pareto_front[:, 0], pareto_front[:, 1]) plt.xlabel('Objective 1') plt.ylabel('Objective 2')
from openbox.benchmark.objective_functions.synthetic import Branin from openbox.optimizer.generic_smbo import SMBO branin = Branin() bo = SMBO(branin.evaluate, # objective function branin.config_space, # config space num_objs=branin.num_objs, # number of objectives num_constraints=branin.num_constraints, # number of constraints max_runs=50, # number of optimization rounds surrogate_type='prf', time_limit_per_trial=180, task_id='quick_start') history = bo.run() print(history)
elif algo == 'adaboost': importance_list = ['n_estimators', 'learning_rate', 'max_depth', 'algorithm'] elif algo == 'random_forest': importance_list = ['n_estimators', 'max_depth', 'max_features', 'min_samples_leaf', 'min_samples_split', 'bootstrap', 'criterion', 'max_leaf_nodes', 'min_impurity_decrease', 'min_weight_fraction_leaf'] else: raise ValueError('Invalid algorithm~') tuner = AdaptiveTuner(objective_func, cs, importance_list, max_run=max_run, step_size=10) tuner.run() print(tuner.get_incumbent()) elif method == 'open-box': bo = SMBO(objective_func, cs, advisor_type='default', max_runs=max_run, task_id='tuning-openbox', logging_dir='logs') bo.run() print(bo.get_incumbent()) elif method == 'tpe': bo = SMBO(objective_func, cs, advisor_type='default', max_runs=max_run, task_id='tuning-tpe', logging_dir='logs') bo.run() print(bo.get_incumbent()) else: raise ValueError('Invalid method id - %s.' % args.method)
import numpy as np import matplotlib.pyplot as plt from openbox.optimizer.generic_smbo import SMBO from openbox.benchmark.objective_functions.synthetic import ZDT2 dim = 3 prob = ZDT2(dim=dim) bo = SMBO(prob.evaluate, prob.config_space, num_objs=prob.num_objs, num_constraints=0, acq_type='ehvi', acq_optimizer_type='random_scipy', surrogate_type='gp', ref_point=prob.ref_point, max_runs=50, initial_runs=2 * (dim + 1), init_strategy='sobol', task_id='mo', random_state=1) bo.run() # plot pareto front pareto_front = np.asarray(bo.get_history().get_pareto_front()) if pareto_front.shape[-1] in (2, 3): if pareto_front.shape[-1] == 2: plt.scatter(pareto_front[:, 0], pareto_front[:, 1]) plt.xlabel('Objective 1') plt.ylabel('Objective 2')
a = 1. b = 5.1 / (4. * np.pi**2) c = 5. / np.pi r = 6. s = 10. t = 1. / (8. * np.pi) y = a * (x2 - b * x1**2 + c * x1 - r)**2 + s * (1 - t) * np.cos(x1) + s ret = dict(objs=(y, )) return ret # Run Optimization bo = SMBO(branin, config_space, num_objs=1, num_constraints=0, max_runs=50, surrogate_type='gp', time_limit_per_trial=180, task_id='quick_start') history = bo.run() print(history) history.plot_convergence(true_minimum=0.397887) plt.show() # history.visualize_jupyter()
model = LGBMClassifier(**params) model.fit(x_train, y_train) y_pred = model.predict(x_test) loss = 1 - balanced_accuracy_score(y_test, y_pred) # minimize return dict(objs=(loss, )) from openbox.optimizer.generic_smbo import SMBO import matplotlib.pyplot as plt # Run Optimization bo = SMBO(objective_function, get_configspace(), num_objs=1, num_constraints=0, max_runs=100, surrogate_type='prf', time_limit_per_trial=180, task_id='so_hpo') bo.run() history = bo.get_history() print(history) history.plot_convergence() #plt.show() plt.savefig('logs/plot_convergence_hpo.png') history.visualize_jupyter()
def evaluate(dataset, method, algo, space_size, max_run, step_size, seed): if algo == 'xgboost': model_class = XGBoost elif algo == 'lightgbm': model_class = LightGBM elif algo == 'adaboost': model_class = Adaboost elif algo == 'random_forest': model_class = RandomForest elif algo == 'extra_trees': model_class = ExtraTrees else: raise ValueError('Invalid algorithm: %s!' % algo) cs = model_class.get_hyperparameter_search_space(space_size=space_size) x_train, y_train, x_val, y_val = load_data(dataset, solnml_path) def objective_func(config): conf_dict = config.get_dictionary() if algo == 'xgboost': model = XGBoost(**conf_dict, n_jobs=n_jobs, seed=1) elif algo == 'lightgbm': model = LightGBM(**conf_dict, n_jobs=n_jobs, random_state=1) elif algo == 'adaboost': model = Adaboost(**conf_dict, random_state=1) elif algo == 'random_forest': model = RandomForest(**conf_dict, n_jobs=n_jobs, random_state=1) elif algo == 'extra_trees': model = ExtraTrees(**conf_dict, n_jobs=n_jobs, random_state=1) else: raise ValueError('Invalid algorithm: %s' % algo) model.fit(x_train, y_train) from sklearn.metrics import balanced_accuracy_score # evaluate on validation data y_pred = model.predict(x_val) perf = -balanced_accuracy_score(y_val, y_pred) # minimize return perf if method == 'random-search': # tuner = RandomTuner(objective_func, cs, max_run=max_run, random_state=seed) # tuner.run() # print(tuner.get_incumbent()) # config_list = list(tuner.history_dict.keys()) # perf_list = list(tuner.history_dict.values()) from openbox.optimizer.generic_smbo import SMBO task_id = 'tuning-random-%s-%s-%s-%d' % (dataset, algo, space_size, seed) bo = SMBO(objective_func, cs, advisor_type='random', max_runs=max_run, task_id=task_id, logging_dir='logs', random_state=seed) bo.run() print(bo.get_incumbent()) history = bo.get_history() config_list = history.configurations perf_list = history.perfs elif method == 'ada-bo': if algo == 'xgboost': importance_list = [ 'n_estimators', 'learning_rate', 'max_depth', 'colsample_bytree', 'gamma', 'min_child_weight', 'reg_alpha', 'reg_lambda', 'subsample' ] elif algo == 'lightgbm': importance_list = [ 'n_estimators', 'learning_rate', 'num_leaves', 'reg_alpha', 'colsample_bytree', 'min_child_weight', 'reg_lambda', 'subsample', 'max_depth' ] elif algo == 'adaboost': importance_list = [ 'n_estimators', 'learning_rate', 'max_depth', 'algorithm' ] elif algo == 'random_forest': importance_list = [ 'n_estimators', 'max_depth', 'max_features', 'min_samples_leaf', 'min_samples_split', 'bootstrap', 'criterion', 'max_leaf_nodes', 'min_impurity_decrease', 'min_weight_fraction_leaf' ] elif algo == 'extra_trees': importance_list = [ 'n_estimators', 'max_depth', 'max_features', 'min_samples_leaf', 'min_samples_split', 'bootstrap', 'criterion', 'max_leaf_nodes', 'min_impurity_decrease', 'min_weight_fraction_leaf' ] else: raise ValueError('Invalid algorithm~') print('Previous important list is', ','.join(importance_list)) if use_meta_order == "yes": data_, scaler_ = load_meta_data(algorithm=algo, dataset_ids=None, include_scaler=True) X, y, labels = data_ from automlspace.ranknet import RankNetAdvisor advisor = RankNetAdvisor(algorithm_id=algo) advisor.fit(X, y) new_embeding = load_meta_feature(dataset_id=dataset) new_embeding = scaler_.transform([new_embeding])[0] importance_list = advisor.predict_ranking(new_embeding, rank_objs=labels) print('New important list is', ','.join(importance_list)) tuner = AdaptiveTuner(objective_func, cs, importance_list, strategy=strategy, max_run=max_run, step_size=step_size, random_state=seed) tuner.run() print(tuner.get_incumbent()) config_list = list(tuner.history_dict.keys()) perf_list = list(tuner.history_dict.values()) elif method == 'openbox': from openbox.optimizer.generic_smbo import SMBO task_id = 'tuning-openbox-%s-%s-%s-%d' % (dataset, algo, space_size, seed) bo = SMBO(objective_func, cs, advisor_type='default', max_runs=max_run, task_id=task_id, logging_dir='logs', random_state=seed) bo.run() print(bo.get_incumbent()) history = bo.get_history() config_list = history.configurations perf_list = history.perfs elif method == 'tpe': from openbox.optimizer.generic_smbo import SMBO task_id = 'tuning-tpe-%s-%s-%s-%d' % (dataset, algo, space_size, seed) bo = SMBO(objective_func, cs, advisor_type='tpe', max_runs=max_run, task_id=task_id, logging_dir='logs', random_state=seed) bo.run() print(bo.get_incumbent()) history = bo.get_history() config_list = history.configurations perf_list = history.perfs else: raise ValueError('Invalid method id - %s.' % args.method) if len(config_list) > max_run: print('len of result: %d. max_run: %d. cut off.' % (len(config_list), max_run)) config_list = config_list[:max_run] perf_list = perf_list[:max_run] if len(config_list) < max_run: print('===== WARNING: len of result: %d. max_run: %d.' % (len(config_list), max_run)) return config_list, perf_list
c = 5. / np.pi r = 6. s = 10. t = 1. / (8. * np.pi) ret = a * (x2 - b * x1 ** 2 + c * x1 - r) ** 2 + s * (1 - t) * np.cos(x1) + s result = dict() result['objs'] = (ret, ) return result cs = ConfigurationSpace() x1 = UniformFloatHyperparameter("x1", -5, 10, default_value=0) x2 = UniformFloatHyperparameter("x2", 0, 15, default_value=0) cs.add_hyperparameters([x1, x2]) i = 10 bo = SMBO(branin, cs, advisor_type='default', surrogate_type='gp', acq_optimizer_type='local_random', initial_runs=3, task_id='local_random_bo', random_state=i, max_runs=31, time_limit_per_trial=3, logging_dir='logs') bo.run() bo2 = SMBO(branin, cs, advisor_type='default', surrogate_type='gp', acq_optimizer_type='random_scipy', initial_runs=3, task_id='random_scipy_bo', random_state=i, max_runs=31, time_limit_per_trial=3, logging_dir='logs') bo2.run() print(bo.get_incumbent()) print(bo2.get_incumbent())
c = 5. / np.pi r = 6. s = 10. t = 1. / (8. * np.pi) ret = a * (x2 - b * x1**2 + c * x1 - r)**2 + s * (1 - t) * np.cos(x1) + s return {'objs': (ret, )} cs = ConfigurationSpace() x1 = UniformFloatHyperparameter("x1", -5, 10, default_value=0) x2 = UniformFloatHyperparameter("x2", 0, 15, default_value=0) cs.add_hyperparameters([x1, x2]) seed = np.random.randint(100) # random search # bo = SMBO(branin, cs, sample_strategy='random', max_runs=100, task_id='mcei', random_state=seed) bo = SMBO(branin, cs, advisor_type='mcadvisor', acq_type='mcei', mc_times=10, max_runs=50, task_id='mcei', random_state=seed) bo.run() inc_value = bo.get_incumbent() print('BO', '=' * 30) print(inc_value)