def evaluate(mth, run_i, seed):
print(mth, run_i, seed, '===== start =====', flush=True)
def objective_function(config):
y = problem.evaluate_config(config)
res = dict()
res['config'] = config
res['objs'] = (y,)
res['constraints'] = None
return res
bo = SMBO(objective_function, cs,
surrogate_type=surrogate_type, # default: gp
acq_optimizer_type=acq_optimizer_type, # default: random_scipy
initial_runs=initial_runs, # default: 3
init_strategy=init_strategy, # default: random_explore_first
max_runs=max_runs,
time_limit_per_trial=time_limit_per_trial, task_id=task_id, random_state=seed)
# bo.run()
time_list = []
global_start_time = time.time()
for i in range(max_runs):
config, trial_state, _, objs = bo.iterate()
global_time = time.time() - global_start_time
print(seed, i, objs, config, trial_state, 'time=', global_time)
time_list.append(global_time)
config_list = bo.get_history().configurations
perf_list = bo.get_history().perfs
return config_list, perf_list, time_list
def evaluate(mth, run_i, seed):
print(mth, run_i, seed, '===== start =====', flush=True)
def objective_function(config):
y = problem.evaluate_config(config)
return y
bo = SMBO(objective_function, cs,
num_constraints=num_constraints,
surrogate_type=surrogate_type, # default: gp
acq_optimizer_type=acq_optimizer_type, # default: random_scipy
initial_runs=initial_runs, # default: 3
init_strategy=init_strategy, # default: random_explore_first
max_runs=max_runs + initial_runs,
time_limit_per_trial=time_limit_per_trial, task_id=task_id, random_state=seed)
# bo.run()
config_list = []
perf_list = []
time_list = []
global_start_time = time.time()
for i in range(max_runs):
config, trial_state, constraints, objs = bo.iterate()
global_time = time.time() - global_start_time
origin_perf = objs[0]
if any(c > 0 for c in constraints):
perf = 9999999.0
else:
perf = origin_perf
print(seed, i, perf, config, constraints, trial_state, 'time=', global_time)
config_list.append(config)
perf_list.append(perf)
time_list.append(global_time)
return config_list, perf_list, time_list
def evaluate(mth, run_i, seed):
print(mth, run_i, seed, '===== start =====', flush=True)
def objective_function(config):
y = problem.evaluate_config(config)
res = dict()
res['config'] = config
res['objs'] = (y, )
res['constraints'] = None
return res
bo = SMBO(objective_function,
cs,
sample_strategy='random',
init_strategy='random',
max_runs=max_runs,
time_limit_per_trial=time_limit_per_trial,
task_id=task_id,
random_state=seed)
# bo.run()
config_list = []
perf_list = []
time_list = []
global_start_time = time.time()
for i in range(max_runs):
config, trial_state, objs, trial_info = bo.iterate()
global_time = time.time() - global_start_time
print(seed, i, objs, config, trial_state, trial_info, 'time=',
global_time)
config_list.append(config)
perf_list.append(objs[0])
time_list.append(global_time)
return config_list, perf_list, time_list
def evaluate(mth, run_i, seed):
print(mth, run_i, seed, '===== start =====', flush=True)
def objective_function(config):
res = problem.evaluate_config(config)
res['config'] = config
res['objs'] = np.asarray(res['objs']).tolist()
res['constraints'] = np.asarray(res['constraints']).tolist()
return res
bo = SMBO(
objective_function,
cs,
num_objs=problem.num_objs,
num_constraints=problem.num_constraints,
surrogate_type=surrogate_type, # default: gp
acq_type=acq_type, # default: ehvic
acq_optimizer_type=acq_optimizer_type, # default: random_scipy
initial_runs=initial_runs, # default: 2 * (problem.dim + 1)
init_strategy=init_strategy, # default: sobol
max_runs=max_runs,
ref_point=problem.ref_point,
time_limit_per_trial=time_limit_per_trial,
task_id=task_id,
random_state=seed)
# bo.run()
hv_diffs = []
config_list = []
perf_list = []
time_list = []
global_start_time = time.time()
for i in range(max_runs):
config, trial_state, constraints, origin_objs = bo.iterate()
global_time = time.time() - global_start_time
if any(c > 0 for c in constraints):
objs = [9999999.0] * problem.num_objs
else:
objs = origin_objs
print(seed, i, origin_objs, objs, constraints, config, trial_state,
'time=', global_time)
config_list.append(config)
perf_list.append(objs)
time_list.append(global_time)
hv = Hypervolume(problem.ref_point).compute(perf_list)
hv_diff = problem.max_hv - hv
hv_diffs.append(hv_diff)
print(seed, i, 'hypervolume =', hv)
print(seed, i, 'hv diff =', hv_diff)
pf = np.asarray(bo.get_history().get_pareto_front())
# plot for debugging
if plot_mode == 1:
Y_init = None
plot_pf(problem, problem_str, mth, pf, Y_init)
return hv_diffs, pf, config_list, perf_list, time_list
def iterate(self):
config_space, hist_list = self.get_configspace()
# print(self._hp_cnt, config_space)
# print(self._hp_cnt, hist_list)
# Set the number of initial number.
if len(hist_list) > 0:
init_num = 0
else:
init_num = 3
# Set the number of iterations.
# eta = 3
# if self._hp_cnt > 0:
# iter_num = eta ** (self._hp_cnt + 1) - eta ** self._hp_cnt
# if eta ** (self._hp_cnt + 1) > self.max_run:
# iter_num = self.max_run - eta ** self._hp_cnt
# else:
# iter_num = eta
iter_num = self.step_size
smbo = SMBO(self.evaluate_wrapper,
config_space,
advisor_type=self.strategy,
max_runs=iter_num,
init_num=init_num,
task_id='smbo%d' % self._hp_cnt,
random_state=self.random_state)
# Set the history trials.
for _config_dict, _perf in hist_list:
config = deactivate_inactive_hyperparameters(
configuration_space=config_space, configuration=_config_dict)
_observation = Observation(config, SUCCESS, None, (_perf, ), None)
smbo.config_advisor.history_container.update_observation(
_observation)
smbo.run()
# Save the runhistory.
self.history_dict = OrderedDict()
for _config, perf in zip(
smbo.config_advisor.history_container.configurations,
smbo.config_advisor.history_container.perfs):
self.history_dict[_config] = perf
self._hp_cnt += self._delta
if self._hp_cnt > self.hp_size:
self._hp_cnt = self.hp_size
def __init__(self,
evaluator,
config_space,
name,
eval_type,
time_limit=None,
evaluation_limit=None,
per_run_time_limit=300,
output_dir='./',
timestamp=None,
inner_iter_num_per_iter=1,
seed=1,
n_jobs=1):
super().__init__(evaluator,
config_space,
name,
eval_type=eval_type,
timestamp=timestamp,
output_dir=output_dir,
seed=seed)
self.time_limit = time_limit
self.evaluation_num_limit = evaluation_limit
self.inner_iter_num_per_iter = inner_iter_num_per_iter
self.per_run_time_limit = per_run_time_limit
# self.per_run_mem_limit = per_run_mem_limit
if n_jobs == 1:
self.optimizer = TPE(objective_function=self.evaluator,
config_space=config_space,
advisor_type='tpe',
task_id='Default',
time_limit_per_trial=self.per_run_time_limit,
random_state=self.seed)
else:
raise ValueError('Openbox dose not support parallel TPE')
self.trial_cnt = 0
self.configs = list()
self.perfs = list()
self.exp_output = dict()
self.incumbent_perf = float("-INF")
self.incumbent_config = self.config_space.get_default_configuration()
hp_num = len(self.config_space.get_hyperparameters())
if hp_num == 0:
self.config_num_threshold = 0
else:
_threshold = int(
len(set(self.config_space.sample_configuration(10000))) * 0.75)
self.config_num_threshold = _threshold
self.logger.debug("The maximum trial number in HPO is :%d" %
self.config_num_threshold)
self.maximum_config_num = min(600, self.config_num_threshold)
self.eval_dict = {}
self.n_jobs = n_jobs
def evaluate(mth, run_i, seed):
print(mth, run_i, seed, '===== start =====', flush=True)
def objective_function(config):
y = problem.evaluate_config(config)
res = dict()
# res['config'] = config
res['objs'] = (y, )
# res['constraints'] = None
return res
task_id = '%s_%s_%d' % (mth, problem_str, seed)
bo = SMBO(
objective_function,
cs,
advisor_type=advisor_type, # choices: default, tpe
surrogate_type=surrogate_type, # choices: gp, gp_mcmc, prf, lightgbm
acq_optimizer_type=acq_optimizer_type, # default: local_random
initial_runs=initial_runs, # default: 3
init_strategy=init_strategy, # default: random_explore_first
max_runs=max_runs,
time_limit_per_trial=time_limit_per_trial,
task_id=task_id,
random_state=seed)
if advisor_type == 'tpe':
bo.config_advisor.num_samples = tpe_num_samples
bo.run()
config_list = bo.get_history().configurations
perf_list = bo.get_history().perfs
time_list = bo.get_history().update_times
return config_list, perf_list, time_list
def evaluate(mth, run_i, seed):
print(mth, run_i, seed, '===== start =====', flush=True)
def objective_function(config):
y = problem.evaluate_config(config)
return y
bo = SMBO(objective_function,
cs,
num_constraints=num_constraints,
sample_strategy='random',
init_strategy='random',
max_runs=max_runs,
time_limit_per_trial=time_limit_per_trial,
task_id=task_id,
random_state=seed)
# bo.run()
config_list = []
perf_list = []
time_list = []
global_start_time = time.time()
for i in range(max_runs):
config, trial_state, objs, trial_info = bo.iterate()
global_time = time.time() - global_start_time
print(seed, i, objs, config, trial_state, trial_info, 'time=',
global_time)
config_list.append(config)
perf_list.append(objs[0])
time_list.append(global_time)
_perf_list = []
for i, c in enumerate(bo.config_advisor.constraint_perfs[0]):
if c > 0:
_perf_list.append(9999999)
else:
_perf_list.append(perf_list[i])
return config_list, _perf_list, time_list
def evaluate(problem, seed):
def objective_function(config):
y = problem.evaluate_config(config)
res = dict()
res['config'] = config
res['objs'] = (y, )
res['constraints'] = None
return res
from openbox.optimizer.generic_smbo import SMBO
bo = SMBO(
objective_function,
cs,
surrogate_type=surrogate_type, # default: prf
acq_optimizer_type=acq_optimizer_type, # default: local_random
initial_runs=initial_runs, # default: 3
init_strategy=init_strategy, # default: random_explore_first
max_runs=max_runs,
time_limit_per_trial=time_limit_per_trial,
task_id=task_id,
random_state=seed)
# bo.run()
config_list = []
perf_list = []
time_list = []
global_start_time = time.time()
for i in range(max_runs):
config, trial_state, _, objs = bo.iterate()
global_time = time.time() - global_start_time
print(seed, i, objs, config, trial_state, 'time=', global_time)
config_list.append(config)
perf_list.append(objs[0])
time_list.append(global_time)
if global_time >= runtime_limit:
break
return config_list, perf_list, time_list
from openbox.benchmark.objective_functions.synthetic import Branin
from openbox.optimizer.generic_smbo import SMBO
branin = Branin()
bo = SMBO(branin.evaluate, # objective function
branin.config_space, # config space
num_objs=branin.num_objs, # number of objectives
num_constraints=branin.num_constraints, # number of constraints
max_runs=50, # number of optimization rounds
surrogate_type='prf',
time_limit_per_trial=180,
task_id='quick_start')
history = bo.run()
print(history)
bc_params = {'float': {'x1': (0, 1, 0.5), 'x2': (0, 1, 0.5)}}
bc_cs = ConfigurationSpace()
bc_cs.add_hyperparameters([
UniformFloatHyperparameter(e, *bc_params['float'][e])
for e in bc_params['float']
])
bc_max_hv = 59.36011874867746
bc_ref_point = [18., 6.]
bo = SMBO(branin_currin,
bc_cs,
advisor_type='mcadvisor',
task_id='mcparego',
num_objs=2,
acq_type='mcparego',
ref_point=bc_ref_point,
max_runs=100,
random_state=2)
bo.run()
hvs = bo.get_history().hv_data
log_hv_diff = np.log10(bc_max_hv - np.asarray(hvs))
import matplotlib.pyplot as plt
plt.plot(log_hv_diff)
# plt.savefig('plt.pdf')
plt.show()
from openbox.utils.tuning import get_config_space, get_objective_function
from openbox.optimizer.generic_smbo import SMBO
# prepare your data
X, y = load_digits(return_X_y=True)
x_train, x_val, y_train, y_val = train_test_split(X,
y,
test_size=0.2,
stratify=y,
random_state=1)
# get config_space and objective_function
config_space = get_config_space('xgboost')
objective_function = get_objective_function('xgboost', x_train, x_val, y_train,
y_val)
# run
bo = SMBO(objective_function,
config_space,
max_runs=100,
time_limit_per_trial=180,
task_id='tuning_xgboost')
history = bo.run()
print(history)
history.plot_convergence()
plt.show()
# history.visualize_jupyter()
from openbox.benchmark.objective_functions.synthetic import *
from openbox.optimizer.generic_smbo import SMBO
problem = Bukin()
bo = SMBO(problem.evaluate,
problem.config_space,
surrogate_type='gp',
initial_runs=10,
max_runs=60,
task_id='bo')
bo.run()
c_problem = Ackley(constrained=True)
cbo = SMBO(c_problem.evaluate,
c_problem.config_space,
num_constraints=2,
surrogate_type='gp',
initial_runs=10,
max_runs=110,
task_id='cbo')
cbo.run()
cbor = SMBO(c_problem.evaluate,
c_problem.config_space,
num_constraints=2,
sample_strategy='random',
initial_runs=10,
max_runs=110,
task_id='c_random',
random_state=trial_id)
cbor.run()
import os
import sys
import numpy as np
import matplotlib.pyplot as plt
sys.path.insert(0, os.getcwd())
from openbox.optimizer.generic_smbo import SMBO
from openbox.benchmark.objective_functions.synthetic import BraninCurrin
prob = BraninCurrin()
bo = SMBO(prob.evaluate, prob.config_space,
advisor_type='mcadvisor',
task_id='mcehvi',
num_objs=prob.num_objs,
num_constraints=prob.num_constraints,
acq_type='mcehvi',
ref_point=prob.ref_point,
max_runs=100, random_state=2)
bo.run()
hvs = bo.get_history().hv_data
log_hv_diff = np.log10(prob.max_hv - np.asarray(hvs))
pf = np.asarray(bo.get_history().get_pareto_front())
plt.scatter(pf[:, 0], pf[:, 1])
# plt.plot(log_hv_diff)
# plt.show()
mishra_params = {'float': {'x0': (-10, 0, -5), 'x1': (-6.5, 0, -3.25)}}
mishra_cs = ConfigurationSpace()
mishra_cs.add_hyperparameters([
UniformFloatHyperparameter(name, *para)
for name, para in mishra_params['float'].items()
])
mishra_optimal_value = -106.7645367
obj_func = mishra
cs = mishra_cs
bo = SMBO(obj_func,
cs,
num_constraints=1,
num_objs=1,
acq_optimizer_type='random_scipy',
max_runs=50,
task_id='soc')
bo.run()
history = bo.get_history()
print(history)
history.plot_convergence(true_minimum=mishra_optimal_value)
import matplotlib.pyplot as plt
plt.show()
#plt.savefig('logs/plot_convergence_mishra.png')
[7.77777778e-01, 7.77777778e-01],
[5.55555556e-01, 0],
[0, 5.55555556e-01],
[1.00000000e+00, 4.44444444e-01],
[4.44444444e-01, 1.00000000e+00],
[8.88888889e-01, 1.11111111e-01],
[1.11111111e-01, 8.88888889e-01],
]) # use latin hypercube
X_init = [Configuration(cs, vector=X_init[i]) for i in range(X_init.shape[0])]
bo = SMBO(
multi_objective_func,
cs,
num_objs=num_objs,
max_runs=max_runs,
# surrogate_type='gp_rbf', # use default
acq_type=mth,
# initial_configurations=X_init, initial_runs=10,
time_limit_per_trial=60,
task_id='mo',
random_state=seed)
bo.config_advisor.optimizer.random_chooser.prob = rand_prob # set rand_prob, default 0
bo.config_advisor.acquisition_function.sample_num = sample_num # set sample_num
#bo.config_advisor.acquisition_function.random_state = seed # set random_state
bo.config_advisor.optimizer.num_mc = 1000 # MESMO optimizer only
bo.config_advisor.optimizer.num_opt = 100 # MESMO optimizer only
print(mth, '===== start =====')
# bo.run()
hv_diffs = []
for i in range(max_runs):
config, trial_state, objs, trial_info = bo.iterate()
elif algo == 'adaboost':
importance_list = ['n_estimators', 'learning_rate', 'max_depth', 'algorithm']
elif algo == 'random_forest':
importance_list = ['n_estimators', 'max_depth', 'max_features', 'min_samples_leaf',
'min_samples_split', 'bootstrap', 'criterion', 'max_leaf_nodes',
'min_impurity_decrease', 'min_weight_fraction_leaf']
else:
raise ValueError('Invalid algorithm~')
tuner = AdaptiveTuner(objective_func, cs, importance_list, max_run=max_run, step_size=10)
tuner.run()
print(tuner.get_incumbent())
elif method == 'open-box':
bo = SMBO(objective_func, cs,
advisor_type='default',
max_runs=max_run,
task_id='tuning-openbox',
logging_dir='logs')
bo.run()
print(bo.get_incumbent())
elif method == 'tpe':
bo = SMBO(objective_func, cs,
advisor_type='default',
max_runs=max_run,
task_id='tuning-tpe',
logging_dir='logs')
bo.run()
print(bo.get_incumbent())
else:
raise ValueError('Invalid method id - %s.' % args.method)
c = 5. / np.pi
r = 6.
s = 10.
t = 1. / (8. * np.pi)
ret = a * (x2 - b * x1**2 + c * x1 - r)**2 + s * (1 - t) * np.cos(x1) + s
return {'objs': (ret, )}
cs = ConfigurationSpace()
x1 = UniformFloatHyperparameter("x1", -5, 10, default_value=0)
x2 = UniformFloatHyperparameter("x2", 0, 15, default_value=0)
cs.add_hyperparameters([x1, x2])
seed = np.random.randint(100)
# random search
# bo = SMBO(branin, cs, sample_strategy='random', max_runs=100, task_id='mcei', random_state=seed)
bo = SMBO(branin,
cs,
advisor_type='mcadvisor',
acq_type='mcei',
mc_times=10,
max_runs=50,
task_id='mcei',
random_state=seed)
bo.run()
inc_value = bo.get_incumbent()
print('BO', '=' * 30)
print(inc_value)
def predict(self, X):
if self.estimator is None:
raise NotImplementedError()
return self.estimator.predict(X)
check_datasets(dataset_list, data_dir)
cs = get_cs()
for dataset in dataset_list:
_x, _y = load_data(dataset, data_dir)
eval = partial(eval_func, x=_x, y=_y)
print('=' * 10, 'SMBO')
bo = SMBO(eval, cs, max_runs=run_count, time_limit_per_trial=60, logging_dir='logs', task_id='test_lgb')
bo.run()
inc_value = bo.get_incumbent()
print('SMBO', '='*30)
print(inc_value)
print('=' * 10, 'Sync Parallel SMBO')
bo = pSMBO(eval, cs, max_runs=run_count, time_limit_per_trial=60, logging_dir='logs',
parallel_strategy='sync', batch_size=4)
bo.run()
inc_value = bo.get_incumbent()
print('Sync Parallel SMBO', '='*30)
print(inc_value)
print('=' * 10, 'Async Parallel SMBO')
bo = pSMBO(eval, cs, max_runs=run_count, time_limit_per_trial=60, logging_dir='logs',
t1 + t2 + t3,
]
result['constraints'] = [
np.sum((X + 5)**2) - 25,
]
return result
if __name__ == "__main__":
params = {'float': {'x0': (-10, 0, -5), 'x1': (-6.5, 0, -3.25)}}
cs = ConfigurationSpace()
cs.add_hyperparameters([
UniformFloatHyperparameter(name, *para)
for name, para in params['float'].items()
])
bo = SMBO(mishra,
cs,
num_constraints=1,
num_objs=1,
acq_optimizer_type='random_scipy',
max_runs=50,
time_limit_per_trial=10,
task_id='soc')
history = bo.run()
print(history)
history.plot_convergence(true_minimum=-106.7645367)
plt.show()
import numpy as np
from openbox.optimizer.generic_smbo import SMBO
from openbox.utils.config_space import ConfigurationSpace, UniformFloatHyperparameter
def townsend(config):
X = np.array(list(config.get_dictionary().values()))
res = dict()
res['objs'] = (-(np.cos((X[0]-0.1)*X[1])**2 + X[0] * np.sin(3*X[0]+X[1])), )
res['constraints'] = (-(-np.cos(1.5*X[0]+np.pi)*np.cos(1.5*X[1])+np.sin(1.5*X[0]+np.pi)*np.sin(1.5*X[1])), )
return res
townsend_params = {
'float': {
'x1': (-2.25, 2.5, 0),
'x2': (-2.5, 1.75, 0)
}
}
townsend_cs = ConfigurationSpace()
townsend_cs.add_hyperparameters([UniformFloatHyperparameter(e, *townsend_params['float'][e]) for e in townsend_params['float']])
bo = SMBO(townsend, townsend_cs,
advisor_type='mcadvisor',
acq_type='mceic',
num_constraints=1,
max_runs=60,
task_id='mceic')
bo.run()
def __init__(self,
evaluator,
config_space,
name,
eval_type,
time_limit=None,
evaluation_limit=None,
per_run_time_limit=300,
per_run_mem_limit=1024,
output_dir='./',
timestamp=None,
inner_iter_num_per_iter=1,
seed=1,
n_jobs=1):
super().__init__(evaluator,
config_space,
name,
eval_type=eval_type,
timestamp=timestamp,
output_dir=output_dir,
seed=seed)
self.time_limit = time_limit
self.evaluation_num_limit = evaluation_limit
self.inner_iter_num_per_iter = inner_iter_num_per_iter
self.per_run_time_limit = per_run_time_limit
self.per_run_mem_limit = per_run_mem_limit
if n_jobs == 1:
self.optimizer = BO(objective_function=self.evaluator,
config_space=config_space,
surrogate_type='prf',
acq_type='ei',
max_runs=int(1e10),
task_id='Default',
time_limit_per_trial=self.per_run_time_limit,
random_state=self.seed)
else:
# TODO: Potential read-write conflict on history file.
self.optimizer = pBO(objective_function=self.evaluator,
config_space=config_space,
batch_size=n_jobs,
surrogate_type='prf',
acq_type='ei',
max_runs=int(1e10),
task_id='Default',
time_limit_per_trial=self.per_run_time_limit,
random_state=self.seed)
self.trial_cnt = 0
self.configs = list()
self.perfs = list()
self.exp_output = dict()
self.incumbent_perf = float("-INF")
self.incumbent_config = self.config_space.get_default_configuration()
# Estimate the size of the hyperparameter space.
hp_num = len(self.config_space.get_hyperparameters())
if hp_num == 0:
self.config_num_threshold = 0
else:
_threshold = int(
len(set(self.config_space.sample_configuration(5000))))
self.config_num_threshold = _threshold
self.logger.debug('The maximum trial number in HPO is: %d' %
self.config_num_threshold)
self.maximum_config_num = min(1500, self.config_num_threshold)
self.eval_dict = {}
self.n_jobs = n_jobs
config_space = ConfigurationSpace()
x1 = UniformFloatHyperparameter("x1", 0, 1)
x2 = UniformFloatHyperparameter("x2", 0, 1)
config_space.add_hyperparameters([x1, x2])
# provide reference point if using EHVI method
ref_point = [18.0, 6.0]
# run
bo = SMBO(BraninCurrin,
config_space,
num_objs=2,
num_constraints=0,
max_runs=50,
surrogate_type='gp',
acq_type='ehvi',
acq_optimizer_type='random_scipy',
initial_runs=6,
init_strategy='sobol',
ref_point=ref_point,
time_limit_per_trial=10,
task_id='mo',
random_state=1)
bo.run()
# plot pareto front
pareto_front = np.asarray(bo.get_history().get_pareto_front())
if pareto_front.shape[-1] in (2, 3):
if pareto_front.shape[-1] == 2:
plt.scatter(pareto_front[:, 0], pareto_front[:, 1])
plt.xlabel('Objective 1')
plt.ylabel('Objective 2')
a = 1.
b = 5.1 / (4. * np.pi**2)
c = 5. / np.pi
r = 6.
s = 10.
t = 1. / (8. * np.pi)
y = a * (x2 - b * x1**2 + c * x1 - r)**2 + s * (1 - t) * np.cos(x1) + s
ret = dict(objs=(y, ))
return ret
# Run Optimization
bo = SMBO(branin,
config_space,
num_objs=1,
num_constraints=0,
max_runs=50,
surrogate_type='gp',
time_limit_per_trial=180,
task_id='quick_start')
history = bo.run()
print(history)
history.plot_convergence(true_minimum=0.397887)
plt.show()
# history.visualize_jupyter()
import numpy as np
import matplotlib.pyplot as plt
from openbox.optimizer.generic_smbo import SMBO
from openbox.benchmark.objective_functions.synthetic import ZDT2
dim = 3
prob = ZDT2(dim=dim)
bo = SMBO(prob.evaluate,
prob.config_space,
num_objs=prob.num_objs,
num_constraints=0,
acq_type='ehvi',
acq_optimizer_type='random_scipy',
surrogate_type='gp',
ref_point=prob.ref_point,
max_runs=50,
initial_runs=2 * (dim + 1),
init_strategy='sobol',
task_id='mo',
random_state=1)
bo.run()
# plot pareto front
pareto_front = np.asarray(bo.get_history().get_pareto_front())
if pareto_front.shape[-1] in (2, 3):
if pareto_front.shape[-1] == 2:
plt.scatter(pareto_front[:, 0], pareto_front[:, 1])
plt.xlabel('Objective 1')
plt.ylabel('Objective 2')
def evaluate(dataset, method, algo, space_size, max_run, step_size, seed):
if algo == 'xgboost':
model_class = XGBoost
elif algo == 'lightgbm':
model_class = LightGBM
elif algo == 'adaboost':
model_class = Adaboost
elif algo == 'random_forest':
model_class = RandomForest
elif algo == 'extra_trees':
model_class = ExtraTrees
else:
raise ValueError('Invalid algorithm: %s!' % algo)
cs = model_class.get_hyperparameter_search_space(space_size=space_size)
x_train, y_train, x_val, y_val = load_data(dataset, solnml_path)
def objective_func(config):
conf_dict = config.get_dictionary()
if algo == 'xgboost':
model = XGBoost(**conf_dict, n_jobs=n_jobs, seed=1)
elif algo == 'lightgbm':
model = LightGBM(**conf_dict, n_jobs=n_jobs, random_state=1)
elif algo == 'adaboost':
model = Adaboost(**conf_dict, random_state=1)
elif algo == 'random_forest':
model = RandomForest(**conf_dict, n_jobs=n_jobs, random_state=1)
elif algo == 'extra_trees':
model = ExtraTrees(**conf_dict, n_jobs=n_jobs, random_state=1)
else:
raise ValueError('Invalid algorithm: %s' % algo)
model.fit(x_train, y_train)
from sklearn.metrics import balanced_accuracy_score
# evaluate on validation data
y_pred = model.predict(x_val)
perf = -balanced_accuracy_score(y_val, y_pred) # minimize
return perf
if method == 'random-search':
# tuner = RandomTuner(objective_func, cs, max_run=max_run, random_state=seed)
# tuner.run()
# print(tuner.get_incumbent())
# config_list = list(tuner.history_dict.keys())
# perf_list = list(tuner.history_dict.values())
from openbox.optimizer.generic_smbo import SMBO
task_id = 'tuning-random-%s-%s-%s-%d' % (dataset, algo, space_size,
seed)
bo = SMBO(objective_func,
cs,
advisor_type='random',
max_runs=max_run,
task_id=task_id,
logging_dir='logs',
random_state=seed)
bo.run()
print(bo.get_incumbent())
history = bo.get_history()
config_list = history.configurations
perf_list = history.perfs
elif method == 'ada-bo':
if algo == 'xgboost':
importance_list = [
'n_estimators', 'learning_rate', 'max_depth',
'colsample_bytree', 'gamma', 'min_child_weight', 'reg_alpha',
'reg_lambda', 'subsample'
]
elif algo == 'lightgbm':
importance_list = [
'n_estimators', 'learning_rate', 'num_leaves', 'reg_alpha',
'colsample_bytree', 'min_child_weight', 'reg_lambda',
'subsample', 'max_depth'
]
elif algo == 'adaboost':
importance_list = [
'n_estimators', 'learning_rate', 'max_depth', 'algorithm'
]
elif algo == 'random_forest':
importance_list = [
'n_estimators', 'max_depth', 'max_features',
'min_samples_leaf', 'min_samples_split', 'bootstrap',
'criterion', 'max_leaf_nodes', 'min_impurity_decrease',
'min_weight_fraction_leaf'
]
elif algo == 'extra_trees':
importance_list = [
'n_estimators', 'max_depth', 'max_features',
'min_samples_leaf', 'min_samples_split', 'bootstrap',
'criterion', 'max_leaf_nodes', 'min_impurity_decrease',
'min_weight_fraction_leaf'
]
else:
raise ValueError('Invalid algorithm~')
print('Previous important list is', ','.join(importance_list))
if use_meta_order == "yes":
data_, scaler_ = load_meta_data(algorithm=algo,
dataset_ids=None,
include_scaler=True)
X, y, labels = data_
from automlspace.ranknet import RankNetAdvisor
advisor = RankNetAdvisor(algorithm_id=algo)
advisor.fit(X, y)
new_embeding = load_meta_feature(dataset_id=dataset)
new_embeding = scaler_.transform([new_embeding])[0]
importance_list = advisor.predict_ranking(new_embeding,
rank_objs=labels)
print('New important list is', ','.join(importance_list))
tuner = AdaptiveTuner(objective_func,
cs,
importance_list,
strategy=strategy,
max_run=max_run,
step_size=step_size,
random_state=seed)
tuner.run()
print(tuner.get_incumbent())
config_list = list(tuner.history_dict.keys())
perf_list = list(tuner.history_dict.values())
elif method == 'openbox':
from openbox.optimizer.generic_smbo import SMBO
task_id = 'tuning-openbox-%s-%s-%s-%d' % (dataset, algo, space_size,
seed)
bo = SMBO(objective_func,
cs,
advisor_type='default',
max_runs=max_run,
task_id=task_id,
logging_dir='logs',
random_state=seed)
bo.run()
print(bo.get_incumbent())
history = bo.get_history()
config_list = history.configurations
perf_list = history.perfs
elif method == 'tpe':
from openbox.optimizer.generic_smbo import SMBO
task_id = 'tuning-tpe-%s-%s-%s-%d' % (dataset, algo, space_size, seed)
bo = SMBO(objective_func,
cs,
advisor_type='tpe',
max_runs=max_run,
task_id=task_id,
logging_dir='logs',
random_state=seed)
bo.run()
print(bo.get_incumbent())
history = bo.get_history()
config_list = history.configurations
perf_list = history.perfs
else:
raise ValueError('Invalid method id - %s.' % args.method)
if len(config_list) > max_run:
print('len of result: %d. max_run: %d. cut off.' %
(len(config_list), max_run))
config_list = config_list[:max_run]
perf_list = perf_list[:max_run]
if len(config_list) < max_run:
print('===== WARNING: len of result: %d. max_run: %d.' %
(len(config_list), max_run))
return config_list, perf_list
model = LGBMClassifier(**params)
model.fit(x_train, y_train)
y_pred = model.predict(x_test)
loss = 1 - balanced_accuracy_score(y_test, y_pred) # minimize
return dict(objs=(loss, ))
from openbox.optimizer.generic_smbo import SMBO
import matplotlib.pyplot as plt
# Run Optimization
bo = SMBO(objective_function,
get_configspace(),
num_objs=1,
num_constraints=0,
max_runs=100,
surrogate_type='prf',
time_limit_per_trial=180,
task_id='so_hpo')
bo.run()
history = bo.get_history()
print(history)
history.plot_convergence()
#plt.show()
plt.savefig('logs/plot_convergence_hpo.png')
history.visualize_jupyter()
import os
import sys
import numpy as np
import matplotlib.pyplot as plt
sys.path.insert(0, os.getcwd())
from openbox.optimizer.generic_smbo import SMBO
from openbox.benchmark.objective_functions.synthetic import BraninCurrin
prob = BraninCurrin(constrained=True)
bo = SMBO(prob.evaluate, prob.config_space,
advisor_type='mcadvisor',
task_id='mccmo',
num_objs=prob.num_objs,
num_constraints=prob.num_constraints,
acq_type='mcparegoc',
ref_point=prob.ref_point,
max_runs=100, random_state=2)
bo.run()
hvs = bo.get_history().hv_data
log_hv_diff = np.log10(prob.max_hv - np.asarray(hvs))
plt.plot(log_hv_diff)
plt.show()
c = 5. / np.pi
r = 6.
s = 10.
t = 1. / (8. * np.pi)
ret = a * (x2 - b * x1 ** 2 + c * x1 - r) ** 2 + s * (1 - t) * np.cos(x1) + s
result = dict()
result['objs'] = (ret, )
return result
cs = ConfigurationSpace()
x1 = UniformFloatHyperparameter("x1", -5, 10, default_value=0)
x2 = UniformFloatHyperparameter("x2", 0, 15, default_value=0)
cs.add_hyperparameters([x1, x2])
i = 10
bo = SMBO(branin, cs, advisor_type='default', surrogate_type='gp',
acq_optimizer_type='local_random', initial_runs=3,
task_id='local_random_bo', random_state=i, max_runs=31, time_limit_per_trial=3, logging_dir='logs')
bo.run()
bo2 = SMBO(branin, cs, advisor_type='default', surrogate_type='gp',
acq_optimizer_type='random_scipy', initial_runs=3,
task_id='random_scipy_bo', random_state=i, max_runs=31, time_limit_per_trial=3, logging_dir='logs')
bo2.run()
print(bo.get_incumbent())
print(bo2.get_incumbent())
class RandomSearchOptimizer(BaseOptimizer):
def __init__(self, evaluator, config_space, name, eval_type, time_limit=None, evaluation_limit=None,
per_run_time_limit=300, output_dir='./', timestamp=None,
inner_iter_num_per_iter=1, seed=1, n_jobs=1):
super().__init__(evaluator, config_space, name, eval_type=eval_type, timestamp=timestamp, output_dir=output_dir,
seed=seed)
self.time_limit = time_limit
self.evaluation_num_limit = evaluation_limit
self.inner_iter_num_per_iter = inner_iter_num_per_iter
self.per_run_time_limit = per_run_time_limit
# self.per_run_mem_limit= per_run_mem_limit
if n_jobs == 1:
self.optimizer = RandomSearch(objective_function=self.evaluator,
config_space=config_space,
advisor_type='random',
task_id='Default',
time_limit_per_trial=self.per_run_time_limit,
random_state=self.seed)
else:
self.optimizer = pRandomSearch(objective_function=self.evaluator,
config_space=config_space,
sample_strategy='random',
batch_size=n_jobs,
task_id='Default',
time_limit_per_trial=self.per_run_time_limit,
random_state=self.seed)
self.trial_cnt = 0
self.configs = list()
self.perfs = list()
self.exp_output = dict()
self.incumbent_perf = float("-INF")
self.incumbent_config = self.config_space.get_default_configuration()
hp_num = len(self.config_space.get_hyperparameters())
if hp_num == 0:
self.config_num_threshold = 0
else:
_threshold = int(len(set(self.config_space.sample_configuration(5000))))
self.config_num_threshold = _threshold
self.logger.debug("The maximum trial number in HPO is :%d" % self.config_num_threshold)
self.maximum_config_num = min(1500, self.config_num_threshold)
self.eval_dict = {}
self.n_jobs = n_jobs
def run(self):
while True:
evaluation_num = len(self.perfs)
if self.evaluation_num_limit is not None and evaluation_num > self.evaluation_num_limit:
break
if self.time_limit is not None and time.time() - self.start_time > self.time_limit:
break
self.iterate()
return np.max(self.perfs)
def iterate(self, budget=MAX_INT):
_start_time = time.time()
if len(self.configs) == 0 and self.init_hpo_iter_num is not None:
inner_iter_num = self.init_hpo_iter_num
print('initial hpo trial num is set to %d' % inner_iter_num)
else:
inner_iter_num = self.inner_iter_num_per_iter
if self.n_jobs == 1:
for _ in range(inner_iter_num):
if len(self.configs) >= self.maximum_config_num:
self.early_stopped_flag = True
self.logger.warning('Already explored 70 percentage of the '
'hyperspace or maximum configuration number met: %d!' % self.maximum_config_num)
break
if time.time() - _start_time > budget:
self.logger.warning('Time limit exceeded!')
break
_config, _status, _, _perf = self.optimizer.iterate()
self.update_saver([_config], [_perf[0]])
if _status == SUCCESS:
self.exp_output[time.time()] = (_config, _perf[0])
self.configs.append(_config)
self.perfs.append(-_perf[0])
else:
if len(self.configs) >= self.maximum_config_num:
self.early_stopped_flag = True
self.logger.warning('Already explored 70 percentage of the '
'hyperspace or maximum configuration number met: %d!' % self.maximum_config_num)
elif time.time() - _start_time > budget:
self.logger.warning('Time limit exceeded!')
else:
_config_list, _status_list, _, _perf_list = self.optimizer.async_iterate(n=inner_iter_num)
self.update_saver(_config_list, _perf_list)
for i, _config in enumerate(_config_list):
if _status_list[i] == SUCCESS:
self.exp_output[time.time()] = (_config, _perf_list[i])
self.configs.append(_config)
self.perfs.append(-_perf_list[i])
run_history = self.optimizer.get_history()
if self.name == 'hpo':
if hasattr(self.evaluator, 'fe_config'):
fe_config = self.evaluator.fe_config
else:
fe_config = None
self.eval_dict = {(fe_config, hpo_config): [-run_history.perfs[i], time.time(), run_history.trial_states[i]]
for i, hpo_config in enumerate(run_history.configurations)}
else:
if hasattr(self.evaluator, 'hpo_config'):
hpo_config = self.evaluator.hpo_config
else:
hpo_config = None
self.eval_dict = {(fe_config, hpo_config): [-run_history.perfs[i], time.time(), run_history.trial_states[i]]
for i, fe_config in enumerate(run_history.configurationsa)}
if len(run_history.get_incumbents()) > 0:
self.incumbent_config, self.incumbent_perf = run_history.get_incumbents()[0]
self.incumbent_perf = -self.incumbent_perf
iteration_cost = time.time() - _start_time
return self.incumbent_perf, iteration_cost, self.incumbent_config
