def evaluate(mth, run_i, seed):
print(mth, run_i, seed, '===== start =====', flush=True)
def objective_function(config):
y = problem.evaluate_config(config)
res = dict()
res['config'] = config
res['objs'] = (y,)
res['constraints'] = None
return res
bo = SMBO(objective_function, cs,
surrogate_type=surrogate_type, # default: gp
acq_optimizer_type=acq_optimizer_type, # default: random_scipy
initial_runs=initial_runs, # default: 3
init_strategy=init_strategy, # default: random_explore_first
max_runs=max_runs,
time_limit_per_trial=time_limit_per_trial, task_id=task_id, random_state=seed)
# bo.run()
time_list = []
global_start_time = time.time()
for i in range(max_runs):
config, trial_state, _, objs = bo.iterate()
global_time = time.time() - global_start_time
print(seed, i, objs, config, trial_state, 'time=', global_time)
time_list.append(global_time)
config_list = bo.get_history().configurations
perf_list = bo.get_history().perfs
return config_list, perf_list, time_list
def evaluate(mth, run_i, seed):
print(mth, run_i, seed, '===== start =====', flush=True)
def objective_function(config):
y = problem.evaluate_config(config)
res = dict()
res['config'] = config
res['objs'] = (y, )
res['constraints'] = None
return res
bo = SMBO(objective_function,
cs,
sample_strategy='random',
init_strategy='random',
max_runs=max_runs,
time_limit_per_trial=time_limit_per_trial,
task_id=task_id,
random_state=seed)
# bo.run()
config_list = []
perf_list = []
time_list = []
global_start_time = time.time()
for i in range(max_runs):
config, trial_state, objs, trial_info = bo.iterate()
global_time = time.time() - global_start_time
print(seed, i, objs, config, trial_state, trial_info, 'time=',
global_time)
config_list.append(config)
perf_list.append(objs[0])
time_list.append(global_time)
return config_list, perf_list, time_list
def evaluate(mth, run_i, seed):
print(mth, run_i, seed, '===== start =====', flush=True)
def objective_function(config):
y = problem.evaluate_config(config)
return y
bo = SMBO(objective_function, cs,
num_constraints=num_constraints,
surrogate_type=surrogate_type, # default: gp
acq_optimizer_type=acq_optimizer_type, # default: random_scipy
initial_runs=initial_runs, # default: 3
init_strategy=init_strategy, # default: random_explore_first
max_runs=max_runs + initial_runs,
time_limit_per_trial=time_limit_per_trial, task_id=task_id, random_state=seed)
# bo.run()
config_list = []
perf_list = []
time_list = []
global_start_time = time.time()
for i in range(max_runs):
config, trial_state, constraints, objs = bo.iterate()
global_time = time.time() - global_start_time
origin_perf = objs[0]
if any(c > 0 for c in constraints):
perf = 9999999.0
else:
perf = origin_perf
print(seed, i, perf, config, constraints, trial_state, 'time=', global_time)
config_list.append(config)
perf_list.append(perf)
time_list.append(global_time)
return config_list, perf_list, time_list
def evaluate(mth, run_i, seed):
print(mth, run_i, seed, '===== start =====', flush=True)
def objective_function(config):
res = problem.evaluate_config(config)
res['config'] = config
res['objs'] = np.asarray(res['objs']).tolist()
res['constraints'] = np.asarray(res['constraints']).tolist()
return res
bo = SMBO(
objective_function,
cs,
num_objs=problem.num_objs,
num_constraints=problem.num_constraints,
surrogate_type=surrogate_type, # default: gp
acq_type=acq_type, # default: ehvic
acq_optimizer_type=acq_optimizer_type, # default: random_scipy
initial_runs=initial_runs, # default: 2 * (problem.dim + 1)
init_strategy=init_strategy, # default: sobol
max_runs=max_runs,
ref_point=problem.ref_point,
time_limit_per_trial=time_limit_per_trial,
task_id=task_id,
random_state=seed)
# bo.run()
hv_diffs = []
config_list = []
perf_list = []
time_list = []
global_start_time = time.time()
for i in range(max_runs):
config, trial_state, constraints, origin_objs = bo.iterate()
global_time = time.time() - global_start_time
if any(c > 0 for c in constraints):
objs = [9999999.0] * problem.num_objs
else:
objs = origin_objs
print(seed, i, origin_objs, objs, constraints, config, trial_state,
'time=', global_time)
config_list.append(config)
perf_list.append(objs)
time_list.append(global_time)
hv = Hypervolume(problem.ref_point).compute(perf_list)
hv_diff = problem.max_hv - hv
hv_diffs.append(hv_diff)
print(seed, i, 'hypervolume =', hv)
print(seed, i, 'hv diff =', hv_diff)
pf = np.asarray(bo.get_history().get_pareto_front())
# plot for debugging
if plot_mode == 1:
Y_init = None
plot_pf(problem, problem_str, mth, pf, Y_init)
return hv_diffs, pf, config_list, perf_list, time_list
def evaluate(mth, run_i, seed):
print(mth, run_i, seed, '===== start =====', flush=True)
def objective_function(config):
y = problem.evaluate_config(config)
return y
bo = SMBO(objective_function,
cs,
num_constraints=num_constraints,
sample_strategy='random',
init_strategy='random',
max_runs=max_runs,
time_limit_per_trial=time_limit_per_trial,
task_id=task_id,
random_state=seed)
# bo.run()
config_list = []
perf_list = []
time_list = []
global_start_time = time.time()
for i in range(max_runs):
config, trial_state, objs, trial_info = bo.iterate()
global_time = time.time() - global_start_time
print(seed, i, objs, config, trial_state, trial_info, 'time=',
global_time)
config_list.append(config)
perf_list.append(objs[0])
time_list.append(global_time)
_perf_list = []
for i, c in enumerate(bo.config_advisor.constraint_perfs[0]):
if c > 0:
_perf_list.append(9999999)
else:
_perf_list.append(perf_list[i])
return config_list, _perf_list, time_list
def evaluate(problem, seed):
def objective_function(config):
y = problem.evaluate_config(config)
res = dict()
res['config'] = config
res['objs'] = (y, )
res['constraints'] = None
return res
from openbox.optimizer.generic_smbo import SMBO
bo = SMBO(
objective_function,
cs,
surrogate_type=surrogate_type, # default: prf
acq_optimizer_type=acq_optimizer_type, # default: local_random
initial_runs=initial_runs, # default: 3
init_strategy=init_strategy, # default: random_explore_first
max_runs=max_runs,
time_limit_per_trial=time_limit_per_trial,
task_id=task_id,
random_state=seed)
# bo.run()
config_list = []
perf_list = []
time_list = []
global_start_time = time.time()
for i in range(max_runs):
config, trial_state, _, objs = bo.iterate()
global_time = time.time() - global_start_time
print(seed, i, objs, config, trial_state, 'time=', global_time)
config_list.append(config)
perf_list.append(objs[0])
time_list.append(global_time)
if global_time >= runtime_limit:
break
return config_list, perf_list, time_list
# surrogate_type='gp_rbf', # use default
acq_type=mth,
# initial_configurations=X_init, initial_runs=10,
time_limit_per_trial=60,
task_id='mo',
random_state=seed)
bo.config_advisor.optimizer.random_chooser.prob = rand_prob # set rand_prob, default 0
bo.config_advisor.acquisition_function.sample_num = sample_num # set sample_num
#bo.config_advisor.acquisition_function.random_state = seed # set random_state
bo.config_advisor.optimizer.num_mc = 1000 # MESMO optimizer only
bo.config_advisor.optimizer.num_opt = 100 # MESMO optimizer only
print(mth, '===== start =====')
# bo.run()
hv_diffs = []
for i in range(max_runs):
config, trial_state, objs, trial_info = bo.iterate()
print(i, objs, config)
hv = Hypervolume(referencePoint).compute(
bo.get_history().get_pareto_front())
print(i, 'hypervolume =', hv)
hv_diff = real_hv - hv
hv_diffs.append(hv_diff)
print(i, 'hv diff =', hv_diff)
# Print result
pf = np.asarray(bo.get_history().get_pareto_front())
print(mth, 'pareto num:', pf.shape[0])
print('real hv =', real_hv)
print('hv_diffs:', hv_diffs)
# Evaluate the random search.
class RandomSearchOptimizer(BaseOptimizer):
def __init__(self, evaluator, config_space, name, eval_type, time_limit=None, evaluation_limit=None,
per_run_time_limit=300, output_dir='./', timestamp=None,
inner_iter_num_per_iter=1, seed=1, n_jobs=1):
super().__init__(evaluator, config_space, name, eval_type=eval_type, timestamp=timestamp, output_dir=output_dir,
seed=seed)
self.time_limit = time_limit
self.evaluation_num_limit = evaluation_limit
self.inner_iter_num_per_iter = inner_iter_num_per_iter
self.per_run_time_limit = per_run_time_limit
# self.per_run_mem_limit= per_run_mem_limit
if n_jobs == 1:
self.optimizer = RandomSearch(objective_function=self.evaluator,
config_space=config_space,
advisor_type='random',
task_id='Default',
time_limit_per_trial=self.per_run_time_limit,
random_state=self.seed)
else:
self.optimizer = pRandomSearch(objective_function=self.evaluator,
config_space=config_space,
sample_strategy='random',
batch_size=n_jobs,
task_id='Default',
time_limit_per_trial=self.per_run_time_limit,
random_state=self.seed)
self.trial_cnt = 0
self.configs = list()
self.perfs = list()
self.exp_output = dict()
self.incumbent_perf = float("-INF")
self.incumbent_config = self.config_space.get_default_configuration()
hp_num = len(self.config_space.get_hyperparameters())
if hp_num == 0:
self.config_num_threshold = 0
else:
_threshold = int(len(set(self.config_space.sample_configuration(5000))))
self.config_num_threshold = _threshold
self.logger.debug("The maximum trial number in HPO is :%d" % self.config_num_threshold)
self.maximum_config_num = min(1500, self.config_num_threshold)
self.eval_dict = {}
self.n_jobs = n_jobs
def run(self):
while True:
evaluation_num = len(self.perfs)
if self.evaluation_num_limit is not None and evaluation_num > self.evaluation_num_limit:
break
if self.time_limit is not None and time.time() - self.start_time > self.time_limit:
break
self.iterate()
return np.max(self.perfs)
def iterate(self, budget=MAX_INT):
_start_time = time.time()
if len(self.configs) == 0 and self.init_hpo_iter_num is not None:
inner_iter_num = self.init_hpo_iter_num
print('initial hpo trial num is set to %d' % inner_iter_num)
else:
inner_iter_num = self.inner_iter_num_per_iter
if self.n_jobs == 1:
for _ in range(inner_iter_num):
if len(self.configs) >= self.maximum_config_num:
self.early_stopped_flag = True
self.logger.warning('Already explored 70 percentage of the '
'hyperspace or maximum configuration number met: %d!' % self.maximum_config_num)
break
if time.time() - _start_time > budget:
self.logger.warning('Time limit exceeded!')
break
_config, _status, _, _perf = self.optimizer.iterate()
self.update_saver([_config], [_perf[0]])
if _status == SUCCESS:
self.exp_output[time.time()] = (_config, _perf[0])
self.configs.append(_config)
self.perfs.append(-_perf[0])
else:
if len(self.configs) >= self.maximum_config_num:
self.early_stopped_flag = True
self.logger.warning('Already explored 70 percentage of the '
'hyperspace or maximum configuration number met: %d!' % self.maximum_config_num)
elif time.time() - _start_time > budget:
self.logger.warning('Time limit exceeded!')
else:
_config_list, _status_list, _, _perf_list = self.optimizer.async_iterate(n=inner_iter_num)
self.update_saver(_config_list, _perf_list)
for i, _config in enumerate(_config_list):
if _status_list[i] == SUCCESS:
self.exp_output[time.time()] = (_config, _perf_list[i])
self.configs.append(_config)
self.perfs.append(-_perf_list[i])
run_history = self.optimizer.get_history()
if self.name == 'hpo':
if hasattr(self.evaluator, 'fe_config'):
fe_config = self.evaluator.fe_config
else:
fe_config = None
self.eval_dict = {(fe_config, hpo_config): [-run_history.perfs[i], time.time(), run_history.trial_states[i]]
for i, hpo_config in enumerate(run_history.configurations)}
else:
if hasattr(self.evaluator, 'hpo_config'):
hpo_config = self.evaluator.hpo_config
else:
hpo_config = None
self.eval_dict = {(fe_config, hpo_config): [-run_history.perfs[i], time.time(), run_history.trial_states[i]]
for i, fe_config in enumerate(run_history.configurationsa)}
if len(run_history.get_incumbents()) > 0:
self.incumbent_config, self.incumbent_perf = run_history.get_incumbents()[0]
self.incumbent_perf = -self.incumbent_perf
iteration_cost = time.time() - _start_time
return self.incumbent_perf, iteration_cost, self.incumbent_config
