def evaluate(mth, run_i, seed): print(mth, run_i, seed, '===== start =====', flush=True) def objective_function(config): res = problem.evaluate_config(config) res['config'] = config return res bo = SMBO(objective_function, cs, num_objs=problem.num_objs, num_constraints=0, surrogate_type=surrogate_type, # default: gp acq_type=acq_type, # default: ehvi acq_optimizer_type=acq_optimizer_type, # default: random_scipy initial_runs=initial_runs, # default: 2 * (problem.dim + 1) init_strategy=init_strategy, # default: sobol max_runs=max_runs, ref_point=problem.ref_point, time_limit_per_trial=time_limit_per_trial, task_id=task_id, random_state=seed) # bo.run() hv_diffs = [] config_list = [] perf_list = [] time_list = [] global_start_time = time.time() for i in range(max_runs): config, trial_state, objs, trial_info = bo.iterate() global_time = time.time() - global_start_time print(seed, i, objs, config, trial_state, trial_info, 'time=', global_time) hv = Hypervolume(problem.ref_point).compute(bo.get_history().get_pareto_front()) hv_diff = problem.max_hv - hv print(seed, i, 'hypervolume =', hv) print(seed, i, 'hv diff =', hv_diff) hv_diffs.append(hv_diff) config_list.append(config) perf_list.append(objs) time_list.append(global_time) pf = np.asarray(bo.get_history().get_pareto_front()) # plot for debugging if plot_mode == 1: Y_init = None plot_pf(problem, problem_str, mth, pf, Y_init) return hv_diffs, pf, config_list, perf_list, time_list
import os import sys import numpy as np import matplotlib.pyplot as plt sys.path.insert(0, os.getcwd()) from litebo.optimizer.generic_smbo import SMBO from litebo.benchmark.objective_functions.synthetic import BraninCurrin prob = BraninCurrin() bo = SMBO(prob.evaluate, prob.config_space, advisor_type='mcadvisor', task_id='mcehvi', num_objs=prob.num_objs, num_constraints=prob.num_constraints, acq_type='mcehvi', ref_point=prob.ref_point, max_runs=100, random_state=2) bo.run() hvs = bo.get_history().hv_data log_hv_diff = np.log10(prob.max_hv - np.asarray(hvs)) pf = np.asarray(bo.get_history().get_pareto_front()) plt.scatter(pf[:, 0], pf[:, 1]) # plt.plot(log_hv_diff) # plt.show()
import os import sys import numpy as np import matplotlib.pyplot as plt sys.path.insert(0, os.getcwd()) from litebo.optimizer.generic_smbo import SMBO from litebo.benchmark.objective_functions.synthetic import BraninCurrin prob = BraninCurrin(constrained=True) bo = SMBO(prob.evaluate, prob.config_space, advisor_type='mcadvisor', task_id='mccmo', num_objs=prob.num_objs, num_constraints=prob.num_constraints, acq_type='mcparegoc', ref_point=prob.ref_point, max_runs=100, random_state=2) bo.run() hvs = bo.get_history().hv_data log_hv_diff = np.log10(prob.max_hv - np.asarray(hvs)) plt.plot(log_hv_diff) plt.show()
time_limit_per_trial=time_limit_per_trial, task_id='mo', random_state=seed) bo.config_advisor.optimizer.random_chooser.prob = rand_prob # set rand_prob, default 0 bo.config_advisor.acquisition_function.sample_num = sample_num # set sample_num bo.config_advisor.acquisition_function.random_state = seed # set random_state bo.config_advisor.optimizer.num_mc = 10000 # MESMO optimizer only bo.config_advisor.optimizer.num_opt = 10 # MESMO optimizer only print(mth, '===== start =====') # bo.run() hv_diffs = [] for i in range(max_runs): config, trial_state, objs, trial_info = bo.iterate() print(i, objs, config) hv = Hypervolume(referencePoint).compute( bo.get_history().get_pareto_front()) print(i, 'hypervolume =', hv) hv_diff = real_hv - hv hv_diffs.append(hv_diff) print(i, 'hv diff =', hv_diff) # Print result pf = np.asarray(bo.get_history().get_pareto_front()) print(mth, 'pareto num:', pf.shape[0]) print('real hv =', real_hv) print('hv_diffs:', hv_diffs) # Evaluate the random search. bo_r = SMBO(multi_objective_func, cs, num_objs=num_objs,
time_limit_per_trial=time_limit_per_trial, task_id='mo', random_state=seed) bo.config_advisor.optimizer.random_chooser.prob = rand_prob # set rand_prob, default 0 bo.config_advisor.acquisition_function.sample_num = sample_num # set sample_num bo.config_advisor.acquisition_function.random_state = seed # set random_state bo.config_advisor.optimizer.num_mc = opt_num_mc # MESMO optimizer only bo.config_advisor.optimizer.num_opt = opt_num_opt # MESMO optimizer only print(seed, mth, '===== start =====') # bo.run() hv_diffs = [] for i in range(max_runs): config, trial_state, objs, trial_info = bo.iterate() print(seed, i, objs, config) hv = Hypervolume(referencePoint).compute( bo.get_history().get_pareto_front()) print(seed, i, 'hypervolume =', hv) hv_diff = real_hv - hv hv_diffs.append(hv_diff) print(seed, i, 'hv diff =', hv_diff) # Save result pf = np.asarray(bo.get_history().get_pareto_front()) data = bo.get_history().data print(seed, mth, 'pareto num:', pf.shape[0]) print(seed, 'real hv =', real_hv) print(seed, 'hv_diffs:', hv_diffs) timestamp = time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime(time.time())) dir_path = 'logs/mo_benchmark_%s_%d/%s-%d/' % (
def evaluate(mth, run_i, seed): print(mth, run_i, seed, '===== start =====', flush=True) def objective_function(config): res = problem.evaluate_config(config) res['config'] = config res['objs'] = np.asarray(res['objs']).tolist() res['constraints'] = np.asarray(res['constraints']).tolist() return res bo = SMBO( objective_function, cs, num_objs=problem.num_objs, num_constraints=problem.num_constraints, surrogate_type=surrogate_type, # default: gp acq_type=acq_type, # default: ehvic acq_optimizer_type=acq_optimizer_type, # default: random_scipy initial_runs=initial_runs, # default: 2 * (problem.dim + 1) init_strategy=init_strategy, # default: sobol max_runs=max_runs, ref_point=problem.ref_point, time_limit_per_trial=time_limit_per_trial, task_id=task_id, random_state=seed) # bo.run() hv_diffs = [] config_list = [] perf_list = [] time_list = [] global_start_time = time.time() for i in range(max_runs): config, trial_state, origin_objs, trial_info = bo.iterate() global_time = time.time() - global_start_time constraints = [ bo.config_advisor.constraint_perfs[i][-1] for i in range(problem.num_constraints) ] if any(c > 0 for c in constraints): objs = [9999999.0] * problem.num_objs else: objs = origin_objs print(seed, i, origin_objs, objs, constraints, config, trial_state, trial_info, 'time=', global_time) assert len(bo.config_advisor.constraint_perfs[0] ) == i + 1 # make sure no repeat or failed config config_list.append(config) perf_list.append(objs) time_list.append(global_time) hv = Hypervolume(problem.ref_point).compute(perf_list) hv_diff = problem.max_hv - hv hv_diffs.append(hv_diff) print(seed, i, 'hypervolume =', hv) print(seed, i, 'hv diff =', hv_diff) pf = np.asarray(bo.get_history().get_pareto_front()) # plot for debugging if plot_mode == 1: Y_init = None plot_pf(problem, problem_str, mth, pf, Y_init) return hv_diffs, pf, config_list, perf_list, time_list
from litebo.optimizer.generic_smbo import SMBO from litebo.benchmark.objective_functions.synthetic import Ackley num_inputs = 10 acq_optimizer_type = 'random_scipy' seed = 1 prob = Ackley(dim=num_inputs, constrained=False) initial_runs = 2 * (num_inputs + 1) max_runs = 250 bo = SMBO(prob.evaluate, prob.config_space, task_id='turbo', advisor_type='mcadvisor', num_objs=prob.num_objs, num_constraints=prob.num_constraints, acq_type='mcei', acq_optimizer_type=acq_optimizer_type, use_trust_region=True, surrogate_type='gp', max_runs=max_runs, initial_runs=initial_runs, init_strategy='latin_hypercube', random_state=seed) bo.run() values = list(bo.get_history().data.values()) plt.plot(values) plt.show()