def evaluate(mth, run_i, seed):
print(mth, run_i, seed, '===== start =====', flush=True)
def objective_function(config):
y = problem.evaluate_config(config)
res = dict()
# res['config'] = config
res['objs'] = (y, )
# res['constraints'] = None
return res
task_id = '%s_%s_%d' % (mth, problem_str, seed)
bo = SMBO(
objective_function,
cs,
advisor_type=advisor_type, # choices: default, tpe
surrogate_type=surrogate_type, # choices: gp, gp_mcmc, prf, lightgbm
acq_optimizer_type=acq_optimizer_type, # default: local_random
initial_runs=initial_runs, # default: 3
init_strategy=init_strategy, # default: random_explore_first
max_runs=max_runs,
time_limit_per_trial=time_limit_per_trial,
task_id=task_id,
random_state=seed)
if advisor_type == 'tpe':
bo.config_advisor.num_samples = tpe_num_samples
bo.run()
config_list = bo.get_history().configurations
perf_list = bo.get_history().perfs
time_list = bo.get_history().update_times
return config_list, perf_list, time_list
def iterate(self):
config_space, hist_list = self.get_configspace()
# print(self._hp_cnt, config_space)
# print(self._hp_cnt, hist_list)
# Set the number of initial number.
if len(hist_list) > 0:
init_num = 0
else:
init_num = 3
# Set the number of iterations.
# eta = 3
# if self._hp_cnt > 0:
# iter_num = eta ** (self._hp_cnt + 1) - eta ** self._hp_cnt
# if eta ** (self._hp_cnt + 1) > self.max_run:
# iter_num = self.max_run - eta ** self._hp_cnt
# else:
# iter_num = eta
iter_num = self.step_size
smbo = SMBO(self.evaluate_wrapper,
config_space,
advisor_type=self.strategy,
max_runs=iter_num,
init_num=init_num,
task_id='smbo%d' % self._hp_cnt,
random_state=self.random_state)
# Set the history trials.
for _config_dict, _perf in hist_list:
config = deactivate_inactive_hyperparameters(
configuration_space=config_space, configuration=_config_dict)
_observation = Observation(config, SUCCESS, None, (_perf, ), None)
smbo.config_advisor.history_container.update_observation(
_observation)
smbo.run()
# Save the runhistory.
self.history_dict = OrderedDict()
for _config, perf in zip(
smbo.config_advisor.history_container.configurations,
smbo.config_advisor.history_container.perfs):
self.history_dict[_config] = perf
self._hp_cnt += self._delta
if self._hp_cnt > self.hp_size:
self._hp_cnt = self.hp_size
c = 5. / np.pi
r = 6.
s = 10.
t = 1. / (8. * np.pi)
ret = a * (x2 - b * x1 ** 2 + c * x1 - r) ** 2 + s * (1 - t) * np.cos(x1) + s
result = dict()
result['objs'] = (ret, )
return result
cs = ConfigurationSpace()
x1 = UniformFloatHyperparameter("x1", -5, 10, default_value=0)
x2 = UniformFloatHyperparameter("x2", 0, 15, default_value=0)
cs.add_hyperparameters([x1, x2])
i = 10
bo = SMBO(branin, cs, advisor_type='default', surrogate_type='gp',
acq_optimizer_type='local_random', initial_runs=3,
task_id='local_random_bo', random_state=i, max_runs=31, time_limit_per_trial=3, logging_dir='logs')
bo.run()
bo2 = SMBO(branin, cs, advisor_type='default', surrogate_type='gp',
acq_optimizer_type='random_scipy', initial_runs=3,
task_id='random_scipy_bo', random_state=i, max_runs=31, time_limit_per_trial=3, logging_dir='logs')
bo2.run()
print(bo.get_incumbent())
print(bo2.get_incumbent())
def predict(self, X):
if self.estimator is None:
raise NotImplementedError()
return self.estimator.predict(X)
check_datasets(dataset_list, data_dir)
cs = get_cs()
for dataset in dataset_list:
_x, _y = load_data(dataset, data_dir)
eval = partial(eval_func, x=_x, y=_y)
print('=' * 10, 'SMBO')
bo = SMBO(eval, cs, max_runs=run_count, time_limit_per_trial=60, logging_dir='logs', task_id='test_lgb')
bo.run()
inc_value = bo.get_incumbent()
print('SMBO', '='*30)
print(inc_value)
print('=' * 10, 'Sync Parallel SMBO')
bo = pSMBO(eval, cs, max_runs=run_count, time_limit_per_trial=60, logging_dir='logs',
parallel_strategy='sync', batch_size=4)
bo.run()
inc_value = bo.get_incumbent()
print('Sync Parallel SMBO', '='*30)
print(inc_value)
print('=' * 10, 'Async Parallel SMBO')
bo = pSMBO(eval, cs, max_runs=run_count, time_limit_per_trial=60, logging_dir='logs',
parallel_strategy='async', batch_size=4)
from openbox.benchmark.objective_functions.synthetic import Branin
from openbox.optimizer.generic_smbo import SMBO
branin = Branin()
bo = SMBO(branin.evaluate, # objective function
branin.config_space, # config space
num_objs=branin.num_objs, # number of objectives
num_constraints=branin.num_constraints, # number of constraints
max_runs=50, # number of optimization rounds
surrogate_type='prf',
time_limit_per_trial=180,
task_id='quick_start')
history = bo.run()
print(history)
from openbox.benchmark.objective_functions.synthetic import *
from openbox.optimizer.generic_smbo import SMBO
problem = Bukin()
bo = SMBO(problem.evaluate,
problem.config_space,
surrogate_type='gp',
initial_runs=10,
max_runs=60,
task_id='bo')
bo.run()
c_problem = Ackley(constrained=True)
cbo = SMBO(c_problem.evaluate,
c_problem.config_space,
num_constraints=2,
surrogate_type='gp',
initial_runs=10,
max_runs=110,
task_id='cbo')
cbo.run()
cbor = SMBO(c_problem.evaluate,
c_problem.config_space,
num_constraints=2,
sample_strategy='random',
initial_runs=10,
max_runs=110,
task_id='c_random',
random_state=trial_id)
cbor.run()
def evaluate(dataset, method, algo, space_size, max_run, step_size, seed):
if algo == 'xgboost':
model_class = XGBoost
elif algo == 'lightgbm':
model_class = LightGBM
elif algo == 'adaboost':
model_class = Adaboost
elif algo == 'random_forest':
model_class = RandomForest
elif algo == 'extra_trees':
model_class = ExtraTrees
else:
raise ValueError('Invalid algorithm: %s!' % algo)
cs = model_class.get_hyperparameter_search_space(space_size=space_size)
x_train, y_train, x_val, y_val = load_data(dataset, solnml_path)
def objective_func(config):
conf_dict = config.get_dictionary()
if algo == 'xgboost':
model = XGBoost(**conf_dict, n_jobs=n_jobs, seed=1)
elif algo == 'lightgbm':
model = LightGBM(**conf_dict, n_jobs=n_jobs, random_state=1)
elif algo == 'adaboost':
model = Adaboost(**conf_dict, random_state=1)
elif algo == 'random_forest':
model = RandomForest(**conf_dict, n_jobs=n_jobs, random_state=1)
elif algo == 'extra_trees':
model = ExtraTrees(**conf_dict, n_jobs=n_jobs, random_state=1)
else:
raise ValueError('Invalid algorithm: %s' % algo)
model.fit(x_train, y_train)
from sklearn.metrics import balanced_accuracy_score
# evaluate on validation data
y_pred = model.predict(x_val)
perf = -balanced_accuracy_score(y_val, y_pred) # minimize
return perf
if method == 'random-search':
# tuner = RandomTuner(objective_func, cs, max_run=max_run, random_state=seed)
# tuner.run()
# print(tuner.get_incumbent())
# config_list = list(tuner.history_dict.keys())
# perf_list = list(tuner.history_dict.values())
from openbox.optimizer.generic_smbo import SMBO
task_id = 'tuning-random-%s-%s-%s-%d' % (dataset, algo, space_size,
seed)
bo = SMBO(objective_func,
cs,
advisor_type='random',
max_runs=max_run,
task_id=task_id,
logging_dir='logs',
random_state=seed)
bo.run()
print(bo.get_incumbent())
history = bo.get_history()
config_list = history.configurations
perf_list = history.perfs
elif method == 'ada-bo':
if algo == 'xgboost':
importance_list = [
'n_estimators', 'learning_rate', 'max_depth',
'colsample_bytree', 'gamma', 'min_child_weight', 'reg_alpha',
'reg_lambda', 'subsample'
]
elif algo == 'lightgbm':
importance_list = [
'n_estimators', 'learning_rate', 'num_leaves', 'reg_alpha',
'colsample_bytree', 'min_child_weight', 'reg_lambda',
'subsample', 'max_depth'
]
elif algo == 'adaboost':
importance_list = [
'n_estimators', 'learning_rate', 'max_depth', 'algorithm'
]
elif algo == 'random_forest':
importance_list = [
'n_estimators', 'max_depth', 'max_features',
'min_samples_leaf', 'min_samples_split', 'bootstrap',
'criterion', 'max_leaf_nodes', 'min_impurity_decrease',
'min_weight_fraction_leaf'
]
elif algo == 'extra_trees':
importance_list = [
'n_estimators', 'max_depth', 'max_features',
'min_samples_leaf', 'min_samples_split', 'bootstrap',
'criterion', 'max_leaf_nodes', 'min_impurity_decrease',
'min_weight_fraction_leaf'
]
else:
raise ValueError('Invalid algorithm~')
print('Previous important list is', ','.join(importance_list))
if use_meta_order == "yes":
data_, scaler_ = load_meta_data(algorithm=algo,
dataset_ids=None,
include_scaler=True)
X, y, labels = data_
from automlspace.ranknet import RankNetAdvisor
advisor = RankNetAdvisor(algorithm_id=algo)
advisor.fit(X, y)
new_embeding = load_meta_feature(dataset_id=dataset)
new_embeding = scaler_.transform([new_embeding])[0]
importance_list = advisor.predict_ranking(new_embeding,
rank_objs=labels)
print('New important list is', ','.join(importance_list))
tuner = AdaptiveTuner(objective_func,
cs,
importance_list,
strategy=strategy,
max_run=max_run,
step_size=step_size,
random_state=seed)
tuner.run()
print(tuner.get_incumbent())
config_list = list(tuner.history_dict.keys())
perf_list = list(tuner.history_dict.values())
elif method == 'openbox':
from openbox.optimizer.generic_smbo import SMBO
task_id = 'tuning-openbox-%s-%s-%s-%d' % (dataset, algo, space_size,
seed)
bo = SMBO(objective_func,
cs,
advisor_type='default',
max_runs=max_run,
task_id=task_id,
logging_dir='logs',
random_state=seed)
bo.run()
print(bo.get_incumbent())
history = bo.get_history()
config_list = history.configurations
perf_list = history.perfs
elif method == 'tpe':
from openbox.optimizer.generic_smbo import SMBO
task_id = 'tuning-tpe-%s-%s-%s-%d' % (dataset, algo, space_size, seed)
bo = SMBO(objective_func,
cs,
advisor_type='tpe',
max_runs=max_run,
task_id=task_id,
logging_dir='logs',
random_state=seed)
bo.run()
print(bo.get_incumbent())
history = bo.get_history()
config_list = history.configurations
perf_list = history.perfs
else:
raise ValueError('Invalid method id - %s.' % args.method)
if len(config_list) > max_run:
print('len of result: %d. max_run: %d. cut off.' %
(len(config_list), max_run))
config_list = config_list[:max_run]
perf_list = perf_list[:max_run]
if len(config_list) < max_run:
print('===== WARNING: len of result: %d. max_run: %d.' %
(len(config_list), max_run))
return config_list, perf_list
