if self.estimator is None:
raise NotImplementedError()
return self.estimator.predict(X)
check_datasets(dataset_list, data_dir)
cs = get_cs()
for dataset in dataset_list:
_x, _y = load_data(dataset, data_dir)
eval = partial(eval_func, x=_x, y=_y)
print('=' * 10, 'SMBO')
bo = SMBO(eval, cs, max_runs=run_count, time_limit_per_trial=60, logging_dir='logs', task_id='test_lgb')
bo.run()
inc_value = bo.get_incumbent()
print('SMBO', '='*30)
print(inc_value)
print('=' * 10, 'Sync Parallel SMBO')
bo = pSMBO(eval, cs, max_runs=run_count, time_limit_per_trial=60, logging_dir='logs',
parallel_strategy='sync', batch_size=4)
bo.run()
inc_value = bo.get_incumbent()
print('Sync Parallel SMBO', '='*30)
print(inc_value)
print('=' * 10, 'Async Parallel SMBO')
bo = pSMBO(eval, cs, max_runs=run_count, time_limit_per_trial=60, logging_dir='logs',
parallel_strategy='async', batch_size=4)
bo.run()
c = 5. / np.pi
r = 6.
s = 10.
t = 1. / (8. * np.pi)
ret = a * (x2 - b * x1 ** 2 + c * x1 - r) ** 2 + s * (1 - t) * np.cos(x1) + s
result = dict()
result['objs'] = (ret, )
return result
cs = ConfigurationSpace()
x1 = UniformFloatHyperparameter("x1", -5, 10, default_value=0)
x2 = UniformFloatHyperparameter("x2", 0, 15, default_value=0)
cs.add_hyperparameters([x1, x2])
i = 10
bo = SMBO(branin, cs, advisor_type='default', surrogate_type='gp',
acq_optimizer_type='local_random', initial_runs=3,
task_id='local_random_bo', random_state=i, max_runs=31, time_limit_per_trial=3, logging_dir='logs')
bo.run()
bo2 = SMBO(branin, cs, advisor_type='default', surrogate_type='gp',
acq_optimizer_type='random_scipy', initial_runs=3,
task_id='random_scipy_bo', random_state=i, max_runs=31, time_limit_per_trial=3, logging_dir='logs')
bo2.run()
print(bo.get_incumbent())
print(bo2.get_incumbent())
def evaluate(dataset, method, algo, space_size, max_run, step_size, seed):
if algo == 'xgboost':
model_class = XGBoost
elif algo == 'lightgbm':
model_class = LightGBM
elif algo == 'adaboost':
model_class = Adaboost
elif algo == 'random_forest':
model_class = RandomForest
elif algo == 'extra_trees':
model_class = ExtraTrees
else:
raise ValueError('Invalid algorithm: %s!' % algo)
cs = model_class.get_hyperparameter_search_space(space_size=space_size)
x_train, y_train, x_val, y_val = load_data(dataset, solnml_path)
def objective_func(config):
conf_dict = config.get_dictionary()
if algo == 'xgboost':
model = XGBoost(**conf_dict, n_jobs=n_jobs, seed=1)
elif algo == 'lightgbm':
model = LightGBM(**conf_dict, n_jobs=n_jobs, random_state=1)
elif algo == 'adaboost':
model = Adaboost(**conf_dict, random_state=1)
elif algo == 'random_forest':
model = RandomForest(**conf_dict, n_jobs=n_jobs, random_state=1)
elif algo == 'extra_trees':
model = ExtraTrees(**conf_dict, n_jobs=n_jobs, random_state=1)
else:
raise ValueError('Invalid algorithm: %s' % algo)
model.fit(x_train, y_train)
from sklearn.metrics import balanced_accuracy_score
# evaluate on validation data
y_pred = model.predict(x_val)
perf = -balanced_accuracy_score(y_val, y_pred) # minimize
return perf
if method == 'random-search':
# tuner = RandomTuner(objective_func, cs, max_run=max_run, random_state=seed)
# tuner.run()
# print(tuner.get_incumbent())
# config_list = list(tuner.history_dict.keys())
# perf_list = list(tuner.history_dict.values())
from openbox.optimizer.generic_smbo import SMBO
task_id = 'tuning-random-%s-%s-%s-%d' % (dataset, algo, space_size,
seed)
bo = SMBO(objective_func,
cs,
advisor_type='random',
max_runs=max_run,
task_id=task_id,
logging_dir='logs',
random_state=seed)
bo.run()
print(bo.get_incumbent())
history = bo.get_history()
config_list = history.configurations
perf_list = history.perfs
elif method == 'ada-bo':
if algo == 'xgboost':
importance_list = [
'n_estimators', 'learning_rate', 'max_depth',
'colsample_bytree', 'gamma', 'min_child_weight', 'reg_alpha',
'reg_lambda', 'subsample'
]
elif algo == 'lightgbm':
importance_list = [
'n_estimators', 'learning_rate', 'num_leaves', 'reg_alpha',
'colsample_bytree', 'min_child_weight', 'reg_lambda',
'subsample', 'max_depth'
]
elif algo == 'adaboost':
importance_list = [
'n_estimators', 'learning_rate', 'max_depth', 'algorithm'
]
elif algo == 'random_forest':
importance_list = [
'n_estimators', 'max_depth', 'max_features',
'min_samples_leaf', 'min_samples_split', 'bootstrap',
'criterion', 'max_leaf_nodes', 'min_impurity_decrease',
'min_weight_fraction_leaf'
]
elif algo == 'extra_trees':
importance_list = [
'n_estimators', 'max_depth', 'max_features',
'min_samples_leaf', 'min_samples_split', 'bootstrap',
'criterion', 'max_leaf_nodes', 'min_impurity_decrease',
'min_weight_fraction_leaf'
]
else:
raise ValueError('Invalid algorithm~')
print('Previous important list is', ','.join(importance_list))
if use_meta_order == "yes":
data_, scaler_ = load_meta_data(algorithm=algo,
dataset_ids=None,
include_scaler=True)
X, y, labels = data_
from automlspace.ranknet import RankNetAdvisor
advisor = RankNetAdvisor(algorithm_id=algo)
advisor.fit(X, y)
new_embeding = load_meta_feature(dataset_id=dataset)
new_embeding = scaler_.transform([new_embeding])[0]
importance_list = advisor.predict_ranking(new_embeding,
rank_objs=labels)
print('New important list is', ','.join(importance_list))
tuner = AdaptiveTuner(objective_func,
cs,
importance_list,
strategy=strategy,
max_run=max_run,
step_size=step_size,
random_state=seed)
tuner.run()
print(tuner.get_incumbent())
config_list = list(tuner.history_dict.keys())
perf_list = list(tuner.history_dict.values())
elif method == 'openbox':
from openbox.optimizer.generic_smbo import SMBO
task_id = 'tuning-openbox-%s-%s-%s-%d' % (dataset, algo, space_size,
seed)
bo = SMBO(objective_func,
cs,
advisor_type='default',
max_runs=max_run,
task_id=task_id,
logging_dir='logs',
random_state=seed)
bo.run()
print(bo.get_incumbent())
history = bo.get_history()
config_list = history.configurations
perf_list = history.perfs
elif method == 'tpe':
from openbox.optimizer.generic_smbo import SMBO
task_id = 'tuning-tpe-%s-%s-%s-%d' % (dataset, algo, space_size, seed)
bo = SMBO(objective_func,
cs,
advisor_type='tpe',
max_runs=max_run,
task_id=task_id,
logging_dir='logs',
random_state=seed)
bo.run()
print(bo.get_incumbent())
history = bo.get_history()
config_list = history.configurations
perf_list = history.perfs
else:
raise ValueError('Invalid method id - %s.' % args.method)
if len(config_list) > max_run:
print('len of result: %d. max_run: %d. cut off.' %
(len(config_list), max_run))
config_list = config_list[:max_run]
perf_list = perf_list[:max_run]
if len(config_list) < max_run:
print('===== WARNING: len of result: %d. max_run: %d.' %
(len(config_list), max_run))
return config_list, perf_list
