def evaluate(mth, run_i, seed): print(mth, run_i, seed, '===== start =====', flush=True) def objective_function(config): y = problem.evaluate_config(config) res = dict() # res['config'] = config res['objs'] = (y, ) # res['constraints'] = None return res task_id = '%s_%s_%d' % (mth, problem_str, seed) bo = SMBO( objective_function, cs, advisor_type=advisor_type, # choices: default, tpe surrogate_type=surrogate_type, # choices: gp, gp_mcmc, prf, lightgbm acq_optimizer_type=acq_optimizer_type, # default: local_random initial_runs=initial_runs, # default: 3 init_strategy=init_strategy, # default: random_explore_first max_runs=max_runs, time_limit_per_trial=time_limit_per_trial, task_id=task_id, random_state=seed) if advisor_type == 'tpe': bo.config_advisor.num_samples = tpe_num_samples bo.run() config_list = bo.get_history().configurations perf_list = bo.get_history().perfs time_list = bo.get_history().update_times return config_list, perf_list, time_list
def iterate(self): config_space, hist_list = self.get_configspace() # print(self._hp_cnt, config_space) # print(self._hp_cnt, hist_list) # Set the number of initial number. if len(hist_list) > 0: init_num = 0 else: init_num = 3 # Set the number of iterations. # eta = 3 # if self._hp_cnt > 0: # iter_num = eta ** (self._hp_cnt + 1) - eta ** self._hp_cnt # if eta ** (self._hp_cnt + 1) > self.max_run: # iter_num = self.max_run - eta ** self._hp_cnt # else: # iter_num = eta iter_num = self.step_size smbo = SMBO(self.evaluate_wrapper, config_space, advisor_type=self.strategy, max_runs=iter_num, init_num=init_num, task_id='smbo%d' % self._hp_cnt, random_state=self.random_state) # Set the history trials. for _config_dict, _perf in hist_list: config = deactivate_inactive_hyperparameters( configuration_space=config_space, configuration=_config_dict) _observation = Observation(config, SUCCESS, None, (_perf, ), None) smbo.config_advisor.history_container.update_observation( _observation) smbo.run() # Save the runhistory. self.history_dict = OrderedDict() for _config, perf in zip( smbo.config_advisor.history_container.configurations, smbo.config_advisor.history_container.perfs): self.history_dict[_config] = perf self._hp_cnt += self._delta if self._hp_cnt > self.hp_size: self._hp_cnt = self.hp_size
c = 5. / np.pi r = 6. s = 10. t = 1. / (8. * np.pi) ret = a * (x2 - b * x1 ** 2 + c * x1 - r) ** 2 + s * (1 - t) * np.cos(x1) + s result = dict() result['objs'] = (ret, ) return result cs = ConfigurationSpace() x1 = UniformFloatHyperparameter("x1", -5, 10, default_value=0) x2 = UniformFloatHyperparameter("x2", 0, 15, default_value=0) cs.add_hyperparameters([x1, x2]) i = 10 bo = SMBO(branin, cs, advisor_type='default', surrogate_type='gp', acq_optimizer_type='local_random', initial_runs=3, task_id='local_random_bo', random_state=i, max_runs=31, time_limit_per_trial=3, logging_dir='logs') bo.run() bo2 = SMBO(branin, cs, advisor_type='default', surrogate_type='gp', acq_optimizer_type='random_scipy', initial_runs=3, task_id='random_scipy_bo', random_state=i, max_runs=31, time_limit_per_trial=3, logging_dir='logs') bo2.run() print(bo.get_incumbent()) print(bo2.get_incumbent())
def predict(self, X): if self.estimator is None: raise NotImplementedError() return self.estimator.predict(X) check_datasets(dataset_list, data_dir) cs = get_cs() for dataset in dataset_list: _x, _y = load_data(dataset, data_dir) eval = partial(eval_func, x=_x, y=_y) print('=' * 10, 'SMBO') bo = SMBO(eval, cs, max_runs=run_count, time_limit_per_trial=60, logging_dir='logs', task_id='test_lgb') bo.run() inc_value = bo.get_incumbent() print('SMBO', '='*30) print(inc_value) print('=' * 10, 'Sync Parallel SMBO') bo = pSMBO(eval, cs, max_runs=run_count, time_limit_per_trial=60, logging_dir='logs', parallel_strategy='sync', batch_size=4) bo.run() inc_value = bo.get_incumbent() print('Sync Parallel SMBO', '='*30) print(inc_value) print('=' * 10, 'Async Parallel SMBO') bo = pSMBO(eval, cs, max_runs=run_count, time_limit_per_trial=60, logging_dir='logs', parallel_strategy='async', batch_size=4)
from openbox.benchmark.objective_functions.synthetic import Branin from openbox.optimizer.generic_smbo import SMBO branin = Branin() bo = SMBO(branin.evaluate, # objective function branin.config_space, # config space num_objs=branin.num_objs, # number of objectives num_constraints=branin.num_constraints, # number of constraints max_runs=50, # number of optimization rounds surrogate_type='prf', time_limit_per_trial=180, task_id='quick_start') history = bo.run() print(history)
from openbox.benchmark.objective_functions.synthetic import * from openbox.optimizer.generic_smbo import SMBO problem = Bukin() bo = SMBO(problem.evaluate, problem.config_space, surrogate_type='gp', initial_runs=10, max_runs=60, task_id='bo') bo.run() c_problem = Ackley(constrained=True) cbo = SMBO(c_problem.evaluate, c_problem.config_space, num_constraints=2, surrogate_type='gp', initial_runs=10, max_runs=110, task_id='cbo') cbo.run() cbor = SMBO(c_problem.evaluate, c_problem.config_space, num_constraints=2, sample_strategy='random', initial_runs=10, max_runs=110, task_id='c_random', random_state=trial_id) cbor.run()
def evaluate(dataset, method, algo, space_size, max_run, step_size, seed): if algo == 'xgboost': model_class = XGBoost elif algo == 'lightgbm': model_class = LightGBM elif algo == 'adaboost': model_class = Adaboost elif algo == 'random_forest': model_class = RandomForest elif algo == 'extra_trees': model_class = ExtraTrees else: raise ValueError('Invalid algorithm: %s!' % algo) cs = model_class.get_hyperparameter_search_space(space_size=space_size) x_train, y_train, x_val, y_val = load_data(dataset, solnml_path) def objective_func(config): conf_dict = config.get_dictionary() if algo == 'xgboost': model = XGBoost(**conf_dict, n_jobs=n_jobs, seed=1) elif algo == 'lightgbm': model = LightGBM(**conf_dict, n_jobs=n_jobs, random_state=1) elif algo == 'adaboost': model = Adaboost(**conf_dict, random_state=1) elif algo == 'random_forest': model = RandomForest(**conf_dict, n_jobs=n_jobs, random_state=1) elif algo == 'extra_trees': model = ExtraTrees(**conf_dict, n_jobs=n_jobs, random_state=1) else: raise ValueError('Invalid algorithm: %s' % algo) model.fit(x_train, y_train) from sklearn.metrics import balanced_accuracy_score # evaluate on validation data y_pred = model.predict(x_val) perf = -balanced_accuracy_score(y_val, y_pred) # minimize return perf if method == 'random-search': # tuner = RandomTuner(objective_func, cs, max_run=max_run, random_state=seed) # tuner.run() # print(tuner.get_incumbent()) # config_list = list(tuner.history_dict.keys()) # perf_list = list(tuner.history_dict.values()) from openbox.optimizer.generic_smbo import SMBO task_id = 'tuning-random-%s-%s-%s-%d' % (dataset, algo, space_size, seed) bo = SMBO(objective_func, cs, advisor_type='random', max_runs=max_run, task_id=task_id, logging_dir='logs', random_state=seed) bo.run() print(bo.get_incumbent()) history = bo.get_history() config_list = history.configurations perf_list = history.perfs elif method == 'ada-bo': if algo == 'xgboost': importance_list = [ 'n_estimators', 'learning_rate', 'max_depth', 'colsample_bytree', 'gamma', 'min_child_weight', 'reg_alpha', 'reg_lambda', 'subsample' ] elif algo == 'lightgbm': importance_list = [ 'n_estimators', 'learning_rate', 'num_leaves', 'reg_alpha', 'colsample_bytree', 'min_child_weight', 'reg_lambda', 'subsample', 'max_depth' ] elif algo == 'adaboost': importance_list = [ 'n_estimators', 'learning_rate', 'max_depth', 'algorithm' ] elif algo == 'random_forest': importance_list = [ 'n_estimators', 'max_depth', 'max_features', 'min_samples_leaf', 'min_samples_split', 'bootstrap', 'criterion', 'max_leaf_nodes', 'min_impurity_decrease', 'min_weight_fraction_leaf' ] elif algo == 'extra_trees': importance_list = [ 'n_estimators', 'max_depth', 'max_features', 'min_samples_leaf', 'min_samples_split', 'bootstrap', 'criterion', 'max_leaf_nodes', 'min_impurity_decrease', 'min_weight_fraction_leaf' ] else: raise ValueError('Invalid algorithm~') print('Previous important list is', ','.join(importance_list)) if use_meta_order == "yes": data_, scaler_ = load_meta_data(algorithm=algo, dataset_ids=None, include_scaler=True) X, y, labels = data_ from automlspace.ranknet import RankNetAdvisor advisor = RankNetAdvisor(algorithm_id=algo) advisor.fit(X, y) new_embeding = load_meta_feature(dataset_id=dataset) new_embeding = scaler_.transform([new_embeding])[0] importance_list = advisor.predict_ranking(new_embeding, rank_objs=labels) print('New important list is', ','.join(importance_list)) tuner = AdaptiveTuner(objective_func, cs, importance_list, strategy=strategy, max_run=max_run, step_size=step_size, random_state=seed) tuner.run() print(tuner.get_incumbent()) config_list = list(tuner.history_dict.keys()) perf_list = list(tuner.history_dict.values()) elif method == 'openbox': from openbox.optimizer.generic_smbo import SMBO task_id = 'tuning-openbox-%s-%s-%s-%d' % (dataset, algo, space_size, seed) bo = SMBO(objective_func, cs, advisor_type='default', max_runs=max_run, task_id=task_id, logging_dir='logs', random_state=seed) bo.run() print(bo.get_incumbent()) history = bo.get_history() config_list = history.configurations perf_list = history.perfs elif method == 'tpe': from openbox.optimizer.generic_smbo import SMBO task_id = 'tuning-tpe-%s-%s-%s-%d' % (dataset, algo, space_size, seed) bo = SMBO(objective_func, cs, advisor_type='tpe', max_runs=max_run, task_id=task_id, logging_dir='logs', random_state=seed) bo.run() print(bo.get_incumbent()) history = bo.get_history() config_list = history.configurations perf_list = history.perfs else: raise ValueError('Invalid method id - %s.' % args.method) if len(config_list) > max_run: print('len of result: %d. max_run: %d. cut off.' % (len(config_list), max_run)) config_list = config_list[:max_run] perf_list = perf_list[:max_run] if len(config_list) < max_run: print('===== WARNING: len of result: %d. max_run: %d.' % (len(config_list), max_run)) return config_list, perf_list