if self.estimator is None: raise NotImplementedError() return self.estimator.predict(X) check_datasets(dataset_list, data_dir) cs = get_cs() for dataset in dataset_list: _x, _y = load_data(dataset, data_dir) eval = partial(eval_func, x=_x, y=_y) print('=' * 10, 'SMBO') bo = SMBO(eval, cs, max_runs=run_count, time_limit_per_trial=60, logging_dir='logs', task_id='test_lgb') bo.run() inc_value = bo.get_incumbent() print('SMBO', '='*30) print(inc_value) print('=' * 10, 'Sync Parallel SMBO') bo = pSMBO(eval, cs, max_runs=run_count, time_limit_per_trial=60, logging_dir='logs', parallel_strategy='sync', batch_size=4) bo.run() inc_value = bo.get_incumbent() print('Sync Parallel SMBO', '='*30) print(inc_value) print('=' * 10, 'Async Parallel SMBO') bo = pSMBO(eval, cs, max_runs=run_count, time_limit_per_trial=60, logging_dir='logs', parallel_strategy='async', batch_size=4) bo.run()
c = 5. / np.pi r = 6. s = 10. t = 1. / (8. * np.pi) ret = a * (x2 - b * x1 ** 2 + c * x1 - r) ** 2 + s * (1 - t) * np.cos(x1) + s result = dict() result['objs'] = (ret, ) return result cs = ConfigurationSpace() x1 = UniformFloatHyperparameter("x1", -5, 10, default_value=0) x2 = UniformFloatHyperparameter("x2", 0, 15, default_value=0) cs.add_hyperparameters([x1, x2]) i = 10 bo = SMBO(branin, cs, advisor_type='default', surrogate_type='gp', acq_optimizer_type='local_random', initial_runs=3, task_id='local_random_bo', random_state=i, max_runs=31, time_limit_per_trial=3, logging_dir='logs') bo.run() bo2 = SMBO(branin, cs, advisor_type='default', surrogate_type='gp', acq_optimizer_type='random_scipy', initial_runs=3, task_id='random_scipy_bo', random_state=i, max_runs=31, time_limit_per_trial=3, logging_dir='logs') bo2.run() print(bo.get_incumbent()) print(bo2.get_incumbent())
def evaluate(dataset, method, algo, space_size, max_run, step_size, seed): if algo == 'xgboost': model_class = XGBoost elif algo == 'lightgbm': model_class = LightGBM elif algo == 'adaboost': model_class = Adaboost elif algo == 'random_forest': model_class = RandomForest elif algo == 'extra_trees': model_class = ExtraTrees else: raise ValueError('Invalid algorithm: %s!' % algo) cs = model_class.get_hyperparameter_search_space(space_size=space_size) x_train, y_train, x_val, y_val = load_data(dataset, solnml_path) def objective_func(config): conf_dict = config.get_dictionary() if algo == 'xgboost': model = XGBoost(**conf_dict, n_jobs=n_jobs, seed=1) elif algo == 'lightgbm': model = LightGBM(**conf_dict, n_jobs=n_jobs, random_state=1) elif algo == 'adaboost': model = Adaboost(**conf_dict, random_state=1) elif algo == 'random_forest': model = RandomForest(**conf_dict, n_jobs=n_jobs, random_state=1) elif algo == 'extra_trees': model = ExtraTrees(**conf_dict, n_jobs=n_jobs, random_state=1) else: raise ValueError('Invalid algorithm: %s' % algo) model.fit(x_train, y_train) from sklearn.metrics import balanced_accuracy_score # evaluate on validation data y_pred = model.predict(x_val) perf = -balanced_accuracy_score(y_val, y_pred) # minimize return perf if method == 'random-search': # tuner = RandomTuner(objective_func, cs, max_run=max_run, random_state=seed) # tuner.run() # print(tuner.get_incumbent()) # config_list = list(tuner.history_dict.keys()) # perf_list = list(tuner.history_dict.values()) from openbox.optimizer.generic_smbo import SMBO task_id = 'tuning-random-%s-%s-%s-%d' % (dataset, algo, space_size, seed) bo = SMBO(objective_func, cs, advisor_type='random', max_runs=max_run, task_id=task_id, logging_dir='logs', random_state=seed) bo.run() print(bo.get_incumbent()) history = bo.get_history() config_list = history.configurations perf_list = history.perfs elif method == 'ada-bo': if algo == 'xgboost': importance_list = [ 'n_estimators', 'learning_rate', 'max_depth', 'colsample_bytree', 'gamma', 'min_child_weight', 'reg_alpha', 'reg_lambda', 'subsample' ] elif algo == 'lightgbm': importance_list = [ 'n_estimators', 'learning_rate', 'num_leaves', 'reg_alpha', 'colsample_bytree', 'min_child_weight', 'reg_lambda', 'subsample', 'max_depth' ] elif algo == 'adaboost': importance_list = [ 'n_estimators', 'learning_rate', 'max_depth', 'algorithm' ] elif algo == 'random_forest': importance_list = [ 'n_estimators', 'max_depth', 'max_features', 'min_samples_leaf', 'min_samples_split', 'bootstrap', 'criterion', 'max_leaf_nodes', 'min_impurity_decrease', 'min_weight_fraction_leaf' ] elif algo == 'extra_trees': importance_list = [ 'n_estimators', 'max_depth', 'max_features', 'min_samples_leaf', 'min_samples_split', 'bootstrap', 'criterion', 'max_leaf_nodes', 'min_impurity_decrease', 'min_weight_fraction_leaf' ] else: raise ValueError('Invalid algorithm~') print('Previous important list is', ','.join(importance_list)) if use_meta_order == "yes": data_, scaler_ = load_meta_data(algorithm=algo, dataset_ids=None, include_scaler=True) X, y, labels = data_ from automlspace.ranknet import RankNetAdvisor advisor = RankNetAdvisor(algorithm_id=algo) advisor.fit(X, y) new_embeding = load_meta_feature(dataset_id=dataset) new_embeding = scaler_.transform([new_embeding])[0] importance_list = advisor.predict_ranking(new_embeding, rank_objs=labels) print('New important list is', ','.join(importance_list)) tuner = AdaptiveTuner(objective_func, cs, importance_list, strategy=strategy, max_run=max_run, step_size=step_size, random_state=seed) tuner.run() print(tuner.get_incumbent()) config_list = list(tuner.history_dict.keys()) perf_list = list(tuner.history_dict.values()) elif method == 'openbox': from openbox.optimizer.generic_smbo import SMBO task_id = 'tuning-openbox-%s-%s-%s-%d' % (dataset, algo, space_size, seed) bo = SMBO(objective_func, cs, advisor_type='default', max_runs=max_run, task_id=task_id, logging_dir='logs', random_state=seed) bo.run() print(bo.get_incumbent()) history = bo.get_history() config_list = history.configurations perf_list = history.perfs elif method == 'tpe': from openbox.optimizer.generic_smbo import SMBO task_id = 'tuning-tpe-%s-%s-%s-%d' % (dataset, algo, space_size, seed) bo = SMBO(objective_func, cs, advisor_type='tpe', max_runs=max_run, task_id=task_id, logging_dir='logs', random_state=seed) bo.run() print(bo.get_incumbent()) history = bo.get_history() config_list = history.configurations perf_list = history.perfs else: raise ValueError('Invalid method id - %s.' % args.method) if len(config_list) > max_run: print('len of result: %d. max_run: %d. cut off.' % (len(config_list), max_run)) config_list = config_list[:max_run] perf_list = perf_list[:max_run] if len(config_list) < max_run: print('===== WARNING: len of result: %d. max_run: %d.' % (len(config_list), max_run)) return config_list, perf_list