def tune_meta_learner(): cs = build_configspace() def_value = obj_function(cs.get_default_configuration()) print("Default Value: %.2f" % (def_value)) bo = BayesianOptimization(obj_function, cs, max_runs=50, time_limit_per_trial=1200) bo.run() inc_value = bo.get_incumbent() config = inc_value[0][0] print('Best hyperparameter config found', config) return config
def tune_meta_learner(): cs = build_configspace() def_value = objective_function(cs.get_default_configuration()) print("Default Value: %.2f" % (def_value)) bo = BayesianOptimization(objective_function, cs, max_runs=50, time_limit_per_trial=150) bo.run() inc_value = bo.get_incumbent() config = inc_value[0][0] with open(meta_dir + 'meta_learner_%s_%s_%s_config.pkl' % (meta_algo, metric, hash_id), 'wb') as f: pk.dump(config, f) print('Best hyperparameter config found', config) return config
def evaluate(mode, dataset, run_id, metric): print(mode, dataset, run_id, metric) metric = get_metric(metric) train_data, test_data = load_train_test_data(dataset, task_type=MULTICLASS_CLS) cs = _classifiers[algo_name].get_hyperparameter_search_space() model = UnParametrizedHyperparameter("estimator", algo_name) cs.add_hyperparameter(model) default_hpo_config = cs.get_default_configuration() fe_evaluator = ClassificationEvaluator(default_hpo_config, scorer=metric, name='fe', resampling_strategy='holdout', seed=1) hpo_evaluator = ClassificationEvaluator(default_hpo_config, scorer=metric, data_node=train_data, name='hpo', resampling_strategy='holdout', seed=1) fe_optimizer = BayesianOptimizationOptimizer(task_type=CLASSIFICATION, input_data=train_data, evaluator=fe_evaluator, model_id=algo_name, time_limit_per_trans=600, mem_limit_per_trans=5120, number_of_unit_resource=10, seed=1) def objective_function(config): if benchmark == 'fe': return fe_optimizer.evaluate_function(config) else: return hpo_evaluator(config) if mode == 'bo': bo = BO(objective_function, config_space, max_runs=max_runs, surrogate_model='prob_rf') bo.run() print('BO result') print(bo.get_incumbent()) perf = bo.history_container.incumbent_value runs = [bo.configurations, bo.perfs] elif mode == 'lite_bo': from litebo.facade.bo_facade import BayesianOptimization bo = BayesianOptimization(objective_function, config_space, max_runs=max_runs) bo.run() print('BO result') print(bo.get_incumbent()) perf = bo.history_container.incumbent_value runs = [bo.configurations, bo.perfs] elif mode.startswith('tlbo'): _, gp_fusion = mode.split('_') meta_feature_vec = metafeature_dict[dataset] past_datasets = test_datasets.copy() if dataset in past_datasets: past_datasets.remove(dataset) past_history = load_runhistory(past_datasets) gp_models = [ gp_models_dict[dataset_name] for dataset_name in past_datasets ] tlbo = TLBO(objective_function, config_space, past_history, gp_models=gp_models, dataset_metafeature=meta_feature_vec, max_runs=max_runs, gp_fusion=gp_fusion) tlbo.run() print('TLBO result') print(tlbo.get_incumbent()) runs = [tlbo.configurations, tlbo.perfs] perf = tlbo.history_container.incumbent_value else: raise ValueError('Invalid mode.') file_saved = '%s_%s_%s_result_%d_%d_%s.pkl' % (mode, algo_name, dataset, max_runs, run_id, benchmark) with open(data_dir + file_saved, 'wb') as f: pk.dump([perf, runs], f)
def test_branin(): space_dict = { "parameters": { "x1": { "type": "float", "bound": [-5, 10], "default": 0 }, "x2": { "type": "float", "bound": [0, 15] }, } } cs = get_config_space_from_dict(space_dict) print(cs) bo = BayesianOptimization(branin, cs, max_runs=30, time_limit_per_trial=3, logging_dir='logs') bo.run() inc_value = bo.get_incumbent() print('BO', '=' * 30) print(inc_value) # Evaluate the random search. bo = BayesianOptimization(branin, cs, max_runs=30, time_limit_per_trial=3, sample_strategy='random', logging_dir='logs') bo.run() inc_value = bo.get_incumbent() print('RANDOM', '=' * 30) print(inc_value) # Evaluate batch BO. bo = BatchBayesianOptimization(branin, cs, max_runs=10, batch_size=3, time_limit_per_trial=3, sample_strategy='median_imputation', logging_dir='logs') bo.run() inc_value = bo.get_incumbent() print('MEDIAN IMPUTATION BATCH BO', '=' * 30) print(inc_value) # Evaluate batch BO. bo = BatchBayesianOptimization(branin, cs, max_runs=10, batch_size=3, time_limit_per_trial=3, sample_strategy='local_penalization', logging_dir='logs') bo.run() inc_value = bo.get_incumbent() print('LOCAL PENALIZATION BATCH BO', '=' * 30) print(inc_value)
"default": 0 }, "x2": { "type": "float", "bound": [0, 15] }, } } from litebo.utils.config_space.space_utils import get_config_space_from_dict cs = get_config_space_from_dict(space_dict) print(cs) bo = BayesianOptimization(branin, cs, max_runs=90, time_limit_per_trial=3, logging_dir='logs') bo.run() inc_value = bo.get_incumbent() print('BO', '=' * 30) print(inc_value) # Evaluate the random search. bo = BayesianOptimization(branin, cs, max_runs=90, time_limit_per_trial=3, sample_strategy='random', logging_dir='logs') bo.run()
def evaluate(mth, dataset, run_id): print(mth, dataset, run_id) train_data, test_data = load_train_test_data(dataset, test_size=0.3, task_type=MULTICLASS_CLS) def objective_function(config): metric = get_metric('bal_acc') _, estimator = get_estimator(config.get_dictionary()) X_train, y_train = train_data.data X_test, y_test = test_data.data estimator.fit(X_train, y_train) return -metric(estimator, X_test, y_test) def tpe_objective_function(config): metric = get_metric('bal_acc') _, estimator = get_estimator(config) X_train, y_train = train_data.data X_test, y_test = test_data.data estimator.fit(X_train, y_train) return -metric(estimator, X_test, y_test) config_space = get_configspace() if mth == 'gp_bo': bo = BO(objective_function, config_space, max_runs=max_runs) bo.run() print('new BO result') print(bo.get_incumbent()) perf_bo = bo.history_container.incumbent_value elif mth == 'rf_bo': bo = BO(objective_function, config_space, surrogate_model='prob_rf', max_runs=max_runs) bo.run() print('new BO result') print(bo.get_incumbent()) perf_bo = bo.history_container.incumbent_value elif mth == 'lite_bo': from litebo.facade.bo_facade import BayesianOptimization bo = BayesianOptimization(objective_function, config_space, max_runs=max_runs) bo.run() print('lite BO result') print(bo.get_incumbent()) perf_bo = bo.history_container.incumbent_value elif mth == 'smac': from smac.scenario.scenario import Scenario from smac.facade.smac_facade import SMAC # Scenario object scenario = Scenario({ "run_obj": "quality", "runcount-limit": max_runs, "cs": config_space, "deterministic": "true" }) smac = SMAC(scenario=scenario, rng=np.random.RandomState(42), tae_runner=objective_function) incumbent = smac.optimize() perf_bo = objective_function(incumbent) print('SMAC BO result') print(perf_bo) elif mth == 'tpe': config_space = get_configspace('tpe') from hyperopt import tpe, fmin, Trials trials = Trials() fmin(tpe_objective_function, config_space, tpe.suggest, max_runs, trials=trials) perfs = [trial['result']['loss'] for trial in trials.trials] perf_bo = min(perfs) elif mth == 'tpe_bo': from mindware.components.transfer_learning.tlbo.tpe_optimizer import TPE_BO bo = TPE_BO(objective_function, config_space, max_runs=max_runs) bo.run() print('lite BO result') print(bo.get_incumbent()) perf_bo = bo.history_container.incumbent_value elif mth == 'random_search': from mindware.components.transfer_learning.tlbo.tpe_optimizer import TPE_BO bo = TPE_BO(objective_function, config_space, surrogate_model=mth, max_runs=max_runs) bo.run() print('lite BO result') print(bo.get_incumbent()) perf_bo = bo.history_container.incumbent_value else: raise ValueError('Invalid method.') return perf_bo
"gamma", ["auto", "value"], default_value="auto") # only rbf, poly, sigmoid gamma_value = UniformFloatHyperparameter("gamma_value", 0.0001, 8, default_value=1) cs.add_hyperparameters([gamma, gamma_value]) # We only activate gamma_value if gamma is set to "value" cs.add_condition(InCondition(child=gamma_value, parent=gamma, values=["value"])) # And again we can restrict the use of gamma in general to the choice of the kernel cs.add_condition( InCondition(child=gamma, parent=kernel, values=["rbf", "poly", "sigmoid"])) # Example call of the function # It returns: Status, Cost, Runtime, Additional Infos def_value = svm_from_cfg(cs.get_default_configuration()) print("Default Value: %.2f" % (def_value)) # Optimize, using a SMAC-object print("Optimizing! Depending on your machine, this might take a few minutes.") bo = BayesianOptimization(svm_from_cfg, cs, max_runs=30, time_limit_per_trial=30, logging_dir='logs') bo.run() inc_value = bo.get_incumbent() print(inc_value)
def evaluate(mth, dataset, run_id): print(mth, dataset, run_id) train_data, test_data = load_train_test_data(dataset, test_size=0.3, task_type=MULTICLASS_CLS) cs = _classifiers[algo_name].get_hyperparameter_search_space() model = UnParametrizedHyperparameter("estimator", algo_name) cs.add_hyperparameter(model) default_hpo_config = cs.get_default_configuration() metric = get_metric('bal_acc') fe_evaluator = ClassificationEvaluator(default_hpo_config, scorer=metric, name='fe', resampling_strategy='holdout', seed=1) fe_optimizer = BayesianOptimizationOptimizer(task_type=MULTICLASS_CLS, input_data=train_data, evaluator=fe_evaluator, model_id=algo_name, time_limit_per_trans=600, mem_limit_per_trans=5120, number_of_unit_resource=10, seed=1) config_space = fe_optimizer.hyperparameter_space def objective_function(config): return fe_optimizer.evaluate_function(config) if mth == 'gp_bo': bo = BO(objective_function, config_space, max_runs=max_runs) bo.run() print('new BO result') print(bo.get_incumbent()) perf_bo = bo.history_container.incumbent_value elif mth == 'lite_bo': from litebo.facade.bo_facade import BayesianOptimization bo = BayesianOptimization(objective_function, config_space, max_runs=max_runs) bo.run() print('lite BO result') print(bo.get_incumbent()) perf_bo = bo.history_container.incumbent_value elif mth == 'smac': from smac.scenario.scenario import Scenario from smac.facade.smac_facade import SMAC # Scenario object scenario = Scenario({ "run_obj": "quality", "runcount-limit": max_runs, "cs": config_space, "deterministic": "true" }) smac = SMAC(scenario=scenario, rng=np.random.RandomState(42), tae_runner=objective_function) incumbent = smac.optimize() perf_bo = objective_function(incumbent) print('SMAC BO result') print(perf_bo) else: raise ValueError('Invalid method.') return perf_bo