def config_space(self): """Decision tree hyperparameter space.""" n_estimators = UniformIntegerHyperparameter( 'n_estimators', lower=10, upper=200, default_value=100 ) criterion = CategoricalHyperparameter( 'criterion', ['gini', 'entropy'], default_value='gini' ) max_depth = CategoricalHyperparameter( 'max_depth', [5, 10, 20, 'none'], default_value='none' ) max_features = CategoricalHyperparameter( 'max_features', ['auto', 'sqrt', 'log2', 'none'], default_value='none' ) # Add hyperparameters to config space. config = ConfigurationSpace() config.seed(self.random_state) config.add_hyperparameters( ( n_estimators, criterion, max_depth, max_features ) ) return config
def config_space(self): """Decision tree hyperparameter space.""" criterion = CategoricalHyperparameter( 'criterion', ['gini', 'entropy'], default_value='gini' ) # NOTE: Default value = 'none' is translated to None in base class. # ConfigSpace does not allow for None values as default. max_depth = CategoricalHyperparameter( 'max_depth', [5, 10, 20, 'none'], default_value='none' ) # NOTE: Default value = 'none' is translated to None in base class. # ConfigSpace does not allow for None values as default. max_features = CategoricalHyperparameter( 'max_features', ['auto', 'sqrt', 'log2', 'none'], default_value='none' ) min_samples_leaf = UniformIntegerHyperparameter( 'min_samples_leaf', lower=2, upper=5, default_value=3 ) # Add hyperparameters to config space. config = ConfigurationSpace() config.seed(self.random_state) config.add_hyperparameters( ( criterion, max_depth, max_features, min_samples_leaf ) ) return config
def config_space(self): """Logistic regression hyperparameter space.""" reg_param = UniformFloatHyperparameter( 'reg_param', lower=1e-9, upper=1-1e-9, default_value=1e-3 ) # Add hyperparameters to config space. config = ConfigurationSpace() config.seed(self.random_state) config.add_hyperparameter(reg_param) return config
def config_space(self): """Logistic regression hyperparameter space.""" alpha = UniformFloatHyperparameter( 'alpha', lower=1e-8, upper=100, default_value=1.0 ) # Add hyperparameters to config space. config = ConfigurationSpace() config.seed(self.random_state) config.add_hyperparameter(alpha) return config
def config_space(self): """Logistic regression hyperparameter space.""" n_components = UniformIntegerHyperparameter( 'n_components', lower=2, upper=50, default_value=10 ) # Add hyperparameters to config space. config = ConfigurationSpace() config.seed(self.random_state) config.add_hyperparameter(n_components) return config
def config_space(self): """Returns the Fisher score hyperparameter configuration space.""" num_features = UniformIntegerHyperparameter('num_features', lower=2, upper=50, default_value=20) config = ConfigurationSpace() config.seed(self.random_state) config.add_hyperparameter(num_features) return config
def config_space(self): """Logistic regression hyperparameter space.""" C_param = UniformFloatHyperparameter( 'C', lower=1e-8, upper=1000.0, default_value=1.0 ) penalty = CategoricalHyperparameter( 'penalty', ['l1', 'l2'], default_value='l1' ) # Add hyperparameters to config space. config = ConfigurationSpace() config.seed(self.random_state) config.add_hyperparameters((C_param, penalty)) return config
def config_space(self): """Returns the MI hyperparameter configuration space.""" num_neighbors = UniformIntegerHyperparameter('num_neighbors', lower=10, upper=100, default_value=20) num_features = UniformIntegerHyperparameter('num_features', lower=2, upper=50, default_value=20) config = ConfigurationSpace() config.seed(self.random_state) config.add_hyperparameters((num_neighbors, num_features)) return config
def _get_acm_cs(self): """ returns a configuration space designed for querying ~smac.optimizer.smbo._component_builder Returns ------- ConfigurationSpace """ cs = ConfigurationSpace() cs.seed(self.rng.randint(0,2**20)) model = CategoricalHyperparameter("model", choices=("RF", "GP")) num_trees = Constant("num_trees", value=10) bootstrap = CategoricalHyperparameter("do_bootstrapping", choices=(True, False), default_value=True) ratio_features = CategoricalHyperparameter("ratio_features", choices=(3 / 6, 4 / 6, 5 / 6, 1), default_value=1) min_split = UniformIntegerHyperparameter("min_samples_to_split", lower=1, upper=10, default_value=2) min_leaves = UniformIntegerHyperparameter("min_samples_in_leaf", lower=1, upper=10, default_value=1) cs.add_hyperparameters([model, num_trees, bootstrap, ratio_features, min_split, min_leaves]) inc_num_trees = InCondition(num_trees, model, ["RF"]) inc_bootstrap = InCondition(bootstrap, model, ["RF"]) inc_ratio_features = InCondition(ratio_features, model, ["RF"]) inc_min_split = InCondition(min_split, model, ["RF"]) inc_min_leavs = InCondition(min_leaves, model, ["RF"]) cs.add_conditions([inc_num_trees, inc_bootstrap, inc_ratio_features, inc_min_split, inc_min_leavs]) acq = CategoricalHyperparameter("acq_func", choices=("EI", "LCB", "PI", "LogEI")) par_ei = UniformFloatHyperparameter("par_ei", lower=-10, upper=10) par_pi = UniformFloatHyperparameter("par_pi", lower=-10, upper=10) par_logei = UniformFloatHyperparameter("par_logei", lower=0.001, upper=100, log=True) par_lcb = UniformFloatHyperparameter("par_lcb", lower=0.0001, upper=0.9999) cs.add_hyperparameters([acq, par_ei, par_pi, par_logei, par_lcb]) inc_par_ei = InCondition(par_ei, acq, ["EI"]) inc_par_pi = InCondition(par_pi, acq, ["PI"]) inc_par_logei = InCondition(par_logei, acq, ["LogEI"]) inc_par_lcb = InCondition(par_lcb, acq, ["LCB"]) cs.add_conditions([inc_par_ei, inc_par_pi, inc_par_logei, inc_par_lcb]) return cs
def config_space(self): """SVC hyperparameter space.""" C_param = UniformFloatHyperparameter( 'C', lower=1e-8, upper=100.0, default_value=1.0 ) shrinking = CategoricalHyperparameter( 'shrinking', [True, False], default_value=True ) kernel = CategoricalHyperparameter( 'kernel', ['linear', 'rbf', 'poly', 'sigmoid'], ) degree = UniformIntegerHyperparameter( 'degree', lower=1, upper=5, default_value=2 ) coef0 = UniformFloatHyperparameter( 'coef0', lower=0.0, upper=10.0, default_value=0.0 ) # Add hyperparameters to config space. config = ConfigurationSpace() config.seed(self.random_state) config.add_hyperparameters( ( C_param, shrinking, kernel, degree, coef0, ) ) # Conditionals on hyperparameters specific to kernels. config.add_conditions( ( InCondition(child=degree, parent=kernel, values=['poly']), InCondition( child=coef0, parent=kernel, values=['poly', 'sigmoid'] ) ) ) return config
def config_space(self): """XGBoost hyperparameter space.""" # The mumber of Decision Trees. n_estimators = UniformIntegerHyperparameter( 'n_estimators', lower=10, upper=200, default_value=100 ) # The maximum depth of each decision tree. Generally, boosting # algorithms are configured with weak learners = shallow decision trees. max_depth = UniformIntegerHyperparameter( 'max_depth', lower=5, upper=500, default_value=100 ) # L1 regularization term on weights. reg_alpha = UniformFloatHyperparameter( 'reg_alpha', lower=1e-8, upper=100, default_value=1e-3 ) # L2 regularization term on weights. reg_lambda = UniformFloatHyperparameter( 'reg_lambda', lower=1e-8, upper=100, default_value=1e-3 ) learning_rate = UniformFloatHyperparameter( 'learning_rate', lower=1e-8, upper=50, default_value=0.01 ) min_data_in_leaf = UniformIntegerHyperparameter( 'min_data_in_leaf', lower=2, upper=5, default_value=3 ) # Add hyperparameters to config space. config = ConfigurationSpace() config.seed(self.random_state) config.add_hyperparameters( ( n_estimators, min_data_in_leaf, max_depth, reg_alpha, reg_lambda, learning_rate ) ) return config
def config_space(self): """KNN hyperparameter space.""" n_neighbors = UniformIntegerHyperparameter( 'n_neighbors', lower=1, upper=100, default_value=5 ) leaf_size = UniformIntegerHyperparameter( 'leaf_size', lower=1, upper=100, default_value=20 ) metric = CategoricalHyperparameter( 'metric', ['euclidean', 'manhattan', 'chebyshev', 'minkowski'], default_value='euclidean' ) p_param = UniformIntegerHyperparameter('p', 1, 5, default_value=2) # Add hyperparameters to config space. config = ConfigurationSpace() config.seed(self.random_state) config.add_hyperparameters((n_neighbors, leaf_size, metric, p_param)) # Conditionals on hyperparameters specific to kernels. config.add_condition( InCondition(child=p_param, parent=metric, values=['minkowski']) ) return config
def test_impute_inactive_hyperparameters(self): cs = ConfigurationSpace() a = cs.add_hyperparameter(CategoricalHyperparameter('a', [0, 1])) b = cs.add_hyperparameter(CategoricalHyperparameter('b', [0, 1])) c = cs.add_hyperparameter(UniformFloatHyperparameter('c', 0, 1)) cs.add_condition(EqualsCondition(b, a, 1)) cs.add_condition(EqualsCondition(c, a, 0)) cs.seed(1) configs = cs.sample_configuration(size=100) config_array = convert_configurations_to_array(configs) for line in config_array: if line[0] == 0: self.assertTrue(np.isnan(line[1])) elif line[0] == 1: self.assertTrue(np.isnan(line[2])) gp = get_gp(3, np.random.RandomState(1)) config_array = gp._impute_inactive(config_array) for line in config_array: if line[0] == 0: self.assertEqual(line[1], -1) elif line[0] == 1: self.assertEqual(line[2], -1)
class TestLocalSearch(unittest.TestCase): def setUp(self): current_dir = os.path.dirname(__file__) self.test_files_dir = os.path.join(current_dir, '..', 'test_files') seed = np.random.randint(1, 100000) self.cs = ConfigurationSpace(seed=seed) x1 = UniformFloatHyperparameter("x1", -5, 5, default_value=5) self.cs.add_hyperparameter(x1) x2 = UniformIntegerHyperparameter("x2", -5, 5, default_value=5) self.cs.add_hyperparameter(x2) x3 = CategoricalHyperparameter("x3", [5, 2, 0, 1, -1, -2, 4, -3, 3, -5, -4], default_value=5) self.cs.add_hyperparameter(x3) x4 = UniformIntegerHyperparameter("x4", -5, 5, default_value=5) self.cs.add_hyperparameter(x4) def test_local_search(self): def acquisition_function(points): rval = [] for point in points: opt = np.array([1, 1, 1, 1]) rval.append([-euclidean(point.get_array(), opt)]) return np.array(rval) ls = LocalSearch(acquisition_function, self.cs, max_steps=100) start_point = self.cs.sample_configuration() acq_val_start_point = acquisition_function([start_point]) acq_val_incumbent, _ = ls._do_search(start_point)[0] # Local search needs to find something that is as least as good as the # start point self.assertLessEqual(acq_val_start_point, acq_val_incumbent) @unittest.mock.patch.object(LocalSearch, '_get_initial_points') def test_local_search_2( self, _get_initial_points_patch, ): pcs_file = os.path.join(self.test_files_dir, "test_local_search.pcs") seed = np.random.randint(1, 100000) runhistory = unittest.mock.Mock() runhistory.data = [None] * 1000 with open(pcs_file) as fh: config_space = pcs.read(fh.readlines()) config_space.seed(seed) def acquisition_function(points): return np.array([[np.count_nonzero(point.get_array())] for point in points]) start_point = config_space.get_default_configuration() _get_initial_points_patch.return_value = [start_point] ls = LocalSearch(acquisition_function, config_space, max_steps=100000) # To have some data in a mock runhistory ls.runhistory = [None] * 1000 acq_val_incumbent, incumbent = ls._maximize(runhistory, None, 1)[0] np.testing.assert_allclose( incumbent.get_array(), np.ones(len(config_space.get_hyperparameters()))) @unittest.mock.patch.object(LocalSearch, '_do_search') @unittest.mock.patch.object(LocalSearch, '_get_initial_points') def test_get_next_by_local_search(self, _get_initial_points_patch, patch): # Without known incumbent class SideEffect(object): def __call__(self, *args, **kwargs): rval = [] for i in range(len(args[0])): rval.append((i, ConfigurationMock(i))) return rval patch.side_effect = SideEffect() cs = test_helpers.get_branin_config_space() rand_confs = cs.sample_configuration(size=9) _get_initial_points_patch.return_value = rand_confs acq_func = EI(None) ls = LocalSearch(acq_func, cs) # To have some data in a mock runhistory runhistory = unittest.mock.Mock() runhistory.data = [None] * 1000 rval = ls._maximize(runhistory, None, 9) self.assertEqual(len(rval), 9) self.assertEqual(patch.call_count, 1) for i in range(9): self.assertIsInstance(rval[i][1], ConfigurationMock) self.assertEqual(rval[i][1].value, 8 - i) self.assertEqual(rval[i][0], 8 - i) self.assertEqual(rval[i][1].origin, 'Local Search') # Check that the known 'incumbent' is transparently passed through patch.side_effect = SideEffect() _get_initial_points_patch.return_value = ['Incumbent'] + rand_confs rval = ls._maximize(runhistory, None, 10) self.assertEqual(len(rval), 10) self.assertEqual(patch.call_count, 2) # Only the first local search in each iteration starts from the # incumbent self.assertEqual(patch.call_args_list[1][0][0][0], 'Incumbent') for i in range(10): self.assertEqual(rval[i][1].origin, 'Local Search') def test_local_search_finds_minimum(self): class AcquisitionFunction: model = None def __call__(self, arrays): rval = [] for array in arrays: rval.append([-rosenbrock_4d(array)]) return np.array(rval) ls = LocalSearch( acquisition_function=AcquisitionFunction(), config_space=self.cs, n_steps_plateau_walk=10, max_steps=np.inf, ) runhistory = RunHistory() self.cs.seed(1) random_configs = self.cs.sample_configuration(size=100) costs = [ rosenbrock_4d(random_config) for random_config in random_configs ] self.assertGreater(np.min(costs), 100) for random_config, cost in zip(random_configs, costs): runhistory.add(config=random_config, cost=cost, time=0, status=StatusType.SUCCESS) minimizer = ls.maximize(runhistory, None, 10) minima = [-rosenbrock_4d(m) for m in minimizer] self.assertGreater(minima[0], -0.05) def test_get_initial_points_moo(self): class Model: def predict_marginalized_over_instances(self, X): return X, X class AcquisitionFunction: model = Model() def __call__(self, X): return np.array([x.get_array().sum() for x in X]).reshape( (-1, 1)) ls = LocalSearch( acquisition_function=AcquisitionFunction(), config_space=self.cs, n_steps_plateau_walk=10, max_steps=np.inf, ) runhistory = RunHistory() random_configs = self.cs.sample_configuration(size=100) costs = np.array( [rosenbrock_4d(random_config) for random_config in random_configs]) for random_config, cost in zip(random_configs, costs): runhistory.add(config=random_config, cost=cost, time=0, status=StatusType.SUCCESS) points = ls._get_initial_points(num_points=5, runhistory=runhistory, additional_start_points=None) self.assertEqual(len(points), 10)