def test_impute_inactive_hyperparameters(self): cs = ConfigurationSpace() a = cs.add_hyperparameter(CategoricalHyperparameter('a', [0, 1])) b = cs.add_hyperparameter(CategoricalHyperparameter('b', [0, 1])) c = cs.add_hyperparameter(UniformFloatHyperparameter('c', 0, 1)) cs.add_condition(EqualsCondition(b, a, 1)) cs.add_condition(EqualsCondition(c, a, 0)) cs.seed(1) configs = cs.sample_configuration(size=100) config_array = convert_configurations_to_array(configs) for line in config_array: if line[0] == 0: self.assertTrue(np.isnan(line[1])) elif line[0] == 1: self.assertTrue(np.isnan(line[2])) gp = get_gp(3, np.random.RandomState(1)) config_array = gp._impute_inactive(config_array) for line in config_array: if line[0] == 0: self.assertEqual(line[1], -1) elif line[0] == 1: self.assertEqual(line[2], -1)
class TestLocalSearch(unittest.TestCase): def setUp(self): current_dir = os.path.dirname(__file__) self.test_files_dir = os.path.join(current_dir, '..', 'test_files') seed = np.random.randint(1, 100000) self.cs = ConfigurationSpace(seed=seed) x1 = UniformFloatHyperparameter("x1", -5, 5, default_value=5) self.cs.add_hyperparameter(x1) x2 = UniformIntegerHyperparameter("x2", -5, 5, default_value=5) self.cs.add_hyperparameter(x2) x3 = CategoricalHyperparameter("x3", [5, 2, 0, 1, -1, -2, 4, -3, 3, -5, -4], default_value=5) self.cs.add_hyperparameter(x3) x4 = UniformIntegerHyperparameter("x4", -5, 5, default_value=5) self.cs.add_hyperparameter(x4) def test_local_search(self): def acquisition_function(points): rval = [] for point in points: opt = np.array([1, 1, 1, 1]) rval.append([-euclidean(point.get_array(), opt)]) return np.array(rval) ls = LocalSearch(acquisition_function, self.cs, max_steps=100) start_point = self.cs.sample_configuration() acq_val_start_point = acquisition_function([start_point]) acq_val_incumbent, _ = ls._do_search(start_point)[0] # Local search needs to find something that is as least as good as the # start point self.assertLessEqual(acq_val_start_point, acq_val_incumbent) @unittest.mock.patch.object(LocalSearch, '_get_initial_points') def test_local_search_2( self, _get_initial_points_patch, ): pcs_file = os.path.join(self.test_files_dir, "test_local_search.pcs") seed = np.random.randint(1, 100000) runhistory = unittest.mock.Mock() runhistory.data = [None] * 1000 with open(pcs_file) as fh: config_space = pcs.read(fh.readlines()) config_space.seed(seed) def acquisition_function(points): return np.array([[np.count_nonzero(point.get_array())] for point in points]) start_point = config_space.get_default_configuration() _get_initial_points_patch.return_value = [start_point] ls = LocalSearch(acquisition_function, config_space, max_steps=100000) # To have some data in a mock runhistory ls.runhistory = [None] * 1000 acq_val_incumbent, incumbent = ls._maximize(runhistory, None, 1)[0] np.testing.assert_allclose( incumbent.get_array(), np.ones(len(config_space.get_hyperparameters()))) @unittest.mock.patch.object(LocalSearch, '_do_search') @unittest.mock.patch.object(LocalSearch, '_get_initial_points') def test_get_next_by_local_search(self, _get_initial_points_patch, patch): # Without known incumbent class SideEffect(object): def __call__(self, *args, **kwargs): rval = [] for i in range(len(args[0])): rval.append((i, ConfigurationMock(i))) return rval patch.side_effect = SideEffect() cs = test_helpers.get_branin_config_space() rand_confs = cs.sample_configuration(size=9) _get_initial_points_patch.return_value = rand_confs acq_func = EI(None) ls = LocalSearch(acq_func, cs) # To have some data in a mock runhistory runhistory = unittest.mock.Mock() runhistory.data = [None] * 1000 rval = ls._maximize(runhistory, None, 9) self.assertEqual(len(rval), 9) self.assertEqual(patch.call_count, 1) for i in range(9): self.assertIsInstance(rval[i][1], ConfigurationMock) self.assertEqual(rval[i][1].value, 8 - i) self.assertEqual(rval[i][0], 8 - i) self.assertEqual(rval[i][1].origin, 'Local Search') # Check that the known 'incumbent' is transparently passed through patch.side_effect = SideEffect() _get_initial_points_patch.return_value = ['Incumbent'] + rand_confs rval = ls._maximize(runhistory, None, 10) self.assertEqual(len(rval), 10) self.assertEqual(patch.call_count, 2) # Only the first local search in each iteration starts from the # incumbent self.assertEqual(patch.call_args_list[1][0][0][0], 'Incumbent') for i in range(10): self.assertEqual(rval[i][1].origin, 'Local Search') def test_local_search_finds_minimum(self): class AcquisitionFunction: model = None def __call__(self, arrays): rval = [] for array in arrays: rval.append([-rosenbrock_4d(array)]) return np.array(rval) ls = LocalSearch( acquisition_function=AcquisitionFunction(), config_space=self.cs, n_steps_plateau_walk=10, max_steps=np.inf, ) runhistory = RunHistory() self.cs.seed(1) random_configs = self.cs.sample_configuration(size=100) costs = [ rosenbrock_4d(random_config) for random_config in random_configs ] self.assertGreater(np.min(costs), 100) for random_config, cost in zip(random_configs, costs): runhistory.add(config=random_config, cost=cost, time=0, status=StatusType.SUCCESS) minimizer = ls.maximize(runhistory, None, 10) minima = [-rosenbrock_4d(m) for m in minimizer] self.assertGreater(minima[0], -0.05) def test_get_initial_points_moo(self): class Model: def predict_marginalized_over_instances(self, X): return X, X class AcquisitionFunction: model = Model() def __call__(self, X): return np.array([x.get_array().sum() for x in X]).reshape( (-1, 1)) ls = LocalSearch( acquisition_function=AcquisitionFunction(), config_space=self.cs, n_steps_plateau_walk=10, max_steps=np.inf, ) runhistory = RunHistory() random_configs = self.cs.sample_configuration(size=100) costs = np.array( [rosenbrock_4d(random_config) for random_config in random_configs]) for random_config, cost in zip(random_configs, costs): runhistory.add(config=random_config, cost=cost, time=0, status=StatusType.SUCCESS) points = ls._get_initial_points(num_points=5, runhistory=runhistory, additional_start_points=None) self.assertEqual(len(points), 10)
class TestLocalSearch(unittest.TestCase): def setUp(self): current_dir = os.path.dirname(__file__) self.test_files_dir = os.path.join(current_dir, '..', 'test_files') seed = np.random.randint(1, 100000) self.cs = ConfigurationSpace(seed=seed) x1 = UniformFloatHyperparameter("x1", -5, 5, default_value=5) self.cs.add_hyperparameter(x1) x2 = UniformIntegerHyperparameter("x2", -5, 5, default_value=5) self.cs.add_hyperparameter(x2) x3 = CategoricalHyperparameter("x3", [5, 2, 0, 1, -1, -2, 4, -3, 3, -5, -4], default_value=5) self.cs.add_hyperparameter(x3) x4 = UniformIntegerHyperparameter("x4", -5, 5, default_value=5) self.cs.add_hyperparameter(x4) def test_local_search(self): def acquisition_function(point): point = [p.get_array() for p in point] opt = np.array([1, 1, 1, 1]) dist = [euclidean(point, opt)] return np.array([-np.min(dist)]) l = LocalSearch(acquisition_function, self.cs, epsilon=1e-10, max_iterations=100000) start_point = self.cs.sample_configuration() acq_val_start_point = acquisition_function([start_point]) acq_val_incumbent, _ = l._one_iter(start_point) # Local search needs to find something that is as least as good as the # start point self.assertLessEqual(acq_val_start_point, acq_val_incumbent) @unittest.mock.patch.object(LocalSearch, '_get_initial_points') @unittest.mock.patch.object(LocalSearch, '_calculate_num_points') def test_local_search_2( self, _calculate_num_points_patch, _get_initial_points_patch, ): pcs_file = os.path.join(self.test_files_dir, "test_local_search.pcs") seed = np.random.randint(1, 100000) with open(pcs_file) as fh: config_space = pcs.read(fh.readlines()) config_space.seed(seed) def acquisition_function(point): return np.array([np.count_nonzero(point[0].get_array())]) start_point = config_space.get_default_configuration() _calculate_num_points_patch.return_value = 1 _get_initial_points_patch.return_value = [start_point] l = LocalSearch(acquisition_function, config_space, epsilon=0.01, max_iterations=100000) acq_val_incumbent, incumbent = l._maximize(None, None, 10)[0] np.testing.assert_allclose( incumbent.get_array(), np.ones(len(config_space.get_hyperparameters()))) @unittest.mock.patch.object(LocalSearch, '_one_iter') @unittest.mock.patch.object(LocalSearch, '_get_initial_points') @unittest.mock.patch.object(LocalSearch, '_calculate_num_points') def test_get_next_by_local_search(self, _calculate_num_points_patch, _get_initial_points_patch, patch): # Without known incumbent class SideEffect(object): def __init__(self): self.call_number = 0 def __call__(self, *args, **kwargs): rval = 9 - self.call_number self.call_number += 1 return (rval, ConfigurationMock(rval)) patch.side_effect = SideEffect() cs = test_helpers.get_branin_config_space() rand_confs = cs.sample_configuration(size=9) _get_initial_points_patch.return_value = rand_confs _calculate_num_points_patch.return_value = 9 acq_func = EI(None) ls = LocalSearch(acq_func, cs) rval = ls._maximize(None, None, 9) self.assertEqual(len(rval), 9) self.assertEqual(patch.call_count, 9) for i in range(9): self.assertIsInstance(rval[i][1], ConfigurationMock) self.assertEqual(rval[i][1].value, 9 - i) self.assertEqual(rval[i][0], 9 - i) self.assertEqual(rval[i][1].origin, 'Local Search') # With known incumbent patch.side_effect = SideEffect() _get_initial_points_patch.return_value = ['Incumbent'] + rand_confs _calculate_num_points_patch.return_value = 10 rval = ls._maximize(None, None, 10) self.assertEqual(len(rval), 10) self.assertEqual(patch.call_count, 19) # Only the first local search in each iteration starts from the # incumbent self.assertEqual(patch.call_args_list[9][0][0], 'Incumbent') for i in range(10): self.assertEqual(rval[i][1].origin, 'Local Search')
class TestLocalSearch(unittest.TestCase): def setUp(self): current_dir = os.path.dirname(__file__) self.test_files_dir = os.path.join(current_dir, '..', 'test_files') seed = np.random.randint(1, 100000) self.cs = ConfigurationSpace(seed=seed) x1 = UniformFloatHyperparameter("x1", -5, 5, default=5) self.cs.add_hyperparameter(x1) x2 = UniformIntegerHyperparameter("x2", -5, 5, default=5) self.cs.add_hyperparameter(x2) x3 = CategoricalHyperparameter("x3", [5, 2, 0, 1, -1, -2, 4, -3, 3, -5, -4], default=5) self.cs.add_hyperparameter(x3) x4 = UniformIntegerHyperparameter("x4", -5, 5, default=5) self.cs.add_hyperparameter(x4) def test_local_search(self): def acquisition_function(point): opt = np.array([1, 1, 1, 1]) dist = [euclidean(point, opt)] return -np.min(dist) l = LocalSearch(acquisition_function, self.cs, epsilon=1e-10, max_iterations=100000) start_point = self.cs.sample_configuration() acq_val_start_point = acquisition_function(start_point.get_array()) _, acq_val_incumbent = l.maximize(start_point) # Local search needs to find something that is as least as good as the # start point self.assertLessEqual(acq_val_start_point, acq_val_incumbent) def test_local_search_2(self): pcs_file = os.path.join(self.test_files_dir, "test_local_search.pcs") seed = np.random.randint(1, 100000) with open(pcs_file) as fh: config_space = pcs.read(fh.readlines()) config_space.seed(seed) def acquisition_function(point): return np.count_nonzero(np.array(point)) start_point = config_space.get_default_configuration() l = LocalSearch(acquisition_function, config_space, epsilon=0.01, max_iterations=100000) incumbent, acq_val_incumbent = l.maximize(start_point) self.assertEqual(acq_val_incumbent, len(start_point.get_array())) self.assertTrue( np.all(incumbent.get_array() == np.ones([acq_val_incumbent])))
## step3_child__xgboost_eta = InCondition(child=hyper_xgboost_eta, parent=step3, values=["xgboost"]) step3_child__xgboost_max_depth = InCondition(child=hyper_xgboost_max_depth, parent=step3, values=["xgboost"]) step3_child__xgboost_subsample = InCondition(child=hyper_xgboost_subsample, parent=step3, values=["xgboost"]) step3_child__xgboost_colsample_bytree = InCondition( child=hyper_xgboost_colsample_bytree, parent=step3, values=["xgboost"]) step3_child__xgboost_min_child_weight = InCondition( child=hyper_xgboost_min_child_weight, parent=step3, values=["xgboost"]) ## step3_child__naiveBayes_laplace = InCondition(child=hyper_naiveBayes, parent=step3, values=["naiveBayes"]) cs.add_conditions([ step3_child_kknn, step3_child_ksvm_c, step3_child_ksvm_sigma, step3_child_ranger_mtry, step3_child_ranger_sample_fraction, step3_child__xgboost_eta, step3_child__xgboost_subsample, step3_child__xgboost_max_depth, step3_child__xgboost_colsample_bytree, step3_child__xgboost_min_child_weight, step3_child__naiveBayes_laplace ]) cfg = cs.sample_configuration() stub = {k: cfg[k] for k in cfg if cfg[k]}