def test_write_forbidden(self): cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("parent", [0, 1]) hp2 = UniformIntegerHyperparameter("child", 0, 2) hp3 = UniformIntegerHyperparameter("child2", 0, 2) hp4 = UniformIntegerHyperparameter("child3", 0, 2) hp5 = CategoricalHyperparameter("child4", [4, 5, 6, 7]) cs.add_hyperparameters([hp1, hp2, hp3, hp4, hp5]) forb2 = ForbiddenEqualsClause(hp1, 1) forb3 = ForbiddenInClause(hp2, range(2, 3)) forb4 = ForbiddenInClause(hp3, range(2, 3)) forb5 = ForbiddenInClause(hp4, range(2, 3)) forb6 = ForbiddenInClause(hp5, [6, 7]) and1 = ForbiddenAndConjunction(forb2, forb3) and2 = ForbiddenAndConjunction(forb2, forb4) and3 = ForbiddenAndConjunction(forb2, forb5) cs.add_forbidden_clauses( [forb2, forb3, forb4, forb5, forb6, and1, and2, and3]) irace.write(cs) # generates file called forbidden.txt
def test_build_new_forbidden(self): expected = "a categorical {a, b, c} [a]\nb categorical {a, b, c} [c]\n\n" \ "{a=a, b=a}\n{a=a, b=b}\n{a=b, b=a}\n{a=b, b=b}\n" cs = ConfigurationSpace() a = CategoricalHyperparameter("a", ["a", "b", "c"], "a") b = CategoricalHyperparameter("b", ["a", "b", "c"], "c") cs.add_hyperparameter(a) cs.add_hyperparameter(b) fb = ForbiddenAndConjunction(ForbiddenInClause(a, ["a", "b"]), ForbiddenInClause(b, ["a", "b"])) cs.add_forbidden_clause(fb) value = pcs_new.write(cs) self.assertEqual(expected, value)
def _construct_forbidden_in( clause: Dict, cs: ConfigurationSpace, ) -> ForbiddenEqualsClause: return ForbiddenInClause(hyperparameter=cs.get_hyperparameter( clause['name']), values=clause['values'])
def test_and_conjunction(self): hp1 = CategoricalHyperparameter("parent", [0, 1]) hp2 = UniformIntegerHyperparameter("child", 0, 2) hp3 = UniformIntegerHyperparameter("child2", 0, 2) hp4 = UniformIntegerHyperparameter("child3", 0, 2) forb2 = ForbiddenEqualsClause(hp1, 1) forb3 = ForbiddenInClause(hp2, range(2, 3)) forb4 = ForbiddenInClause(hp3, range(2, 3)) forb5 = ForbiddenInClause(hp4, range(2, 3)) and1 = ForbiddenAndConjunction(forb2, forb3) and2 = ForbiddenAndConjunction(forb2, forb4) and3 = ForbiddenAndConjunction(forb2, forb5) total_and = ForbiddenAndConjunction(and1, and2, and3) self.assertEqual( "((Forbidden: parent == 1 && Forbidden: child in {2}) " "&& (Forbidden: parent == 1 && Forbidden: child2 in {2}) " "&& (Forbidden: parent == 1 && Forbidden: child3 in " "{2}))", str(total_and)) results = [ False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, True ] for i, values in enumerate( product(range(2), range(3), range(3), range(3))): is_forbidden = total_and.is_forbidden( { "parent": values[0], "child": values[1], "child2": values[2], "child3": values[3] }, True, ) self.assertEqual(results[i], is_forbidden) self.assertFalse(total_and.is_forbidden({}, strict=False))
def set_probabilities_in_cs(self, cs: ConfigurationSpace, relied2models: Dict[str, List[str]], relied2AllModels: Dict[str, List[str]], all_models: List[str], **kwargs): estimator = cs.get_hyperparameter("estimator:__choice__") probabilities = [] model2prob = {} L = 0 for rely_model in relied2models: cur_models = relied2models[rely_model] L += len(cur_models) for model in cur_models: model2prob[model] = kwargs[rely_model] / len(cur_models) p_rest = (1 - sum(model2prob.values())) / (len(all_models) - L) for model in estimator.choices: probabilities.append(model2prob.get(model, p_rest)) estimator.probabilities = probabilities default_estimator_choice = None for models in relied2models.values(): if models: default_estimator_choice = models[0] estimator.default_value = default_estimator_choice for rely_model, path in RelyModels.info: forbid_eq_value = path[-1] path = path[:-1] forbid_eq_key = ":".join(path + ["__choice__"]) forbid_eq_key_hp = cs.get_hyperparameter(forbid_eq_key) forbid_in_key = "estimator:__choice__" hit = relied2AllModels.get(rely_model) if not hit: choices = list(forbid_eq_key_hp.choices) choices.remove(forbid_eq_value) forbid_eq_key_hp.choices = tuple(choices) forbid_eq_key_hp.default_value = choices[0] forbid_eq_key_hp.probabilities = [1 / len(choices) ] * len(choices) # fixme 最后我放弃了在这上面进行修改,在hdl部分就做了预处理 continue forbid_in_value = list(set(all_models) - set(hit)) # 只选择了boost模型 if not forbid_in_value: continue choices = forbid_eq_key_hp.choices probabilities = [] p: float = kwargs[rely_model] p_rest = (1 - p) * (len(choices) - 1) for choice in choices: if choice == forbid_eq_value: probabilities.append(p) else: probabilities.append(p_rest) forbid_eq_key_hp.probabilities = probabilities cs.add_forbidden_clause( ForbiddenAndConjunction( ForbiddenEqualsClause(forbid_eq_key_hp, forbid_eq_value), ForbiddenInClause(cs.get_hyperparameter(forbid_in_key), forbid_in_value), ))
def get_hyperparameter_search_space(**kwargs): n_clusters_factor = UniformFloatHyperparameter("n_clusters_factor", 0., 1., default_value=1.) affinity = CategoricalHyperparameter("affinity", ["euclidean", "manhattan", "cosine"], default_value="euclidean") linkage = CategoricalHyperparameter("linkage", ["ward", "complete", "average"], default_value="ward") pooling_func = CategoricalHyperparameter("pooling_func", ["mean", "median", "max"], default_value="mean") cs = ConfigurationSpace() cs.add_hyperparameters([n_clusters_factor, affinity, linkage, pooling_func]) affinity_and_linkage = ForbiddenAndConjunction(ForbiddenInClause(affinity, ["manhattan", "cosine"]), ForbiddenEqualsClause(linkage, "ward")) cs.add_forbidden_clause(affinity_and_linkage) return cs
def __forbidden(self, value: List, store: Dict, cs: ConfigurationSpace): assert isinstance(value, list) for item in value: assert isinstance(item, dict) clauses = [] for k, v in item.items(): if isinstance(v, list) and len(v) == 1: v = v[0] if isinstance(v, list): clauses.append( ForbiddenInClause(store[k], list(map(smac_hdl._encode, v)))) else: clauses.append( ForbiddenEqualsClause(store[k], smac_hdl._encode(v))) cs.add_forbidden_clause(ForbiddenAndConjunction(*clauses))
def get_hyperparameter_search_space(dataset_properties=None, optimizer='smac'): cs = ConfigurationSpace() n_clusters = UniformIntegerHyperparameter("n_clusters", 2, 400, default_value=25) affinity = CategoricalHyperparameter( "affinity", ["euclidean", "manhattan", "cosine"], default_value="euclidean") linkage = CategoricalHyperparameter( "linkage", ["ward", "complete", "average"], default_value="ward") pooling_func = CategoricalHyperparameter( "pooling_func", ["mean", "median", "max"], default_value="mean") cs.add_hyperparameters([n_clusters, affinity, linkage, pooling_func]) affinity_and_linkage = ForbiddenAndConjunction( ForbiddenInClause(affinity, ["manhattan", "cosine"]), ForbiddenEqualsClause(linkage, "ward")) cs.add_forbidden_clause(affinity_and_linkage) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() n_clusters = cs.add_hyperparameter( UniformIntegerHyperparameter("n_clusters", 2, 400, 25)) affinity = cs.add_hyperparameter( CategoricalHyperparameter("affinity", ["euclidean", "manhattan", "cosine"], "euclidean")) linkage = cs.add_hyperparameter( CategoricalHyperparameter("linkage", ["ward", "complete", "average"], "ward")) pooling_func = cs.add_hyperparameter( CategoricalHyperparameter("pooling_func", ["mean", "median", "max"])) affinity_and_linkage = ForbiddenAndConjunction( ForbiddenInClause(affinity, ["manhattan", "cosine"]), ForbiddenEqualsClause(linkage, "ward")) cs.add_forbidden_clause(affinity_and_linkage) return cs
def test_in_condition(self): hp1 = CategoricalHyperparameter("parent", [0, 1]) hp2 = UniformIntegerHyperparameter("child", 0, 10) hp3 = UniformIntegerHyperparameter("child2", 0, 10) self.assertRaisesRegexp( TypeError, "Argument 'hyperparameter' is not of" " type <class 'ConfigSpace.hyperparameters.Hyperparameter'>.", ForbiddenInClause, "HP1", 1) self.assertRaisesRegexp( ValueError, "Forbidden clause must be instantiated with a " "legal hyperparameter value for " "'parent, Type: Categorical, Choices: {0, 1}, " "Default: 0', but got '2'", ForbiddenInClause, hp1, [2]) forb1 = ForbiddenInClause(hp2, [5, 6, 7, 8, 9]) forb1_ = ForbiddenInClause(hp2, [9, 8, 7, 6, 5]) forb2 = ForbiddenInClause(hp2, [5, 6, 7, 8]) forb3 = ForbiddenInClause(hp3, [5, 6, 7, 8, 9]) self.assertEqual(forb1, forb1_) self.assertNotEqual(forb1, forb2) self.assertNotEqual(forb1, forb3) self.assertEqual("Forbidden: child in {5, 6, 7, 8, 9}", str(forb1)) self.assertRaisesRegexp( ValueError, "Is_forbidden must be called with the " "instanstatiated hyperparameter in the " "forbidden clause; you are missing " "'child'", forb1.is_forbidden, {'parent': 1}) self.assertFalse(forb1.is_forbidden({'parent': 1}, strict=False)) for i in range(0, 5): self.assertFalse(forb1.is_forbidden({'child': i})) for i in range(5, 10): self.assertTrue(forb1.is_forbidden({'child': i}))
def test_in_condition(self): hp1 = CategoricalHyperparameter("parent", [0, 1, 2, 3, 4]) hp2 = UniformIntegerHyperparameter("child", 0, 10) hp3 = UniformIntegerHyperparameter("child2", 0, 10) hp4 = CategoricalHyperparameter("grandchild", ["hot", "cold", "warm"]) self.assertRaisesRegex( ValueError, "Forbidden clause must be instantiated with a " "legal hyperparameter value for " "'parent, Type: Categorical, Choices: {0, 1, 2, 3, 4}, " "Default: 0', but got '5'", ForbiddenInClause, hp1, [5], ) forb1 = ForbiddenInClause(hp2, [5, 6, 7, 8, 9]) forb1_ = ForbiddenInClause(hp2, [9, 8, 7, 6, 5]) forb2 = ForbiddenInClause(hp2, [5, 6, 7, 8]) forb3 = ForbiddenInClause(hp3, [5, 6, 7, 8, 9]) forb4 = ForbiddenInClause(hp4, ["hot", "cold"]) forb4_ = ForbiddenInClause(hp4, ["hot", "cold"]) forb5 = ForbiddenInClause(hp1, [3, 4]) forb5_ = ForbiddenInClause(hp1, [3, 4]) self.assertEqual(forb5, forb5_) self.assertEqual(forb4, forb4_) # print("\nTest1:") self.assertEqual(forb1, forb1_) # print("\nTest2:") self.assertNotEqual(forb1, forb2) # print("\nTest3:") self.assertNotEqual(forb1, forb3) # print("\nTest4:") self.assertEqual("Forbidden: child in {5, 6, 7, 8, 9}", str(forb1)) # print("\nTest5:") self.assertRaisesRegex( ValueError, "Is_forbidden must be called with the " "instanstatiated hyperparameter in the " "forbidden clause; you are missing " "'child'", forb1.is_forbidden, {'parent': 1}, True) # print("\nTest6:") self.assertFalse(forb1.is_forbidden({'parent': 1}, strict=False)) # print("\nTest7:") for i in range(0, 5): self.assertFalse(forb1.is_forbidden({'child': i}, True)) # print("\nTest8:") for i in range(5, 10): self.assertTrue(forb1.is_forbidden({'child': i}, True)) # Test forbidden on vector values hyperparameter_idx = {hp1.name: 0, hp2.name: 1} forb1.set_vector_idx(hyperparameter_idx) # print("\nTest9:") self.assertFalse( forb1.is_forbidden_vector(np.array([np.NaN, np.NaN]), strict=False)) # print("\nTest10:") self.assertFalse( forb1.is_forbidden_vector(np.array([np.NaN, 0]), strict=False)) correct_vector_value = hp2._inverse_transform(6) # print("\nTest11:") # print(correct_vector_value, np.array([np.NaN, correct_vector_value])) self.assertTrue( forb1.is_forbidden_vector(np.array([np.NaN, correct_vector_value]), strict=False))
def test_in_condition(self): hp1 = CategoricalHyperparameter("parent", [0, 1, 2, 3, 4]) hp2 = UniformIntegerHyperparameter("child", 0, 10) hp3 = UniformIntegerHyperparameter("child2", 0, 10) hp4 = CategoricalHyperparameter("grandchild", ["hot", "cold", "warm"]) self.assertRaisesRegex( ValueError, "Forbidden clause must be instantiated with a " "legal hyperparameter value for " "'parent, Type: Categorical, Choices: {0, 1, 2, 3, 4}, " "Default: 0', but got '5'", ForbiddenInClause, hp1, [5], ) forb1 = ForbiddenInClause(hp2, [5, 6, 7, 8, 9]) forb1_ = ForbiddenInClause(hp2, [9, 8, 7, 6, 5]) forb2 = ForbiddenInClause(hp2, [5, 6, 7, 8]) forb3 = ForbiddenInClause(hp3, [5, 6, 7, 8, 9]) forb4 = ForbiddenInClause(hp4, ["hot", "cold"]) forb4_ = ForbiddenInClause(hp4, ["hot", "cold"]) forb5 = ForbiddenInClause(hp1, [3, 4]) forb5_ = ForbiddenInClause(hp1, [3, 4]) self.assertEqual(forb5, forb5_) self.assertEqual(forb4, forb4_) # print("\nTest1:") self.assertEqual(forb1, forb1_) # print("\nTest2:") self.assertNotEqual(forb1, forb2) # print("\nTest3:") self.assertNotEqual(forb1, forb3) # print("\nTest4:") self.assertEqual("Forbidden: child in {5, 6, 7, 8, 9}", str(forb1)) # print("\nTest5:") self.assertRaisesRegex(ValueError, "Is_forbidden must be called with the " "instanstatiated hyperparameter in the " "forbidden clause; you are missing " "'child'", forb1.is_forbidden, {'parent': 1}, True) # print("\nTest6:") self.assertFalse(forb1.is_forbidden({'parent': 1}, strict=False)) # print("\nTest7:") for i in range(0, 5): self.assertFalse(forb1.is_forbidden({'child': i}, True)) # print("\nTest8:") for i in range(5, 10): self.assertTrue(forb1.is_forbidden({'child': i}, True)) # Test forbidden on vector values hyperparameter_idx = { hp1.name: 0, hp2.name: 1 } forb1.set_vector_idx(hyperparameter_idx) # print("\nTest9:") self.assertFalse(forb1.is_forbidden_vector(np.array([np.NaN, np.NaN]), strict=False)) # print("\nTest10:") self.assertFalse(forb1.is_forbidden_vector(np.array([np.NaN, 0]), strict=False)) correct_vector_value = hp2._inverse_transform(6) # print("\nTest11:") # print(correct_vector_value, np.array([np.NaN, correct_vector_value])) self.assertTrue(forb1.is_forbidden_vector(np.array([np.NaN, correct_vector_value]), strict=False))