def get_hyperparameter_search_space(self, mf: MetaFeaturesDict = None,
default=None,
include=None,
exclude=None,
**kwargs):
if include is not None and exclude is not None:
raise ValueError("The arguments include and exclude cannot be used together.")
cs = ConfigurationSpace()
# Compile a list of all estimator objects for this problem
available_estimators = self.get_available_components(mf=mf, include=include, exclude=exclude)
if len(available_estimators) == 0:
raise ValueError("No classifiers found")
if default is None:
for default_ in self.defaults:
if default_ in available_estimators:
if include is not None and default_ not in include:
continue
if exclude is not None and default_ in exclude:
continue
default = default_
break
estimator = CategoricalHyperparameter('__choice__', list(available_estimators.keys()), default_value=default)
cs.add_hyperparameter(estimator)
for estimator_name in available_estimators.keys():
estimator_configuration_space = available_estimators[estimator_name].get_hyperparameter_search_space()
parent_hyperparameter = {'parent': estimator, 'value': estimator_name}
cs.add_configuration_space(estimator_name, estimator_configuration_space,
parent_hyperparameter=parent_hyperparameter)
self.configuration_space_ = cs
return cs
def get_combined_cs(estimator_id,
task_type=REGRESSION,
include_image=False,
include_text=False,
include_preprocessors=None,
if_imbal=False):
cs = ConfigurationSpace()
hpo_cs = get_hpo_cs(estimator_id, task_type)
fe_cs = get_fe_cs(estimator_id,
task_type,
include_image=include_image,
include_text=include_text,
include_preprocessors=include_preprocessors)
config_cand = [estimator_id]
config_option = CategoricalHyperparameter('hpo', config_cand)
cs.add_hyperparameter(config_option)
for config_item in config_cand:
sub_configuration_space = hpo_cs
parent_hyperparameter = {'parent': config_option, 'value': config_item}
cs.add_configuration_space(config_item,
sub_configuration_space,
parent_hyperparameter=parent_hyperparameter)
for hp in fe_cs.get_hyperparameters():
cs.add_hyperparameter(hp)
for cond in fe_cs.get_conditions():
cs.add_condition(cond)
for bid in fe_cs.get_forbiddens():
cs.add_forbidden_clause(bid)
return cs
def get_pipeline_config_space(self, algorithm_candidates):
cs = ConfigurationSpace()
estimator_choice = CategoricalHyperparameter(
"estimator",
algorithm_candidates,
default_value=algorithm_candidates[0])
cs.add_hyperparameter(estimator_choice)
if self.task_type == IMG_CLS:
aug_space = get_aug_hyperparameter_space()
cs.add_hyperparameters(aug_space.get_hyperparameters())
cs.add_conditions(aug_space.get_conditions())
for estimator_id in algorithm_candidates:
sub_cs = self.get_model_config_space(estimator_id,
include_estimator=False,
include_aug=False)
parent_hyperparameter = {
'parent': estimator_choice,
'value': estimator_id
}
cs.add_configuration_space(
estimator_id,
sub_cs,
parent_hyperparameter=parent_hyperparameter)
return cs
def get_child_hyperparameter_search_space(children: List[Tuple[str, EstimatorComponent]],
mf: Optional[MetaFeaturesDict] = None) -> ConfigurationSpace:
cs = ConfigurationSpace()
for name, step in children:
try:
step_configuration_space = step.get_hyperparameter_search_space(mf=mf)
except TypeError:
step_configuration_space = step.get_hyperparameter_search_space()
except AttributeError:
# Provided step does not provide a configuration space the expected way.
# Simply pretend it is not configurable
step_configuration_space = ConfigurationSpace()
cs.add_configuration_space(name, step_configuration_space)
return cs
def get_cash_cs(include_algorithms=None, task_type=CLASSIFICATION):
_candidates = get_combined_candidtates(_classifiers, _addons)
if include_algorithms is not None:
_candidates = set(include_algorithms).intersection(set(_candidates.keys()))
if len(_candidates) == 0:
raise ValueError("No algorithms included! Please check the spelling of the included algorithms!")
cs = ConfigurationSpace()
algo = CategoricalHyperparameter('algorithm', list(_candidates))
cs.add_hyperparameter(algo)
for estimator_id in _candidates:
estimator_cs = get_hpo_cs(estimator_id)
parent_hyperparameter = {'parent': algo,
'value': estimator_id}
cs.add_configuration_space(estimator_id, estimator_cs, parent_hyperparameter=parent_hyperparameter)
return cs
def test_meta_field(self):
cs = ConfigurationSpace()
cs.add_hyperparameter(
UniformIntegerHyperparameter("uihp",
lower=1,
upper=10,
meta=dict(uihp=True)))
cs.add_hyperparameter(
NormalIntegerHyperparameter("nihp",
mu=0,
sigma=1,
meta=dict(nihp=True)))
cs.add_hyperparameter(
UniformFloatHyperparameter("ufhp",
lower=1,
upper=10,
meta=dict(ufhp=True)))
cs.add_hyperparameter(
NormalFloatHyperparameter("nfhp",
mu=0,
sigma=1,
meta=dict(nfhp=True)))
cs.add_hyperparameter(
CategoricalHyperparameter("chp",
choices=['1', '2', '3'],
meta=dict(chp=True)))
cs.add_hyperparameter(
OrdinalHyperparameter("ohp",
sequence=['1', '2', '3'],
meta=dict(ohp=True)))
cs.add_hyperparameter(Constant("const", value=1,
meta=dict(const=True)))
parent = ConfigurationSpace()
parent.add_configuration_space("sub", cs, delimiter=':')
self.assertEqual(
parent.get_hyperparameter("sub:uihp").meta, dict(uihp=True))
self.assertEqual(
parent.get_hyperparameter("sub:nihp").meta, dict(nihp=True))
self.assertEqual(
parent.get_hyperparameter("sub:ufhp").meta, dict(ufhp=True))
self.assertEqual(
parent.get_hyperparameter("sub:nfhp").meta, dict(nfhp=True))
self.assertEqual(
parent.get_hyperparameter("sub:chp").meta, dict(chp=True))
self.assertEqual(
parent.get_hyperparameter("sub:ohp").meta, dict(ohp=True))
self.assertEqual(
parent.get_hyperparameter("sub:const").meta, dict(const=True))
def _generate_pipeline(self, depth: int) -> \
Tuple[ConfigurationSpace, List[Tuple[str, EstimatorComponent]]]:
cs = ConfigurationSpace()
steps = []
i = 0
while i < depth:
name = f'step_{i}'
clazz = random.sample(self.candidates, 1)[0]
instance = clazz()
steps.append((name, instance))
cs.add_configuration_space(
name, instance.get_hyperparameter_search_space())
i += 1
return cs, steps
def test_subspace_switches(self):
# create a switch to select one of two algorithms
algo_switch = CategoricalHyperparameter(
name="switch",
choices=["algo1", "algo2"],
weights=[0.25, 0.75],
default_value="algo1"
)
# create sub-configuration space for algorithm 1
algo1_cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter(name="algo1_param1", choices=["A", "B"], weights=[0.3, 0.7], default_value="B")
algo1_cs.add_hyperparameter(hp1)
# create sub-configuration space for algorithm 2
algo2_cs = ConfigurationSpace()
hp2 = CategoricalHyperparameter(name="algo2_param1", choices=["X", "Y"], default_value="Y")
algo2_cs.add_hyperparameter(hp2)
# create a configuration space and populate it with both the switch and the two sub-configuration spaces
cs = ConfigurationSpace()
cs.add_hyperparameter(algo_switch)
cs.add_configuration_space(
prefix="algo1_subspace",
configuration_space=algo1_cs,
parent_hyperparameter={'parent': algo_switch, 'value': "algo1"}
)
cs.add_configuration_space(
prefix="algo2_subspace",
configuration_space=algo2_cs,
parent_hyperparameter={'parent': algo_switch, 'value': "algo2"}
)
# check choices in the final configuration space
self.assertEqual(("algo1", "algo2"), cs.get_hyperparameter("switch").choices)
self.assertEqual(("A", "B"), cs.get_hyperparameter("algo1_subspace:algo1_param1").choices)
self.assertEqual(("X", "Y"), cs.get_hyperparameter("algo2_subspace:algo2_param1").choices)
# check probabilities in the final configuration space
self.assertEqual((0.25, 0.75), cs.get_hyperparameter("switch").probabilities)
self.assertEqual((0.3, 0.7), cs.get_hyperparameter("algo1_subspace:algo1_param1").probabilities)
self.assertEqual(None, cs.get_hyperparameter("algo2_subspace:algo2_param1").probabilities)
# check default values in the final configuration space
self.assertEqual("algo1", cs.get_hyperparameter("switch").default_value)
self.assertEqual("B", cs.get_hyperparameter("algo1_subspace:algo1_param1").default_value)
self.assertEqual("Y", cs.get_hyperparameter("algo2_subspace:algo2_param1").default_value)
def test_add_configuration_space(self):
cs = ConfigurationSpace()
hp1 = cs.add_hyperparameter(CategoricalHyperparameter("input1", [0, 1]))
cs.add_forbidden_clause(ForbiddenEqualsClause(hp1, 1))
hp2 = cs.add_hyperparameter(UniformIntegerHyperparameter("child", 0, 10))
cs.add_condition(EqualsCondition(hp2, hp1, 0))
cs2 = ConfigurationSpace()
cs2.add_configuration_space('prefix', cs, delimiter='__')
self.assertEqual(str(cs2), '''Configuration space object:
Hyperparameters:
prefix__child, Type: UniformInteger, Range: [0, 10], Default: 5
prefix__input1, Type: Categorical, Choices: {0, 1}, Default: 0
Conditions:
prefix__child | prefix__input1 == 0
Forbidden Clauses:
Forbidden: prefix__input1 == 1
''')
def test_add_configuration_space(self):
cs = ConfigurationSpace()
hp1 = cs.add_hyperparameter(CategoricalHyperparameter("input1", [0, 1]))
cs.add_forbidden_clause(ForbiddenEqualsClause(hp1, 1))
hp2 = cs.add_hyperparameter(UniformIntegerHyperparameter("child", 0, 10))
cs.add_condition(EqualsCondition(hp2, hp1, 0))
cs2 = ConfigurationSpace()
cs2.add_configuration_space('prefix', cs, delimiter='__')
self.assertEqual(str(cs2), '''Configuration space object:
Hyperparameters:
prefix__child, Type: UniformInteger, Range: [0, 10], Default: 5
prefix__input1, Type: Categorical, Choices: {0, 1}, Default: 0
Conditions:
prefix__child | prefix__input1 == 0
Forbidden Clauses:
Forbidden: prefix__input1 == 1
''')
def add_configuration_space(self,
cs: ConfigurationSpace,
cs_name: str,
hdl_value: dict,
is_choice: bool,
option_hp: Configuration,
children_is_choice=False):
if is_choice:
cs.add_configuration_space(cs_name,
self.recursion(hdl_value,
children_is_choice),
parent_hyperparameter={
"parent": option_hp,
"value": cs_name
})
else:
cs.add_configuration_space(
cs_name, self.recursion(hdl_value, children_is_choice))
def test_add_configuration_space_conjunctions(self):
cs1 = ConfigurationSpace()
cs2 = ConfigurationSpace()
hp1 = cs1.add_hyperparameter(CategoricalHyperparameter("input1", [0, 1]))
hp2 = cs1.add_hyperparameter(CategoricalHyperparameter("input2", [0, 1]))
hp3 = cs1.add_hyperparameter(UniformIntegerHyperparameter("child1", 0, 10))
hp4 = cs1.add_hyperparameter(UniformIntegerHyperparameter("child2", 0, 10))
cond1 = EqualsCondition(hp2, hp3, 0)
cond2 = EqualsCondition(hp1, hp3, 5)
cond3 = EqualsCondition(hp1, hp4, 1)
andCond = AndConjunction(cond2, cond3)
cs1.add_conditions([cond1, andCond])
cs2.add_configuration_space(prefix='test', configuration_space=cs1)
self.assertEqual(str(cs2).count('test:'), 10)
# Check that they're equal except for the "test:" prefix
self.assertEqual(str(cs1), str(cs2).replace('test:', ''))
def test_add_configuration_space_conjunctions(self):
cs1 = ConfigurationSpace()
cs2 = ConfigurationSpace()
hp1 = cs1.add_hyperparameter(CategoricalHyperparameter("input1", [0, 1]))
hp2 = cs1.add_hyperparameter(CategoricalHyperparameter("input2", [0, 1]))
hp3 = cs1.add_hyperparameter(UniformIntegerHyperparameter("child1", 0, 10))
hp4 = cs1.add_hyperparameter(UniformIntegerHyperparameter("child2", 0, 10))
cond1 = EqualsCondition(hp2, hp3, 0)
cond2 = EqualsCondition(hp1, hp3, 5)
cond3 = EqualsCondition(hp1, hp4, 1)
andCond = AndConjunction(cond2, cond3)
cs1.add_conditions([cond1, andCond])
cs2.add_configuration_space(prefix='test', configuration_space=cs1)
self.assertEqual(str(cs2).count('test:'), 10)
# Check that they're equal except for the "test:" prefix
self.assertEqual(str(cs1), str(cs2).replace('test:', ''))
def get_combined_cs(estimator_id, node, task_type=0):
cs = ConfigurationSpace()
hpo_cs = get_hpo_cs(estimator_id, task_type)
fe_cs = get_fe_cs(estimator_id, node, task_type)
config_cand = ['placeholder']
config_option = CategoricalHyperparameter('hpo', config_cand)
cs.add_hyperparameter(config_option)
for config_item in config_cand:
sub_configuration_space = hpo_cs
parent_hyperparameter = {'parent': config_option, 'value': config_item}
cs.add_configuration_space(config_item,
sub_configuration_space,
parent_hyperparameter=parent_hyperparameter)
for hp in fe_cs.get_hyperparameters():
cs.add_hyperparameter(hp)
for cond in fe_cs.get_conditions():
cs.add_condition(cond)
for bid in fe_cs.get_forbiddens():
cs.add_forbidden_clause(bid)
model = UnParametrizedHyperparameter("estimator", estimator_id)
cs.add_hyperparameter(model)
return cs
def test_add_configuration_space_probabilities(self):
from copy import copy, deepcopy
from pickle import dumps, loads
weights = [0.25, 0.5, 0.25]
hp = CategoricalHyperparameter("B", ["1", "2", "3"], weights=weights)
sub_cs = ConfigurationSpace()
sub_cs.add_hyperparameter(hp)
cs = ConfigurationSpace()
cs.add_configuration_space("A", sub_cs)
self.assertEqual(
deepcopy(sub_cs).get_hyperparameter("B").probabilities, weights)
self.assertEqual(
copy(sub_cs).get_hyperparameter("B").probabilities, weights)
self.assertEqual(
loads(dumps(sub_cs)).get_hyperparameter("B").probabilities,
weights)
self.assertEqual(cs.get_hyperparameter("A:B").probabilities, weights)
self.assertEqual(
deepcopy(cs).get_hyperparameter("A:B").probabilities, weights)
self.assertEqual(
copy(cs).get_hyperparameter("A:B").probabilities, weights)
self.assertEqual(
loads(dumps(cs)).get_hyperparameter("A:B").probabilities, weights)
def build_hierarchical_configspace(config_dict):
"""
Reference: pipeline/base=325, classification/__init__=121
"""
cs = ConfigurationSpace()
candidates = list(config_dict.keys())
# TODO: set the default model.
model_option = CategoricalHyperparameter("estimator",
candidates,
default_value=candidates[0])
cs.add_hyperparameter(model_option)
for model_item in candidates:
sub_configuration_space = config_dict[model_item]
parent_hyperparameter = {
'parent': model_option,
'value': model_item
}
cs.add_configuration_space(
model_item,
sub_configuration_space,
parent_hyperparameter=parent_hyperparameter)
return cs
def get_combined_cs(node, task_type=0):
cs = ConfigurationSpace()
config_cand = []
cand_space = {}
for estimator_id in _classifiers:
cand_space[estimator_id] = get_hpo_cs(estimator_id, task_type)
config_cand.append(estimator_id)
config_option = CategoricalHyperparameter('estimator', config_cand)
cs.add_hyperparameter(config_option)
for config_item in config_cand:
sub_configuration_space = cand_space[config_item]
parent_hyperparameter = {'parent': config_option,
'value': config_item}
cs.add_configuration_space(config_item, sub_configuration_space,
parent_hyperparameter=parent_hyperparameter)
fe_cs = get_fe_cs(estimator_id, node, task_type)
for hp in fe_cs.get_hyperparameters():
cs.add_hyperparameter(hp)
for cond in fe_cs.get_conditions():
cs.add_condition(cond)
for bid in fe_cs.get_forbiddens():
cs.add_forbidden_clause(bid)
return cs
# preprocessing:0num->final:scale.standardize:placeholder | preprocessing:0num->final:__choice__ == 'scale.standardize'
# scale.standardize
standardize_cs = ConfigurationSpace()
standardize_cs.add_hyperparameter(Constant("copy", "True:bool"))
# scale.normalize
normalize_cs = ConfigurationSpace()
normalize_cs.add_hyperparameter(Constant("copy", "True:bool"))
# scale
scale_cs = ConfigurationSpace()
scale_choice = CategoricalHyperparameter(
'__choice__', ["scale.standardize", "scale.normalize"])
scale_cs.add_hyperparameter(scale_choice)
scale_cs.add_configuration_space("scale.standardize",
standardize_cs,
parent_hyperparameter={
"parent": scale_choice,
"value": "scale.standardize"
})
scale_cs.add_configuration_space("scale.normalize",
normalize_cs,
parent_hyperparameter={
"parent": scale_choice,
"value": "scale.normalize"
})
# reduce.pca
pca_cs = ConfigurationSpace()
pca_cs.add_hyperparameter(Constant("whiten", "True:bool"))
# reduce.ica
ica_cs = ConfigurationSpace()
ica_cs.add_hyperparameter(Constant("whiten", "True:bool"))
def recursion(self, hdl: Dict, path=()) -> ConfigurationSpace:
cs = ConfigurationSpace()
# 检测一下这个dict是否在直接描述超参
key_list = list(hdl.keys())
if len(key_list) == 0:
cs.add_hyperparameter(Constant("placeholder", "placeholder"))
return cs
else:
sample_key = key_list[0]
sample_value = hdl[sample_key]
if is_hdl_bottom(sample_key, sample_value):
store = {}
conditions_dict = {}
for key, value in hdl.items():
if purify_key(key).startswith("__"):
conditions_dict[key] = value
else:
hp = self.__parse_dict_to_config(key, value)
cs.add_hyperparameter(hp)
store[key] = hp
for key, value in conditions_dict.items():
if SERIES_CONNECT_LEADER_TOKEN in key:
leader_model, condition_indicator = key.split(
SERIES_CONNECT_LEADER_TOKEN)
else:
leader_model, condition_indicator = None, key
if condition_indicator == "__condition":
assert isinstance(value, list)
for item in value:
cond = self.__condition(item, store, leader_model)
cs.add_condition(cond)
elif condition_indicator == "__activate":
self.__activate(value, store, cs, leader_model)
elif condition_indicator == "__forbidden":
self.__forbidden(value, store, cs, leader_model)
elif condition_indicator == "__rely_model":
RelyModels.info.append([value, deepcopy(path)])
return cs
pattern = re.compile(r"(.*)\((.*)\)")
for key, value in hdl.items():
mat = pattern.match(key)
if mat:
groups = mat.groups()
assert len(groups) == 2
prefix_name, method = groups
value_list = list(value.keys())
assert len(value_list) >= 1
if method == "choice":
pass
else:
raise NotImplementedError()
cur_cs = ConfigurationSpace()
assert isinstance(value, dict)
# 不能用constant，会报错
choice2proba = {}
not_specific_proba_choices = []
sum_proba = 0
for k in value_list:
v = value[k]
if isinstance(v, dict) and "__proba" in v:
proba = v.pop("__proba")
choice2proba[k] = proba
sum_proba += proba
else:
not_specific_proba_choices.append(k)
if sum_proba <= 1:
if len(not_specific_proba_choices) > 0:
p_rest = (1 -
sum_proba) / len(not_specific_proba_choices)
for not_specific_proba_choice in not_specific_proba_choices:
choice2proba[not_specific_proba_choice] = p_rest
else:
choice2proba = {k: 1 / len(value_list) for k in value_list}
proba_list = [choice2proba[k] for k in value_list]
value_list = list(map(smac_hdl._encode,
value_list)) # choices must be str
option_param = CategoricalHyperparameter(
'__choice__', value_list,
weights=proba_list) # todo : default
cur_cs.add_hyperparameter(option_param)
for sub_key, sub_value in value.items():
assert isinstance(sub_value, dict)
sub_cs = self.recursion(sub_value,
path=list(path) +
[prefix_name, sub_key])
parent_hyperparameter = {
'parent': option_param,
'value': sub_key
}
cur_cs.add_configuration_space(
sub_key,
sub_cs,
parent_hyperparameter=parent_hyperparameter)
cs.add_configuration_space(prefix_name, cur_cs)
elif isinstance(value, dict):
sub_cs = self.recursion(value, path=list(path) + [key])
cs.add_configuration_space(key, sub_cs)
else:
raise NotImplementedError()
return cs