def test_get_hyperparameters_topological_sort(self):
# and now for something more complicated
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("input1", [0, 1])
hp2 = CategoricalHyperparameter("input2", [0, 1])
hp3 = CategoricalHyperparameter("input3", [0, 1])
hp4 = CategoricalHyperparameter("input4", [0, 1])
hp5 = CategoricalHyperparameter("input5", [0, 1])
hp6 = Constant("AND", "True")
# More top-level hyperparameters
hp7 = CategoricalHyperparameter("input7", [0, 1])
# Somewhat shuffled
hyperparameters = [hp1, hp2, hp3, hp4, hp5, hp6, hp7]
for hp in hyperparameters:
cs.add_hyperparameter(hp)
cond1 = EqualsCondition(hp6, hp1, 1)
cond2 = NotEqualsCondition(hp6, hp2, 1)
cond3 = InCondition(hp6, hp3, [1])
cond4 = EqualsCondition(hp5, hp3, 1)
cond5 = EqualsCondition(hp4, hp5, 1)
cond6 = EqualsCondition(hp6, hp4, 1)
cond7 = EqualsCondition(hp6, hp5, 1)
conj1 = AndConjunction(cond1, cond2)
conj2 = OrConjunction(conj1, cond3)
conj3 = AndConjunction(conj2, cond6, cond7)
cs.add_condition(cond4)
hps = cs.get_hyperparameters()
# AND is moved to the front because of alphabetical sorting
for hp, idx in zip(hyperparameters, [1, 2, 3, 4, 6, 0, 5]):
self.assertEqual(hps.index(hp), idx)
self.assertEqual(cs._hyperparameter_idx[hp.name], idx)
self.assertEqual(cs._idx_to_hyperparameter[idx], hp.name)
cs.add_condition(cond5)
hps = cs.get_hyperparameters()
for hp, idx in zip(hyperparameters, [1, 2, 3, 6, 5, 0, 4]):
self.assertEqual(hps.index(hp), idx)
self.assertEqual(cs._hyperparameter_idx[hp.name], idx)
self.assertEqual(cs._idx_to_hyperparameter[idx], hp.name)
cs.add_condition(conj3)
hps = cs.get_hyperparameters()
print(hps, hyperparameters)
for hp, idx in zip(hyperparameters, [0, 1, 2, 5, 4, 6, 3]):
print(hp, idx)
self.assertEqual(hps.index(hp), idx)
self.assertEqual(cs._hyperparameter_idx[hp.name], idx)
self.assertEqual(cs._idx_to_hyperparameter[idx], hp.name)
def test_sample_configuration(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
# This automatically checks the configuration!
Configuration(cs, dict(parent=0, child=5))
# and now for something more complicated
cs = ConfigurationSpace(seed=1)
hp1 = CategoricalHyperparameter("input1", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = CategoricalHyperparameter("input2", [0, 1])
cs.add_hyperparameter(hp2)
hp3 = CategoricalHyperparameter("input3", [0, 1])
cs.add_hyperparameter(hp3)
hp4 = CategoricalHyperparameter("input4", [0, 1])
cs.add_hyperparameter(hp4)
hp5 = CategoricalHyperparameter("input5", [0, 1])
cs.add_hyperparameter(hp5)
hp6 = Constant("AND", "True")
cs.add_hyperparameter(hp6)
cond1 = EqualsCondition(hp6, hp1, 1)
cond2 = NotEqualsCondition(hp6, hp2, 1)
cond3 = InCondition(hp6, hp3, [1])
cond4 = EqualsCondition(hp5, hp3, 1)
cond5 = EqualsCondition(hp4, hp5, 1)
cond6 = EqualsCondition(hp6, hp4, 1)
cond7 = EqualsCondition(hp6, hp5, 1)
conj1 = AndConjunction(cond1, cond2)
conj2 = OrConjunction(conj1, cond3)
conj3 = AndConjunction(conj2, cond6, cond7)
cs.add_condition(cond4)
cs.add_condition(cond5)
cs.add_condition(conj3)
samples = []
for i in range(5):
cs.seed(1)
samples.append([])
for j in range(100):
sample = cs.sample_configuration()
samples[-1].append(sample)
if i > 0:
for j in range(100):
self.assertEqual(samples[-1][j], samples[-2][j])
def test_add_forbidden_illegal(self):
cs = ConfigurationSpace()
hp = CategoricalHyperparameter("input1", [0, 1])
forb = ForbiddenEqualsClause(hp, 1)
self.assertRaisesRegex(
ValueError,
"Cannot add clause '%s'" % forb,
cs.add_forbidden_clause,
forb,
)
forb2 = ForbiddenEqualsClause(hp, 0)
self.assertRaisesRegex(
ValueError,
"Cannot add clause '%s'" % forb,
cs.add_forbidden_clauses,
[forb, forb2],
)
def test_keys(self):
# A regression test to make sure issue #49 does no longer pop up. By
# iterating over the configuration in the for loop, it should not raise
# a KeyError if the child hyperparameter is inactive.
cs = ConfigurationSpace()
shrinkage = CategoricalHyperparameter(
"shrinkage", ["None", "auto", "manual"], default_value="None",
)
shrinkage_factor = UniformFloatHyperparameter(
"shrinkage_factor", 0., 1., 0.5,
)
cs.add_hyperparameters([shrinkage, shrinkage_factor])
cs.add_condition(EqualsCondition(shrinkage_factor, shrinkage, "manual"))
for i in range(10):
config = cs.sample_configuration()
{hp_name: config[hp_name] for hp_name in config if config[hp_name] is not None}
def ising(n_eval, lamda, random_seed_pair):
evaluator = Ising(lamda, random_seed_pair)
name_tag = '_'.join([
'ising', ('%.2E' % lamda),
datetime.now().strftime("%Y-%m-%d-%H:%M:%S:%f")
])
cs = ConfigurationSpace()
for i in range(ISING_N_EDGES):
car_var = CategoricalHyperparameter('x' + str(i + 1).zfill(2),
[str(elm) for elm in range(2)],
default_value='0')
cs.add_hyperparameter(car_var)
init_points_numpy = evaluator.suggested_init.long().numpy()
init_points = []
for i in range(init_points_numpy.shape[0]):
init_points.append(
Configuration(
cs, {
'x' + str(j + 1).zfill(2): str(init_points_numpy[i][j])
for j in range(ISING_N_EDGES)
}))
def evaluate(x):
x_tensor = torch.LongTensor(
[int(x['x' + str(j + 1).zfill(2)]) for j in range(ISING_N_EDGES)])
return evaluator.evaluate(x_tensor).item()
print('Began at ' + datetime.now().strftime("%H:%M:%S"))
scenario = Scenario({
"run_obj": "quality",
"runcount-limit": n_eval,
"cs": cs,
"deterministic": "true",
'output_dir': os.path.join(EXP_DIR, name_tag)
})
smac = SMAC(scenario=scenario,
tae_runner=evaluate,
initial_configurations=init_points)
smac.optimize()
evaluations, optimum = evaluations_from_smac(smac)
print('Finished at ' + datetime.now().strftime("%H:%M:%S"))
return optimum
def test_condition_without_added_hyperparameters(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cond = EqualsCondition(hp2, hp1, 0)
self.assertRaisesRegex(
ValueError, "Child hyperparameter 'child' not "
"in configuration space.", cs.add_condition, cond)
cs.add_hyperparameter(hp1)
self.assertRaisesRegex(
ValueError, "Child hyperparameter 'child' not "
"in configuration space.", cs.add_condition, cond)
# Test also the parent hyperparameter
cs2 = ConfigurationSpace()
cs2.add_hyperparameter(hp2)
self.assertRaisesRegex(
ValueError, "Parent hyperparameter 'parent' "
"not in configuration space.", cs2.add_condition, cond)
def get_cash_cs(include_algorithms=None, task_type=CLASSIFICATION):
_candidates = get_combined_candidtates(_classifiers, _addons)
if include_algorithms is not None:
_candidates = set(include_algorithms).intersection(
set(_candidates.keys()))
if len(_candidates) == 0:
raise ValueError(
"No algorithms included! Please check the spelling of the included algorithms!"
)
cs = ConfigurationSpace()
algo = CategoricalHyperparameter('algorithm', list(_candidates))
cs.add_hyperparameter(algo)
for estimator_id in _candidates:
estimator_cs = get_hpo_cs(estimator_id)
parent_hyperparameter = {'parent': algo, 'value': estimator_id}
cs.add_configuration_space(estimator_id,
estimator_cs,
parent_hyperparameter=parent_hyperparameter)
return cs
def test_repr_roundtrip(self):
cs = ConfigurationSpace()
cs.add_hyperparameter(
UniformIntegerHyperparameter("uihp", lower=1, upper=10))
cs.add_hyperparameter(
NormalIntegerHyperparameter("nihp", mu=0, sigma=1))
cs.add_hyperparameter(
UniformFloatHyperparameter("ufhp", lower=1, upper=10))
cs.add_hyperparameter(NormalFloatHyperparameter("nfhp", mu=0, sigma=1))
cs.add_hyperparameter(
CategoricalHyperparameter("chp", choices=['1', '2', '3']))
cs.add_hyperparameter(
OrdinalHyperparameter("ohp", sequence=['1', '2', '3']))
cs.add_hyperparameter(Constant("const", value=1))
default = cs.get_default_configuration()
repr = default.__repr__()
repr = repr.replace('})', '}, configuration_space=cs)')
config = eval(repr)
self.assertEqual(default, config)
def test_add_configuration_space(self):
cs = ConfigurationSpace()
hp1 = cs.add_hyperparameter(CategoricalHyperparameter(
"input1", [0, 1]))
cs.add_forbidden_clause(ForbiddenEqualsClause(hp1, 1))
hp2 = cs.add_hyperparameter(
UniformIntegerHyperparameter("child", 0, 10))
cs.add_condition(EqualsCondition(hp2, hp1, 0))
cs2 = ConfigurationSpace()
cs2.add_configuration_space('prefix', cs, delimiter='__')
self.assertEqual(
str(cs2), '''Configuration space object:
Hyperparameters:
prefix__child, Type: UniformInteger, Range: [0, 10], Default: 5
prefix__input1, Type: Categorical, Choices: {0, 1}, Default: 0
Conditions:
prefix__child | prefix__input1 == 0
Forbidden Clauses:
Forbidden: prefix__input1 == 1
''')
def test_get_parent_and_children_of(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
self.assertEqual([hp1], cs.get_parents_of(hp2.name))
self.assertEqual([hp1], cs.get_parents_of(hp2))
self.assertEqual([hp2], cs.get_children_of(hp1.name))
self.assertEqual([hp2], cs.get_children_of(hp1))
self.assertRaisesRegex(
KeyError, "Hyperparameter 'Foo' does not exist in this "
"configuration space.", cs.get_parents_of, "Foo")
self.assertRaisesRegex(
KeyError, "Hyperparameter 'Foo' does not exist in this "
"configuration space.", cs.get_children_of, "Foo")
def test_repr(self):
cs1 = ConfigurationSpace()
retval = cs1.__str__()
self.assertEqual("Configuration space object:\n Hyperparameters:\n",
retval)
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs1.add_hyperparameter(hp1)
retval = cs1.__str__()
self.assertEqual("Configuration space object:\n Hyperparameters:\n"
" %s\n" % str(hp1), retval)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cond1 = EqualsCondition(hp2, hp1, 0)
cs1.add_hyperparameter(hp2)
cs1.add_condition(cond1)
retval = cs1.__str__()
self.assertEqual("Configuration space object:\n Hyperparameters:\n"
" %s\n %s\n Conditions:\n %s\n" %
(str(hp2), str(hp1), str(cond1)), retval)
def test_with_ordinal(self):
cs = smac.configspace.ConfigurationSpace()
a = cs.add_hyperparameter(CategoricalHyperparameter('a', [0, 1], default_value=0))
b = cs.add_hyperparameter(OrdinalHyperparameter('b', [0, 1], default_value=1))
b = cs.add_hyperparameter(UniformFloatHyperparameter('c', lower=0., upper=1., default_value=1))
b = cs.add_hyperparameter(UniformIntegerHyperparameter('d', lower=0, upper=10, default_value=1))
cs.seed(1)
feat_array = np.array([0, 0, 0]).reshape(1, -1)
types, bounds = get_types(cs, feat_array)
model = RandomForestWithInstances(
configspace=cs,
types=types,
bounds=bounds,
instance_features=feat_array,
seed=1,
ratio_features=1.0,
pca_components=9,
)
self.assertEqual(bounds[0][0], 2)
self.assertTrue(bounds[0][1] is np.nan)
self.assertEqual(bounds[1][0], 0)
self.assertEqual(bounds[1][1], 1)
self.assertEqual(bounds[2][0], 0.)
self.assertEqual(bounds[2][1], 1.)
self.assertEqual(bounds[3][0], 0.)
self.assertEqual(bounds[3][1], 1.)
X = np.array([
[0., 0., 0., 0., 0., 0., 0.],
[0., 0., 1., 0., 0., 0., 0.],
[0., 1., 0., 9., 0., 0., 0.],
[0., 1., 1., 4., 0., 0., 0.]], dtype=np.float64)
y = np.array([0, 1, 2, 3], dtype=np.float64)
X_train = np.vstack((X, X, X, X, X, X, X, X, X, X))
y_train = np.vstack((y, y, y, y, y, y, y, y, y, y))
model.train(X_train, y_train.reshape((-1, 1)))
mean, _ = model.predict(X)
for idx, m in enumerate(mean):
self.assertAlmostEqual(y[idx], m, 0.05)
def get_combined_cs(estimator_id, node, task_type=0):
cs = ConfigurationSpace()
hpo_cs = get_hpo_cs(estimator_id, task_type)
fe_cs = get_fe_cs(estimator_id, node, task_type)
config_cand = ['placeholder']
config_option = CategoricalHyperparameter('hpo', config_cand)
cs.add_hyperparameter(config_option)
for config_item in config_cand:
sub_configuration_space = hpo_cs
parent_hyperparameter = {'parent': config_option, 'value': config_item}
cs.add_configuration_space(config_item,
sub_configuration_space,
parent_hyperparameter=parent_hyperparameter)
for hp in fe_cs.get_hyperparameters():
cs.add_hyperparameter(hp)
for cond in fe_cs.get_conditions():
cs.add_condition(cond)
for bid in fe_cs.get_forbiddens():
cs.add_forbidden_clause(bid)
model = UnParametrizedHyperparameter("estimator", estimator_id)
cs.add_hyperparameter(model)
return cs
def test_add_configuration_space_probabilities(self):
from copy import copy, deepcopy
from pickle import dumps, loads
weights = [0.25, 0.5, 0.25]
hp = CategoricalHyperparameter("B", ["1", "2", "3"], weights=weights)
sub_cs = ConfigurationSpace()
sub_cs.add_hyperparameter(hp)
cs = ConfigurationSpace()
cs.add_configuration_space("A", sub_cs)
self.assertEqual(
deepcopy(sub_cs).get_hyperparameter("B").probabilities, weights)
self.assertEqual(
copy(sub_cs).get_hyperparameter("B").probabilities, weights)
self.assertEqual(
loads(dumps(sub_cs)).get_hyperparameter("B").probabilities,
weights)
self.assertEqual(cs.get_hyperparameter("A:B").probabilities, weights)
self.assertEqual(
deepcopy(cs).get_hyperparameter("A:B").probabilities, weights)
self.assertEqual(
copy(cs).get_hyperparameter("A:B").probabilities, weights)
self.assertEqual(
loads(dumps(cs)).get_hyperparameter("A:B").probabilities, weights)
def test_eq(self):
# Compare empty configuration spaces
cs1 = ConfigurationSpace()
cs2 = ConfigurationSpace()
self.assertEqual(cs1, cs2)
# Compare to something which isn't a configuration space
self.assertTrue(not (cs1 == "ConfigurationSpace"))
# Compare to equal configuration spaces
hp1 = CategoricalHyperparameter("parent", [0, 1])
hp2 = UniformIntegerHyperparameter("child", 0, 10)
hp3 = UniformIntegerHyperparameter("friend", 0, 5)
cond1 = EqualsCondition(hp2, hp1, 0)
cs1.add_hyperparameter(hp1)
cs1.add_hyperparameter(hp2)
cs1.add_condition(cond1)
cs2.add_hyperparameter(hp1)
cs2.add_hyperparameter(hp2)
cs2.add_condition(cond1)
self.assertEqual(cs1, cs2)
cs1.add_hyperparameter(hp3)
self.assertFalse(cs1 == cs2)
def get_config_space_from_dict(space_dict: dict):
cs = ConfigurationSpace()
params_dict = space_dict['parameters']
for key in params_dict:
param_dict = params_dict[key]
param_type = param_dict['type']
if param_type in ['float', 'int']:
bound = param_dict['bound']
optional_args = dict()
if 'default' in param_dict:
optional_args['default_value'] = param_dict['default']
if 'log' in param_dict:
optional_args['log'] = parse_bool(param_dict['log'])
if 'q' in param_dict:
optional_args['q'] = param_dict['q']
if param_type == 'float':
param = UniformFloatHyperparameter(key, bound[0], bound[1], **optional_args)
else:
param = UniformIntegerHyperparameter(key, bound[0], bound[1], **optional_args)
elif param_type == 'cat':
choices = param_dict['choice']
optional_args = dict()
if 'default' in param_dict:
optional_args['default_value'] = param_dict['default']
param = CategoricalHyperparameter(key, choices, **optional_args)
elif param_type == 'const':
value = param_dict['value']
param = Constant(key, value)
else:
raise ValueError("Parameter type %s not supported!" % param_type)
cs.add_hyperparameter(param)
return cs
def get_combined_cs(node, task_type=0):
cs = ConfigurationSpace()
config_cand = []
cand_space = {}
for estimator_id in _classifiers:
cand_space[estimator_id] = get_hpo_cs(estimator_id, task_type)
config_cand.append(estimator_id)
config_option = CategoricalHyperparameter('estimator', config_cand)
cs.add_hyperparameter(config_option)
for config_item in config_cand:
sub_configuration_space = cand_space[config_item]
parent_hyperparameter = {'parent': config_option,
'value': config_item}
cs.add_configuration_space(config_item, sub_configuration_space,
parent_hyperparameter=parent_hyperparameter)
fe_cs = get_fe_cs(estimator_id, node, task_type)
for hp in fe_cs.get_hyperparameters():
cs.add_hyperparameter(hp)
for cond in fe_cs.get_conditions():
cs.add_condition(cond)
for bid in fe_cs.get_forbiddens():
cs.add_forbidden_clause(bid)
return cs
def test_subspace_switches(self):
# create a switch to select one of two algorithms
algo_switch = CategoricalHyperparameter(name="switch",
choices=["algo1", "algo2"],
weights=[0.25, 0.75],
default_value="algo1")
# create sub-configuration space for algorithm 1
algo1_cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter(name="algo1_param1",
choices=["A", "B"],
weights=[0.3, 0.7],
default_value="B")
algo1_cs.add_hyperparameter(hp1)
# create sub-configuration space for algorithm 2
algo2_cs = ConfigurationSpace()
hp2 = CategoricalHyperparameter(name="algo2_param1",
choices=["X", "Y"],
default_value="Y")
algo2_cs.add_hyperparameter(hp2)
# create a configuration space and populate it with both the switch
# and the two sub-configuration spaces
cs = ConfigurationSpace()
cs.add_hyperparameter(algo_switch)
cs.add_configuration_space(prefix="algo1_subspace",
configuration_space=algo1_cs,
parent_hyperparameter={
'parent': algo_switch,
'value': "algo1"
})
cs.add_configuration_space(prefix="algo2_subspace",
configuration_space=algo2_cs,
parent_hyperparameter={
'parent': algo_switch,
'value': "algo2"
})
# check choices in the final configuration space
self.assertEqual(("algo1", "algo2"),
cs.get_hyperparameter("switch").choices)
self.assertEqual(
("A", "B"),
cs.get_hyperparameter("algo1_subspace:algo1_param1").choices)
self.assertEqual(
("X", "Y"),
cs.get_hyperparameter("algo2_subspace:algo2_param1").choices)
# check probabilities in the final configuration space
self.assertEqual((0.25, 0.75),
cs.get_hyperparameter("switch").probabilities)
self.assertEqual(
(0.3, 0.7),
cs.get_hyperparameter("algo1_subspace:algo1_param1").probabilities)
self.assertEqual(
None,
cs.get_hyperparameter("algo2_subspace:algo2_param1").probabilities)
# check default values in the final configuration space
self.assertEqual("algo1",
cs.get_hyperparameter("switch").default_value)
self.assertEqual(
"B",
cs.get_hyperparameter("algo1_subspace:algo1_param1").default_value)
self.assertEqual(
"Y",
cs.get_hyperparameter("algo2_subspace:algo2_param1").default_value)
def test_check_configuration(self):
# TODO this is only a smoke test
# TODO actually, this rather tests the evaluate methods in the
# conditions module!
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
# This automatically checks the configuration!
Configuration(cs, dict(parent=0, child=5))
# and now for something more complicated
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("input1", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = CategoricalHyperparameter("input2", [0, 1])
cs.add_hyperparameter(hp2)
hp3 = CategoricalHyperparameter("input3", [0, 1])
cs.add_hyperparameter(hp3)
hp4 = CategoricalHyperparameter("input4", [0, 1])
cs.add_hyperparameter(hp4)
hp5 = CategoricalHyperparameter("input5", [0, 1])
cs.add_hyperparameter(hp5)
hp6 = Constant("AND", "True")
cs.add_hyperparameter(hp6)
cond1 = EqualsCondition(hp6, hp1, 1)
cond2 = NotEqualsCondition(hp6, hp2, 1)
cond3 = InCondition(hp6, hp3, [1])
cond4 = EqualsCondition(hp6, hp4, 1)
cond5 = EqualsCondition(hp6, hp5, 1)
conj1 = AndConjunction(cond1, cond2)
conj2 = OrConjunction(conj1, cond3)
conj3 = AndConjunction(conj2, cond4, cond5)
cs.add_condition(conj3)
expected_outcomes = [
False, False, False, False, False, False, False, True, False,
False, False, False, False, False, False, True, False, False,
False, True, False, False, False, True, False, False, False, False,
False, False, False, True
]
for idx, values in enumerate(product([0, 1], repeat=5)):
# The hyperparameters aren't sorted, but the test assumes them to
# be sorted.
hyperparameters = sorted(cs.get_hyperparameters(),
key=lambda t: t.name)
instantiations = {
hyperparameters[jdx + 1].name: values[jdx]
for jdx in range(len(values))
}
evaluation = conj3.evaluate(instantiations)
self.assertEqual(expected_outcomes[idx], evaluation)
if not evaluation:
self.assertRaisesRegex(
ValueError,
r"Inactive hyperparameter 'AND' must "
r"not be specified, but has the vector value: "
r"'0.0'.",
Configuration,
cs,
values={
"input1": values[0],
"input2": values[1],
"input3": values[2],
"input4": values[3],
"input5": values[4],
"AND": "True",
},
)
else:
Configuration(
cs,
values={
"input1": values[0],
"input2": values[1],
"input3": values[2],
"input4": values[3],
"input5": values[4],
"AND": "True",
},
)
def recursion(self, hdl: Dict, path=()) -> ConfigurationSpace:
cs = ConfigurationSpace()
# 检测一下这个dict是否在直接描述超参
key_list = list(hdl.keys())
if len(key_list) == 0:
cs.add_hyperparameter(Constant("placeholder", "placeholder"))
return cs
else:
sample_key = key_list[0]
sample_value = hdl[sample_key]
if is_hdl_bottom(sample_key, sample_value):
store = {}
conditions_dict = {}
for key, value in hdl.items():
if purify_key(key).startswith("__"):
conditions_dict[key] = value
else:
hp = self.__parse_dict_to_config(key, value)
cs.add_hyperparameter(hp)
store[key] = hp
for key, value in conditions_dict.items():
if SERIES_CONNECT_LEADER_TOKEN in key:
leader_model, condition_indicator = key.split(
SERIES_CONNECT_LEADER_TOKEN)
else:
leader_model, condition_indicator = None, key
if condition_indicator == "__condition":
assert isinstance(value, list)
for item in value:
cond = self.__condition(item, store, leader_model)
cs.add_condition(cond)
elif condition_indicator == "__activate":
self.__activate(value, store, cs, leader_model)
elif condition_indicator == "__forbidden":
self.__forbidden(value, store, cs, leader_model)
elif condition_indicator == "__rely_model":
RelyModels.info.append([value, deepcopy(path)])
return cs
pattern = re.compile(r"(.*)\((.*)\)")
for key, value in hdl.items():
mat = pattern.match(key)
if mat:
groups = mat.groups()
assert len(groups) == 2
prefix_name, method = groups
value_list = list(value.keys())
assert len(value_list) >= 1
if method == "choice":
pass
else:
raise NotImplementedError()
cur_cs = ConfigurationSpace()
assert isinstance(value, dict)
# 不能用constant，会报错
choice2proba = {}
not_specific_proba_choices = []
sum_proba = 0
for k in value_list:
v = value[k]
if isinstance(v, dict) and "__proba" in v:
proba = v.pop("__proba")
choice2proba[k] = proba
sum_proba += proba
else:
not_specific_proba_choices.append(k)
if sum_proba <= 1:
if len(not_specific_proba_choices) > 0:
p_rest = (1 -
sum_proba) / len(not_specific_proba_choices)
for not_specific_proba_choice in not_specific_proba_choices:
choice2proba[not_specific_proba_choice] = p_rest
else:
choice2proba = {k: 1 / len(value_list) for k in value_list}
proba_list = [choice2proba[k] for k in value_list]
value_list = list(map(smac_hdl._encode,
value_list)) # choices must be str
option_param = CategoricalHyperparameter(
'__choice__', value_list,
weights=proba_list) # todo : default
cur_cs.add_hyperparameter(option_param)
for sub_key, sub_value in value.items():
assert isinstance(sub_value, dict)
sub_cs = self.recursion(sub_value,
path=list(path) +
[prefix_name, sub_key])
parent_hyperparameter = {
'parent': option_param,
'value': sub_key
}
cur_cs.add_configuration_space(
sub_key,
sub_cs,
parent_hyperparameter=parent_hyperparameter)
cs.add_configuration_space(prefix_name, cur_cs)
elif isinstance(value, dict):
sub_cs = self.recursion(value, path=list(path) + [key])
cs.add_configuration_space(key, sub_cs)
else:
raise NotImplementedError()
return cs
from ConfigSpace import ConfigurationSpace, UniformFloatHyperparameter, CategoricalHyperparameter, \
UniformIntegerHyperparameter
from ConfigSpace import EqualsCondition, InCondition, ForbiddenAndConjunction, ForbiddenEqualsClause, ForbiddenInClause
import os
import sys
sys.path.append(os.getcwd())
cs1 = ConfigurationSpace()
x1 = UniformFloatHyperparameter("x1", -5, 10, default_value=0)
x2 = UniformIntegerHyperparameter("x2", 1, 15, q=2.0, log=True)
x3 = CategoricalHyperparameter("x3", ['x1', 'x2', 'x3'])
x4 = CategoricalHyperparameter("x9", ['x1', 'x2', 'x3'])
x5 = CategoricalHyperparameter("x5", ['x1', 'x2', 'x3'])
x6 = CategoricalHyperparameter("x6", ['x1', 'x2', 'x3'])
x7 = CategoricalHyperparameter("x7", ['x1', 'x3', 'x2'], default_value='x2')
cs1.add_hyperparameters([x1, x2, x3, x7, x6, x5, x4])
cond1 = ForbiddenAndConjunction(
ForbiddenEqualsClause(x3, "x1"),
ForbiddenEqualsClause(x4, "x2"),
ForbiddenEqualsClause(x5, "x3")
)
cs1.add_forbidden_clause(cond1)
cond2 = EqualsCondition(x1, x5, "x1")
cond7 = EqualsCondition(x2, x6, "x1")
cond3 = InCondition(x2, x6, ["x1", "x2", "x3"])
cs1.add_condition(cond3)
cs1.add_condition(cond2)
# estimating:lightgbm:n_estimator, Type: Constant, Value: 100:int
# preprocessing:0num->final:__choice__, Type: Categorical, Choices: {scale.standardize}, Default: scale.standardize
# preprocessing:0num->final:scale.standardize:placeholder, Type: Constant, Value: placeholder
# Conditions:
# estimating:lightgbm:n_estimator | estimating:__choice__ == 'lightgbm'
# preprocessing:0num->final:scale.standardize:placeholder | preprocessing:0num->final:__choice__ == 'scale.standardize'
# scale.standardize
standardize_cs = ConfigurationSpace()
standardize_cs.add_hyperparameter(Constant("copy", "True:bool"))
# scale.normalize
normalize_cs = ConfigurationSpace()
normalize_cs.add_hyperparameter(Constant("copy", "True:bool"))
# scale
scale_cs = ConfigurationSpace()
scale_choice = CategoricalHyperparameter(
'__choice__', ["scale.standardize", "scale.normalize"])
scale_cs.add_hyperparameter(scale_choice)
scale_cs.add_configuration_space("scale.standardize",
standardize_cs,
parent_hyperparameter={
"parent": scale_choice,
"value": "scale.standardize"
})
scale_cs.add_configuration_space("scale.normalize",
normalize_cs,
parent_hyperparameter={
"parent": scale_choice,
"value": "scale.normalize"
})
# reduce.pca
import time
import numpy as np
from ConfigSpace import ConfigurationSpace, CategoricalHyperparameter, \
ForbiddenEqualsClause
from ConfigSpace.io import pcs
from ConfigSpace.util import get_one_exchange_neighbourhood
#with open('/home/feurerm/projects/ConfigSpace/test/test_searchspaces/auto'
# '-sklearn_2017_04.pcs') as fh:
# cs = pcs.read(fh)
cs = ConfigurationSpace()
hp1 = cs.add_hyperparameter(CategoricalHyperparameter("hp1", [0, 1, 2, 3, 4, 5]))
cs.add_forbidden_clause(ForbiddenEqualsClause(hp1, 1))
cs.add_forbidden_clause(ForbiddenEqualsClause(hp1, 3))
cs.add_forbidden_clause(ForbiddenEqualsClause(hp1, 5))
times = []
for i in range(20):
start_time = time.time()
configs = cs.sample_configuration(500000)
end_time = time.time()
times.append(end_time - start_time)
print("all times:", times)
print('Sampling 500000 configurations took on average:', np.mean(times))
def get_hyperparameter_search_space(dataset_properties=None):
max_num_layers = 7 # Maximum number of layers coded
# Hacky way to condition layers params based on the number of layers
# 'c'=1, 'd'=2, 'e'=3 ,'f'=4', g ='5', h='6' + output_layer
layer_choices = [
chr(i) for i in range(ord('c'),
ord('b') + max_num_layers)
]
batch_size = UniformIntegerHyperparameter("batch_size",
32,
4096,
log=True,
default=32)
number_epochs = UniformIntegerHyperparameter("number_epochs",
2,
80,
default=5)
num_layers = CategoricalHyperparameter("num_layers",
choices=layer_choices,
default='c')
lr = UniformFloatHyperparameter("learning_rate",
1e-6,
1.0,
log=True,
default=0.01)
l2 = UniformFloatHyperparameter("lambda2",
1e-7,
1e-2,
log=True,
default=1e-4)
dropout_output = UniformFloatHyperparameter("dropout_output",
0.0,
0.99,
default=0.5)
# Define basic hyperparameters and define the config space
# basic means that are independent from the number of layers
cs = ConfigurationSpace()
cs.add_hyperparameter(number_epochs)
cs.add_hyperparameter(batch_size)
cs.add_hyperparameter(num_layers)
cs.add_hyperparameter(lr)
cs.add_hyperparameter(l2)
cs.add_hyperparameter(dropout_output)
# Define parameters with different child parameters and conditions
solver_choices = [
"adam", "adadelta", "adagrad", "sgd", "momentum", "nesterov",
"smorm3s"
]
solver = CategoricalHyperparameter(name="solver",
choices=solver_choices,
default="smorm3s")
beta1 = UniformFloatHyperparameter("beta1",
1e-4,
0.1,
log=True,
default=0.1)
beta2 = UniformFloatHyperparameter("beta2",
1e-4,
0.1,
log=True,
default=0.01)
rho = UniformFloatHyperparameter("rho",
0.05,
0.99,
log=True,
default=0.95)
momentum = UniformFloatHyperparameter("momentum",
0.3,
0.999,
default=0.9)
# TODO: Add policy based on this sklearn sgd
policy_choices = ['fixed', 'inv', 'exp', 'step']
lr_policy = CategoricalHyperparameter(name="lr_policy",
choices=policy_choices,
default='fixed')
gamma = UniformFloatHyperparameter(name="gamma",
lower=1e-3,
upper=1e-1,
default=1e-2)
power = UniformFloatHyperparameter("power", 0.0, 1.0, default=0.5)
epoch_step = UniformIntegerHyperparameter("epoch_step",
2,
20,
default=5)
cs.add_hyperparameter(solver)
cs.add_hyperparameter(beta1)
cs.add_hyperparameter(beta2)
cs.add_hyperparameter(momentum)
cs.add_hyperparameter(rho)
cs.add_hyperparameter(lr_policy)
cs.add_hyperparameter(gamma)
cs.add_hyperparameter(power)
cs.add_hyperparameter(epoch_step)
# Define parameters that are needed it for each layer
output_activation_choices = ['softmax', 'sigmoid', 'softplus', 'tanh']
activations_choices = [
'sigmoid', 'tanh', 'scaledTanh', 'elu', 'relu', 'leaky', 'linear'
]
weight_choices = [
'constant', 'normal', 'uniform', 'glorot_normal', 'glorot_uniform',
'he_normal', 'he_uniform', 'ortogonal', 'sparse'
]
# Iterate over parameters that are used in each layer
for i in range(1, max_num_layers):
layer_units = UniformIntegerHyperparameter("num_units_layer_" +
str(i),
64,
4096,
log=True,
default=128)
cs.add_hyperparameter(layer_units)
layer_dropout = UniformFloatHyperparameter("dropout_layer_" +
str(i),
0.0,
0.99,
default=0.5)
cs.add_hyperparameter(layer_dropout)
weight_initialization = CategoricalHyperparameter(
'weight_init_' + str(i),
choices=weight_choices,
default='he_normal')
cs.add_hyperparameter(weight_initialization)
layer_std = UniformFloatHyperparameter("std_layer_" + str(i),
1e-6,
0.1,
log=True,
default=0.005)
cs.add_hyperparameter(layer_std)
layer_activation = CategoricalHyperparameter(
"activation_layer_" + str(i),
choices=activations_choices,
default="relu")
cs.add_hyperparameter(layer_activation)
layer_leakiness = UniformFloatHyperparameter('leakiness_layer_' +
str(i),
0.01,
0.99,
default=0.3)
cs.add_hyperparameter(layer_leakiness)
layer_tanh_alpha = UniformFloatHyperparameter('tanh_alpha_layer_' +
str(i),
0.5,
1.0,
default=2. / 3.)
cs.add_hyperparameter(layer_tanh_alpha)
layer_tanh_beta = UniformFloatHyperparameter('tanh_beta_layer_' +
str(i),
1.1,
3.0,
log=True,
default=1.7159)
cs.add_hyperparameter(layer_tanh_beta)
# TODO: Could be in a function in a new module
for i in range(2, max_num_layers):
# Condition layers parameter on layer choice
layer_unit_param = cs.get_hyperparameter("num_units_layer_" +
str(i))
layer_cond = InCondition(child=layer_unit_param,
parent=num_layers,
values=[l for l in layer_choices[i - 1:]])
cs.add_condition(layer_cond)
# Condition dropout parameter on layer choice
layer_dropout_param = cs.get_hyperparameter("dropout_layer_" +
str(i))
layer_cond = InCondition(child=layer_dropout_param,
parent=num_layers,
values=[l for l in layer_choices[i - 1:]])
cs.add_condition(layer_cond)
# Condition weight initialization on layer choice
layer_weight_param = cs.get_hyperparameter("weight_init_" + str(i))
layer_cond = InCondition(child=layer_weight_param,
parent=num_layers,
values=[l for l in layer_choices[i - 1:]])
cs.add_condition(layer_cond)
# Condition std parameter on weight layer initialization choice
layer_std_param = cs.get_hyperparameter("std_layer_" + str(i))
weight_cond = EqualsCondition(child=layer_std_param,
parent=layer_weight_param,
value='normal')
cs.add_condition(weight_cond)
# Condition activation parameter on layer choice
layer_activation_param = cs.get_hyperparameter(
"activation_layer_" + str(i))
layer_cond = InCondition(child=layer_activation_param,
parent=num_layers,
values=[l for l in layer_choices[i - 1:]])
cs.add_condition(layer_cond)
# Condition leakiness on activation choice
layer_leakiness_param = cs.get_hyperparameter("leakiness_layer_" +
str(i))
activation_cond = EqualsCondition(child=layer_leakiness_param,
parent=layer_activation_param,
value='leaky')
cs.add_condition(activation_cond)
# Condition tanh on activation choice
layer_tanh_alpha_param = cs.get_hyperparameter(
"tanh_alpha_layer_" + str(i))
activation_cond = EqualsCondition(child=layer_tanh_alpha_param,
parent=layer_activation_param,
value='scaledTanh')
cs.add_condition(activation_cond)
layer_tanh_beta_param = cs.get_hyperparameter("tanh_beta_layer_" +
str(i))
activation_cond = EqualsCondition(child=layer_tanh_beta_param,
parent=layer_activation_param,
value='scaledTanh')
cs.add_condition(activation_cond)
# Conditioning on solver
momentum_depends_on_solver = InCondition(
momentum, solver, values=["momentum", "nesterov"])
beta1_depends_on_solver = EqualsCondition(beta1, solver, "adam")
beta2_depends_on_solver = EqualsCondition(beta2, solver, "adam")
rho_depends_on_solver = EqualsCondition(rho, solver, "adadelta")
cs.add_condition(momentum_depends_on_solver)
cs.add_condition(beta1_depends_on_solver)
cs.add_condition(beta2_depends_on_solver)
cs.add_condition(rho_depends_on_solver)
# Conditioning on learning rate policy
lr_policy_depends_on_solver = InCondition(
lr_policy, solver,
["adadelta", "adagrad", "sgd", "momentum", "nesterov"])
gamma_depends_on_policy = InCondition(child=gamma,
parent=lr_policy,
values=["inv", "exp", "step"])
power_depends_on_policy = EqualsCondition(power, lr_policy, "inv")
epoch_step_depends_on_policy = EqualsCondition(epoch_step, lr_policy,
"step")
cs.add_condition(lr_policy_depends_on_solver)
cs.add_condition(gamma_depends_on_policy)
cs.add_condition(power_depends_on_policy)
cs.add_condition(epoch_step_depends_on_policy)
return cs
def test_deactivate_inactive_hyperparameters(self):
diamond = ConfigurationSpace()
head = CategoricalHyperparameter('head', [0, 1])
left = CategoricalHyperparameter('left', [0, 1])
right = CategoricalHyperparameter('right', [0, 1])
bottom = CategoricalHyperparameter('bottom', [0, 1])
diamond.add_hyperparameters([head, left, right, bottom])
diamond.add_condition(EqualsCondition(left, head, 0))
diamond.add_condition(EqualsCondition(right, head, 0))
diamond.add_condition(
AndConjunction(EqualsCondition(bottom, left, 0),
EqualsCondition(bottom, right, 0)))
c = deactivate_inactive_hyperparameters(
{
'head': 0,
'left': 0,
'right': 0,
'bottom': 0
}, diamond)
diamond._check_configuration_rigorous(c)
c = deactivate_inactive_hyperparameters(
{
'head': 1,
'left': 0,
'right': 0,
'bottom': 0
}, diamond)
diamond._check_configuration_rigorous(c)
c = deactivate_inactive_hyperparameters(
{
'head': 0,
'left': 1,
'right': 0,
'bottom': 0
}, diamond)
diamond._check_configuration_rigorous(c)
diamond = ConfigurationSpace()
head = CategoricalHyperparameter('head', [0, 1])
left = CategoricalHyperparameter('left', [0, 1])
right = CategoricalHyperparameter('right', [0, 1])
bottom = CategoricalHyperparameter('bottom', [0, 1])
diamond.add_hyperparameters([head, left, right, bottom])
diamond.add_condition(EqualsCondition(left, head, 0))
diamond.add_condition(EqualsCondition(right, head, 0))
diamond.add_condition(
OrConjunction(EqualsCondition(bottom, left, 0),
EqualsCondition(bottom, right, 0)))
c = deactivate_inactive_hyperparameters(
{
'head': 0,
'left': 0,
'right': 0,
'bottom': 0
}, diamond)
diamond._check_configuration_rigorous(c)
c = deactivate_inactive_hyperparameters(
{
'head': 1,
'left': 1,
'right': 0,
'bottom': 0
}, diamond)
diamond._check_configuration_rigorous(c)
c = deactivate_inactive_hyperparameters(
{
'head': 0,
'left': 1,
'right': 0,
'bottom': 0
}, diamond)
diamond._check_configuration_rigorous(c)
plain = ConfigurationSpace()
a = UniformIntegerHyperparameter('a', 0, 10)
b = UniformIntegerHyperparameter('b', 0, 10)
plain.add_hyperparameters([a, b])
c = deactivate_inactive_hyperparameters({'a': 5, 'b': 6}, plain)
plain.check_configuration(c)
def test_random_neighborhood_cat(self):
hp = CategoricalHyperparameter('a', [5, 6, 7, 8])
self._test_random_neigbor(hp)
def test_random_neighbor_cat(self):
hp = CategoricalHyperparameter('a', [5, 6, 7, 8])
all_neighbors = self._test_get_one_exchange_neighbourhood(hp)
all_neighbors = [neighbor for neighbor in all_neighbors]
self.assertEqual(len(all_neighbors),
300) # 3 (neighbors) * 100 (samples)
def setUp(self):
cs = ConfigurationSpace()
cs.add_hyperparameter(CategoricalHyperparameter("parent", [0, 1]))
cs.add_hyperparameter(UniformIntegerHyperparameter("child", 0, 10))
cs.add_hyperparameter(UniformIntegerHyperparameter("friend", 0, 5))
self.cs = cs
def test_setitem(self):
'''
Checks overriding a sampled configuration
'''
pcs = ConfigurationSpace()
pcs.add_hyperparameter(
UniformIntegerHyperparameter('x0', 1, 5, default_value=1))
x1 = pcs.add_hyperparameter(
CategoricalHyperparameter('x1', ['ab', 'bc', 'cd', 'de'],
default_value='ab'))
# Condition
x2 = pcs.add_hyperparameter(CategoricalHyperparameter('x2', [1, 2]))
pcs.add_condition(EqualsCondition(x2, x1, 'ab'))
# Forbidden
x3 = pcs.add_hyperparameter(CategoricalHyperparameter('x3', [1, 2]))
pcs.add_forbidden_clause(ForbiddenEqualsClause(x3, 2))
conf = pcs.get_default_configuration()
# failed because it's a invalid configuration
with self.assertRaisesRegex(ValueError,
"Illegal value '0' for hyperparameter x0"):
conf['x0'] = 0
# failed because the variable didn't exists
with self.assertRaisesRegex(
KeyError,
"Hyperparameter 'x_0' does not exist in this configuration space.",
):
conf['x_0'] = 1
# failed because forbidden clause is violated
with self.assertRaisesRegex(
ForbiddenValueError,
"Given vector violates forbidden clause Forbidden: x3 == 2",
):
conf['x3'] = 2
self.assertEqual(conf['x3'], 1)
# successful operation 1
x0_old = conf['x0']
if x0_old == 1:
conf['x0'] = 2
else:
conf['x0'] = 1
x0_new = conf['x0']
self.assertNotEqual(x0_old, x0_new)
pcs._check_configuration_rigorous(conf)
self.assertEqual(conf['x2'], 1)
# successful operation 2
x1_old = conf['x1']
if x1_old == 'ab':
conf['x1'] = 'cd'
else:
conf['x1'] = 'ab'
x1_new = conf['x1']
self.assertNotEqual(x1_old, x1_new)
pcs._check_configuration_rigorous(conf)
self.assertRaises(KeyError, conf.__getitem__, 'x2')
def get_categorical_param(name: str):
return CategoricalHyperparameter(name, choices=["a", "b", "c"])
