def test_write_categorical(self):
expected = "cat_a '--cat_a ' c {a,b,c}\n"
cs = ConfigurationSpace()
cs.add_hyperparameter(
CategoricalHyperparameter("cat_a", ["a", "b", "c"]))
value = irace.write(cs)
self.assertEqual(expected, value)
def read(jason_string):
jason = json.loads(jason_string)
if 'name' in jason:
configuration_space = ConfigurationSpace(name=jason['name'])
else:
configuration_space = ConfigurationSpace()
for hyperparameter in jason['hyperparameters']:
configuration_space.add_hyperparameter(
_construct_hyperparameter(hyperparameter, ))
for condition in jason['conditions']:
configuration_space.add_condition(
_construct_condition(
condition,
configuration_space,
))
for forbidden in jason['forbiddens']:
configuration_space.add_forbidden_clause(
_construct_forbidden(
forbidden,
configuration_space,
))
return configuration_space
def test_write_new_log10(self):
expected = "a real [10.0, 1000.0] [100.0]log"
cs = ConfigurationSpace()
cs.add_hyperparameter(
UniformFloatHyperparameter("a", 10, 1000, log=True))
value = pcs_new.write(cs)
self.assertEqual(expected, value)
def test_write_log_int(self):
expected = "int_log '--int_log ' i (2, 4)\n"
int_log = UniformIntegerHyperparameter("int_log", 10, 100, log=True)
cs = ConfigurationSpace()
cs.add_hyperparameter(int_log)
value = irace.write(cs)
self.assertEqual(expected, value)
def test_build_new_GreaterThanIntCondition(self):
expected = "a real [0.0, 1.0] [0.5]\n" \
"b integer [0, 10] [5]\n\n" \
"b | a > 0.5"
cs = ConfigurationSpace()
a = UniformFloatHyperparameter("a", 0, 1, 0.5)
b = UniformIntegerHyperparameter("b", 0, 10, 5)
cs.add_hyperparameter(a)
cs.add_hyperparameter(b)
cond = GreaterThanCondition(b, a, 0.5)
cs.add_condition(cond)
value = pcs_new.write(cs)
self.assertEqual(expected, value)
expected = "a integer [0, 10] [5]\n" \
"b integer [0, 10] [5]\n\n" \
"b | a > 5"
cs = ConfigurationSpace()
a = UniformIntegerHyperparameter("a", 0, 10, 5)
b = UniformIntegerHyperparameter("b", 0, 10, 5)
cs.add_hyperparameter(a)
cs.add_hyperparameter(b)
cond = GreaterThanCondition(b, a, 5)
cs.add_condition(cond)
value = pcs_new.write(cs)
self.assertEqual(expected, value)
def test_write_new_q_float(self):
expected = "Q16_float_a real [16.0, 1024.0] [520.0]"
cs = ConfigurationSpace()
cs.add_hyperparameter(
UniformFloatHyperparameter("float_a", 16, 1024, q=16))
value = pcs_new.write(cs)
self.assertEqual(expected, value)
def test_write_new_q_int(self):
expected = "Q16_int_a integer [16, 1024] [520]"
cs = ConfigurationSpace()
cs.add_hyperparameter(
UniformIntegerHyperparameter("int_a", 16, 1024, q=16))
value = pcs_new.write(cs)
self.assertEqual(expected, value)
def test_write_log_float(self):
import numpy as np
expected = "float_log '--float_log ' r (2.000000, 5.000000)\n"
float_log = UniformFloatHyperparameter("float_log",
np.exp(2),
np.exp(5),
log=True)
cs = ConfigurationSpace()
cs.add_hyperparameter(float_log)
value = irace.write(cs)
self.assertEqual(expected, value)
def test_write_in_condition(self):
expected = "ls '--ls ' c {sa,ca,ny}\ntemp '--temp ' r (0.500000, 1.000000)| ls %in% c(sa,ca)\n"
temp = UniformFloatHyperparameter("temp", 0.5, 1)
ls = CategoricalHyperparameter("ls", ["sa", "ca", "ny"], "sa")
cs = ConfigurationSpace()
cs.add_hyperparameter(temp)
cs.add_hyperparameter(ls)
c1 = InCondition(temp, ls, ['sa', 'ca'])
cs.add_condition(c1)
value = irace.write(cs)
self.assertEqual(expected, value)
def test_write_equals_condition_numerical(self):
expected = "temp '--temp ' i (1, 2)\nls '--ls ' c {sa,ca,ny}| temp==2\n"
temp = UniformIntegerHyperparameter("temp", 1, 2)
ls = CategoricalHyperparameter("ls", ["sa", "ca", "ny"], "sa")
cs = ConfigurationSpace()
cs.add_hyperparameter(temp)
cs.add_hyperparameter(ls)
c1 = EqualsCondition(ls, temp, 2)
cs.add_condition(c1)
value = irace.write(cs)
self.assertEqual(expected, value)
def test_build_new_forbidden(self):
expected = "a categorical {a, b, c} [a]\nb categorical {a, b, c} [c]\n\n" \
"{a=a, b=a}\n{a=a, b=b}\n{a=b, b=a}\n{a=b, b=b}\n"
cs = ConfigurationSpace()
a = CategoricalHyperparameter("a", ["a", "b", "c"], "a")
b = CategoricalHyperparameter("b", ["a", "b", "c"], "c")
cs.add_hyperparameter(a)
cs.add_hyperparameter(b)
fb = ForbiddenAndConjunction(ForbiddenInClause(a, ["a", "b"]),
ForbiddenInClause(b, ["a", "b"]))
cs.add_forbidden_clause(fb)
value = pcs_new.write(cs)
self.assertEqual(expected, value)
def test_write_AndConjunction_condition(self):
expected = "lp '--lp ' c {mi,bo}\nls '--ls ' c {sa,ca,ny}\ntemp '--temp ' r (0.500000, 1.000000)| ls %in% c(sa,ca) && lp %in% c(bo)\n"
temp = UniformFloatHyperparameter("temp", 0.5, 1)
ls = CategoricalHyperparameter("ls", ["sa", "ca", "ny"], "sa")
lp = CategoricalHyperparameter("lp", ["mi", "bo"], "bo")
cs = ConfigurationSpace()
cs.add_hyperparameter(temp)
cs.add_hyperparameter(lp)
cs.add_hyperparameter(ls)
c1 = InCondition(temp, ls, ['sa', 'ca'])
c2 = InCondition(temp, lp, ['bo'])
c3 = AndConjunction(c1, c2)
cs.add_condition(c3)
value = irace.write(cs)
self.assertEqual(expected, value)
def test_write_OrConjunction_condition(self):
import numpy as np
expected = "lp '--lp ' c {mi,bo}\ntemp '--temp ' r (2.000000, 5.000000)\nls '--ls ' c {sa,ca,ny}| temp==3.0 || lp %in% c(bo)\n"
temp = UniformFloatHyperparameter("temp",
np.exp(2),
np.exp(5),
log=True)
ls = CategoricalHyperparameter("ls", ["sa", "ca", "ny"], "sa")
lp = CategoricalHyperparameter("lp", ["mi", "bo"], "bo")
cs = ConfigurationSpace()
cs.add_hyperparameter(temp)
cs.add_hyperparameter(lp)
cs.add_hyperparameter(ls)
c1 = EqualsCondition(ls, temp, np.exp(3))
c2 = InCondition(ls, lp, ['bo'])
c3 = OrConjunction(c1, c2)
cs.add_condition(c3)
value = irace.write(cs)
self.assertEqual(expected, value)
def test_write_new_log_int(self):
expected = "int_log_a integer [1, 6] [2]log"
cs = ConfigurationSpace()
cs.add_hyperparameter(int_log_a)
value = pcs_new.write(cs)
self.assertEqual(expected, value)
def test_write_float(self):
expected = "float_a '--float_a ' r (16.000000, 1024.000000)\n"
cs = ConfigurationSpace()
cs.add_hyperparameter(UniformFloatHyperparameter("float_a", 16, 1024))
value = irace.write(cs)
self.assertEqual(expected, value)
def test_read_new_configuration_space_complex_conditionals(self):
classi = OrdinalHyperparameter("classi", [
"random_forest", "extra_trees", "k_nearest_neighbors", "something"
])
knn_weights = CategoricalHyperparameter("knn_weights",
["uniform", "distance"])
weather = OrdinalHyperparameter(
"weather", ["sunny", "rainy", "cloudy", "snowing"])
temperature = CategoricalHyperparameter("temperature", ["high", "low"])
rain = CategoricalHyperparameter("rain", ["yes", "no"])
gloves = OrdinalHyperparameter("gloves",
["none", "yarn", "leather", "gortex"])
heur1 = CategoricalHyperparameter("heur1", ["off", "on"])
heur2 = CategoricalHyperparameter("heur2", ["off", "on"])
heur_order = CategoricalHyperparameter("heur_order",
["heur1then2", "heur2then1"])
gloves_condition = OrConjunction(
EqualsCondition(gloves, rain, "yes"),
EqualsCondition(gloves, temperature, "low"))
heur_condition = AndConjunction(
EqualsCondition(heur_order, heur1, "on"),
EqualsCondition(heur_order, heur2, "on"))
and_conjunction = AndConjunction(
NotEqualsCondition(knn_weights, classi, "extra_trees"),
EqualsCondition(knn_weights, classi, "random_forest"))
Cl_condition = OrConjunction(
EqualsCondition(knn_weights, classi, "k_nearest_neighbors"),
and_conjunction, EqualsCondition(knn_weights, classi, "something"))
and1 = AndConjunction(EqualsCondition(temperature, weather, "rainy"),
EqualsCondition(temperature, weather, "cloudy"))
and2 = AndConjunction(
EqualsCondition(temperature, weather, "sunny"),
NotEqualsCondition(temperature, weather, "snowing"))
another_condition = OrConjunction(and1, and2)
complex_conditional_space = ConfigurationSpace()
complex_conditional_space.add_hyperparameter(classi)
complex_conditional_space.add_hyperparameter(knn_weights)
complex_conditional_space.add_hyperparameter(weather)
complex_conditional_space.add_hyperparameter(temperature)
complex_conditional_space.add_hyperparameter(rain)
complex_conditional_space.add_hyperparameter(gloves)
complex_conditional_space.add_hyperparameter(heur1)
complex_conditional_space.add_hyperparameter(heur2)
complex_conditional_space.add_hyperparameter(heur_order)
complex_conditional_space.add_condition(gloves_condition)
complex_conditional_space.add_condition(heur_condition)
complex_conditional_space.add_condition(Cl_condition)
complex_conditional_space.add_condition(another_condition)
complex_cs = list()
complex_cs.append(
"classi ordinal {random_forest,extra_trees,k_nearest_neighbors, something} [random_forest]"
)
complex_cs.append(
"knn_weights categorical {uniform, distance} [uniform]")
complex_cs.append(
"weather ordinal {sunny, rainy, cloudy, snowing} [sunny]")
complex_cs.append("temperature categorical {high, low} [high]")
complex_cs.append("rain categorical { yes, no } [yes]")
complex_cs.append(
"gloves ordinal { none, yarn, leather, gortex } [none]")
complex_cs.append("heur1 categorical { off, on } [off]")
complex_cs.append("heur2 categorical { off, on } [off]")
complex_cs.append(
"heur_order categorical { heur1then2, heur2then1 } [heur1then2]")
complex_cs.append("gloves | rain == yes || temperature == low")
complex_cs.append("heur_order | heur1 == on && heur2 == on")
complex_cs.append(
"knn_weights | classi == k_nearest_neighbors || classi != extra_trees && classi == random_forest || classi == something"
)
complex_cs.append(
"temperature | weather == rainy && weather == cloudy || weather == sunny && weather != snowing"
)
cs_new = pcs_new.read(complex_cs)
self.assertEqual(cs_new, complex_conditional_space)
def read(pcs_string, debug=False):
configuration_space = ConfigurationSpace()
conditions = []
forbidden = []
# some statistics
ct = 0
cont_ct = 0
cat_ct = 0
ord_ct = 0
line_ct = 0
for line in pcs_string:
line_ct += 1
if "#" in line:
# It contains a comment
pos = line.find("#")
line = line[:pos]
# Remove quotes and whitespaces at beginning and end
ori_line = line
line = line.replace('"', "").replace("'", "")
line = line.strip()
if "|" in line:
# It's a condition
try:
c = pp_condition.parseString(line)
conditions.append(c)
except pyparsing.ParseException:
raise NotImplementedError("Could not parse condition: %s" % line)
continue
if "}" not in line and "]" not in line:
continue
if line.startswith("{") and line.endswith("}"):
forbidden.append(line)
continue
if len(line.strip()) == 0:
continue
ct += 1
param = None
create = {"int": UniformIntegerHyperparameter,
"float": UniformFloatHyperparameter,
"categorical": CategoricalHyperparameter,
"ordinal": OrdinalHyperparameter
}
try:
param_list = pp_cont_param.parseString(line)
name = param_list[0]
if param_list[1] == 'integer':
paramtype = 'int'
elif param_list[1] == 'real':
paramtype = 'float'
else:
paramtype = None
if paramtype in ['int', 'float']:
log = param_list[10:]
param_list = param_list[:10]
if len(log) > 0:
log = log[0]
lower = float(param_list[3])
upper = float(param_list[5])
log_on = True if "log" in log else False
default_value = float(param_list[8])
param = create[paramtype](name=name, lower=lower, upper=upper,
q=None, log=log_on, default_value=default_value)
cont_ct += 1
except pyparsing.ParseException:
pass
try:
if "categorical" in line:
param_list = pp_cat_param.parseString(line)
name = param_list[0]
#choices = [choice for choice in param_list[3:-4:2]]
ori_line_segs = ori_line.split('{')
choices_str = '[' + ori_line_segs[1].split('}')[0] + ']'
choices = json.loads(choices_str)
#default_value = param_list[-2]
ori_line_segs = ori_line.split('[')
default_str = ori_line_segs[-1].split(']')[0]
default_value = json.loads(default_str)
param = create["categorical"](name=name, choices=choices, default_value=default_value)
cat_ct += 1
elif "ordinal" in line:
param_list = pp_ord_param.parseString(line)
name = param_list[0]
#sequence = [seq for seq in param_list[3:-4:2]]
ori_line_segs = ori_line.split('{')
sequence_str = '[' + ori_line_segs[1].split('}')[0] + ']'
sequence = json.loads(sequence_str)
#default_value = param_list[-2]
ori_line_segs = ori_line.split('[')
default_str = ori_line_segs[-1].split(']')[0]
default_value = json.loads(default_str)
param = create["ordinal"](name=name, sequence=sequence, default_value=default_value)
ord_ct += 1
except pyparsing.ParseException:
pass
if param is None:
raise NotImplementedError("Could not parse: %s" % line)
configuration_space.add_hyperparameter(param)
for clause in forbidden:
param_list = pp_forbidden_clause.parseString(clause)
tmp_list = []
clause_list = []
for value in param_list[1:]:
if len(tmp_list) < 3:
tmp_list.append(value)
else:
# So far, only equals is supported by SMAC
if tmp_list[1] == '=':
# TODO maybe add a check if the hyperparameter is
# actually in the configuration space
clause_list.append(ForbiddenEqualsClause(
configuration_space.get_hyperparameter(tmp_list[0]),
tmp_list[2]))
else:
raise NotImplementedError()
tmp_list = []
configuration_space.add_forbidden_clause(ForbiddenAndConjunction(
*clause_list))
conditions_per_child = OrderedDict()
for condition in conditions:
child_name = condition[0]
if child_name not in conditions_per_child:
conditions_per_child[child_name] = list()
conditions_per_child[child_name].append(condition)
for child_name in conditions_per_child:
for condition in conditions_per_child[child_name]:
condition = condition[2:]
condition = ' '.join(condition)
if '||' in str(condition):
ors = []
# 1st case we have a mixture of || and &&
if '&&' in str(condition):
ors_combis = []
for cond_parts in str(condition).split('||'):
condition = str(cond_parts).split('&&')
# if length is 1 it must be or
if len(condition) == 1:
element_list = condition[0].split()
ors_combis.append(condition_specification(child_name, element_list, configuration_space))
else:
# now taking care of ands
ands = []
for and_part in condition:
element_list = [element for part in condition for element in and_part.split()]
ands.append(condition_specification(child_name, element_list, configuration_space))
ors_combis.append(AndConjunction(*ands))
mixed_conjunction = OrConjunction(*ors_combis)
configuration_space.add_condition(mixed_conjunction)
else:
# 2nd case: we only have ors
for cond_parts in str(condition).split('||'):
element_list = [element for element in cond_parts.split()]
ors.append(condition_specification(child_name, element_list, configuration_space))
or_conjunction = OrConjunction(*ors)
configuration_space.add_condition(or_conjunction)
else:
# 3rd case: we only have ands
if '&&' in str(condition):
ands = []
for cond_parts in str(condition).split('&&'):
element_list = [element for element in cond_parts.split()]
ands.append(condition_specification(child_name, element_list, configuration_space))
and_conjunction = AndConjunction(*ands)
configuration_space.add_condition(and_conjunction)
else:
# 4th case: we have a normal condition
element_list = [element for element in condition.split()]
normal_condition = condition_specification(child_name, element_list, configuration_space)
configuration_space.add_condition(normal_condition)
return configuration_space
def test_write_int(self):
expected = "int_a '--int_a ' i (-1, 6)\n"
cs = ConfigurationSpace()
cs.add_hyperparameter(int_a)
value = irace.write(cs)
self.assertEqual(expected, value)
classifier = CategoricalHyperparameter("classifier", ["svm", "nn"])
kernel = CategoricalHyperparameter("kernel", ["rbf", "poly", "sigmoid"])
kernel_condition = EqualsCondition(kernel, classifier, "svm")
C = UniformFloatHyperparameter("C", 0.03125, 32768, log=True)
C_condition = EqualsCondition(C, classifier, "svm")
gamma = UniformFloatHyperparameter("gamma", 0.000030518, 8, log=True)
gamma_condition = EqualsCondition(gamma, kernel, "rbf")
degree = UniformIntegerHyperparameter("degree", 1, 5)
degree_condition = InCondition(degree, kernel, ["poly", "sigmoid"])
neurons = UniformIntegerHyperparameter("neurons", 16, 1024)
neurons_condition = EqualsCondition(neurons, classifier, "nn")
lr = UniformFloatHyperparameter("lr", 0.0001, 1.0)
lr_condition = EqualsCondition(lr, classifier, "nn")
preprocessing = CategoricalHyperparameter("preprocessing", ["None", "pca"])
conditional_space = ConfigurationSpace()
conditional_space.add_hyperparameter(preprocessing)
conditional_space.add_hyperparameter(classifier)
conditional_space.add_hyperparameter(kernel)
conditional_space.add_hyperparameter(C)
conditional_space.add_hyperparameter(neurons)
conditional_space.add_hyperparameter(lr)
conditional_space.add_hyperparameter(degree)
conditional_space.add_hyperparameter(gamma)
conditional_space.add_condition(C_condition)
conditional_space.add_condition(kernel_condition)
conditional_space.add_condition(lr_condition)
conditional_space.add_condition(neurons_condition)
conditional_space.add_condition(degree_condition)
conditional_space.add_condition(gamma_condition)
def test_write_ordinal(self):
expected = "ord_a '--ord_a ' o {a,b,3}\n"
cs = ConfigurationSpace()
cs.add_hyperparameter(OrdinalHyperparameter("ord_a", ["a", "b", 3]))
value = irace.write(cs)
self.assertEqual(expected, value)
def test_write_int(self):
expected = "int_a [-1, 6] [2]i"
cs = ConfigurationSpace()
cs.add_hyperparameter(int_a)
value = pcs.write(cs)
self.assertEqual(expected, value)
def read(pcs_string, debug=False):
configuration_space = ConfigurationSpace()
conditions = []
forbidden = []
# some statistics
ct = 0
cont_ct = 0
cat_ct = 0
line_ct = 0
for line in pcs_string:
line_ct += 1
if "#" in line:
# It contains a comment
pos = line.find("#")
line = line[:pos]
# Remove quotes and whitespaces at beginning and end
ori_line = line
line = line.replace('"', "").replace("'", "")
line = line.strip()
if "|" in line:
# It's a condition
try:
c = pp_condition.parseString(line)
conditions.append(c)
except pyparsing.ParseException:
raise NotImplementedError("Could not parse condition: %s" %
line)
continue
if "}" not in line and "]" not in line:
continue
if line.startswith("{") and line.endswith("}"):
forbidden.append(line)
continue
if len(line.strip()) == 0:
continue
ct += 1
param = None
create = {
"int": UniformIntegerHyperparameter,
"float": UniformFloatHyperparameter,
"categorical": CategoricalHyperparameter
}
try:
param_list = pp_cont_param.parseString(line)
il = param_list[9:]
if len(il) > 0:
il = il[0]
param_list = param_list[:9]
name = param_list[0]
lower = float(param_list[2])
upper = float(param_list[4])
paramtype = "int" if "i" in il else "float"
log = True if "l" in il else False
default_value = float(param_list[7])
param = create[paramtype](name=name,
lower=lower,
upper=upper,
q=None,
log=log,
default_value=default_value)
cont_ct += 1
except pyparsing.ParseException:
pass
try:
param_list = pp_cat_param.parseString(line)
name = param_list[0]
#choices = [c for c in param_list[2:-4:2]]
ori_line_segs = ori_line.split('{')
choices_str = '[' + ori_line_segs[1].split('}')[0] + ']'
choices = json.loads(choices_str)
#default_value = param_list[-2]
ori_line_segs = ori_line.split('[')
default_str = ori_line_segs[-1].split(']')[0]
default_value = json.loads(default_str)
param = create["categorical"](name=name,
choices=choices,
default_value=default_value)
cat_ct += 1
except pyparsing.ParseException:
pass
if param is None:
raise NotImplementedError("Could not parse: %s" % line)
configuration_space.add_hyperparameter(param)
for clause in forbidden:
# TODO test this properly!
# TODO Add a try/catch here!
# noinspection PyUnusedLocal
param_list = pp_forbidden_clause.parseString(clause)
tmp_list = []
clause_list = []
for value in param_list[1:]:
if len(tmp_list) < 3:
tmp_list.append(value)
else:
# So far, only equals is supported by SMAC
if tmp_list[1] == '=':
# TODO maybe add a check if the hyperparameter is
# actually in the configuration space
clause_list.append(
ForbiddenEqualsClause(
configuration_space.get_hyperparameter(
tmp_list[0]), tmp_list[2]))
else:
raise NotImplementedError()
tmp_list = []
configuration_space.add_forbidden_clause(
ForbiddenAndConjunction(*clause_list))
#Now handle conditions
# If there are two conditions for one child, these two conditions are an
# AND-conjunction of conditions, thus we have to connect them
conditions_per_child = OrderedDict()
for condition in conditions:
child_name = condition[0]
if child_name not in conditions_per_child:
conditions_per_child[child_name] = list()
conditions_per_child[child_name].append(condition)
for child_name in conditions_per_child:
condition_objects = []
for condition in conditions_per_child[child_name]:
child = configuration_space.get_hyperparameter(child_name)
parent_name = condition[2]
parent = configuration_space.get_hyperparameter(parent_name)
restrictions = condition[5:-1:2]
# TODO: cast the type of the restriction!
if len(restrictions) == 1:
condition = EqualsCondition(child, parent, restrictions[0])
else:
condition = InCondition(child, parent, values=restrictions)
condition_objects.append(condition)
# Now we have all condition objects for this child, so we can build a
# giant AND-conjunction of them (if number of conditions >= 2)!
if len(condition_objects) > 1:
and_conjunction = AndConjunction(*condition_objects)
configuration_space.add_condition(and_conjunction)
else:
configuration_space.add_condition(condition_objects[0])
return configuration_space
def get_branin_config_space():
"""TODO"""
cs = ConfigurationSpace()
cs.add_hyperparameter(UniformFloatHyperparameter('x', -5, 10))
cs.add_hyperparameter(UniformFloatHyperparameter('y', 0, 15))
return cs