def test_constant(self):
# Test construction
c1 = Constant("value", 1)
c2 = Constant("value", 1)
c3 = Constant("value", 2)
c4 = Constant("valuee", 1)
c5 = Constant("valueee", 2)
# Test the representation
self.assertEqual("value, Type: Constant, Value: 1", c1.__repr__())
# Test the equals operator (and the ne operator in the last line)
self.assertFalse(c1 == 1)
self.assertEqual(c1, c2)
self.assertFalse(c1 == c3)
self.assertFalse(c1 == c4)
self.assertTrue(c1 != c5)
# Test that only string, integers and floats are allowed
self.assertRaises(TypeError, Constant, "value", dict())
self.assertRaises(TypeError, Constant, "value", None)
self.assertRaises(TypeError, Constant, "value", True)
# Test that only string names are allowed
self.assertRaises(TypeError, Constant, 1, "value")
self.assertRaises(TypeError, Constant, dict(), "value")
self.assertRaises(TypeError, Constant, None, "value")
self.assertRaises(TypeError, Constant, True, "value")
def get_hyperparameter_search_space(dataset_properties=None):
n_neighbors = UniformIntegerHyperparameter(name="n_neighbors",
lower=1,
upper=100,
default=1)
weights = CategoricalHyperparameter(name="weights",
choices=["uniform", "distance"],
default="uniform")
metric = UnParametrizedHyperparameter(name="metric", value="minkowski")
algorithm = Constant(name='algorithm', value="auto")
p = CategoricalHyperparameter(name="p", choices=[1, 2, 5], default=2)
leaf_size = Constant(name="leaf_size", value=30)
# Unparametrized
# TODO: If we further parametrize 'metric' we need more metric params
metric = UnParametrizedHyperparameter(name="metric", value="minkowski")
cs = ConfigurationSpace()
cs.add_hyperparameter(n_neighbors)
cs.add_hyperparameter(weights)
cs.add_hyperparameter(metric)
cs.add_hyperparameter(algorithm)
cs.add_hyperparameter(p)
cs.add_hyperparameter(leaf_size)
# Conditions
metric_p = EqualsCondition(parent=metric, child=p, value="minkowski")
cs.add_condition(metric_p)
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
C = cs.add_hyperparameter(UniformFloatHyperparameter(
"C", 0.03125, 32768, log=True, default=1.0))
loss = cs.add_hyperparameter(CategoricalHyperparameter(
"loss", ["epsilon_insensitive", "squared_epsilon_insensitive"],
default="squared_epsilon_insensitive"))
# Random Guess
epsilon = cs.add_hyperparameter(UniformFloatHyperparameter(
name="epsilon", lower=0.001, upper=1, default=0.1, log=True))
dual = cs.add_hyperparameter(Constant("dual", "False"))
# These are set ad-hoc
tol = cs.add_hyperparameter(UniformFloatHyperparameter(
"tol", 1e-5, 1e-1, default=1e-4, log=True))
fit_intercept = cs.add_hyperparameter(Constant("fit_intercept", "True"))
intercept_scaling = cs.add_hyperparameter(Constant(
"intercept_scaling", 1))
dual_and_loss = ForbiddenAndConjunction(
ForbiddenEqualsClause(dual, "False"),
ForbiddenEqualsClause(loss, "epsilon_insensitive")
)
cs.add_forbidden_clause(dual_and_loss)
return cs
def test_and_conjunction(self):
self.assertRaises(TypeError, AndConjunction, "String1", "String2")
hp1 = CategoricalHyperparameter("input1", [0, 1])
hp2 = CategoricalHyperparameter("input2", [0, 1])
hp3 = CategoricalHyperparameter("input3", [0, 1])
hp4 = Constant("And", "True")
cond1 = EqualsCondition(hp4, hp1, 1)
# Only one condition in an AndConjunction!
self.assertRaises(ValueError, AndConjunction, cond1)
cond2 = EqualsCondition(hp4, hp2, 1)
cond3 = EqualsCondition(hp4, hp3, 1)
andconj1 = AndConjunction(cond1, cond2)
andconj1_ = AndConjunction(cond1, cond2)
self.assertEqual(andconj1, andconj1_)
andconj2 = AndConjunction(cond2, cond3)
self.assertNotEqual(andconj1, andconj2)
andconj3 = AndConjunction(cond1, cond2, cond3)
self.assertEqual(
"(And | input1 == 1 && And | input2 == 1 && And | "
"input3 == 1)", str(andconj3))
# Test __eq__
self.assertNotEqual(andconj1, andconj3)
self.assertNotEqual(andconj1, "String")
def test_or_conjunction(self):
self.assertRaises(TypeError, AndConjunction, "String1", "String2")
hp1 = CategoricalHyperparameter("input1", [0, 1])
hp2 = CategoricalHyperparameter("input2", [0, 1])
hp3 = CategoricalHyperparameter("input3", [0, 1])
hp4 = Constant("Or", "True")
cond1 = EqualsCondition(hp4, hp1, 1)
self.assertRaises(ValueError, OrConjunction, cond1)
cond2 = EqualsCondition(hp4, hp2, 1)
cond3 = EqualsCondition(hp4, hp3, 1)
andconj1 = OrConjunction(cond1, cond2)
andconj1_ = OrConjunction(cond1, cond2)
self.assertEqual(andconj1, andconj1_)
andconj2 = OrConjunction(cond2, cond3)
self.assertNotEqual(andconj1, andconj2)
andconj3 = OrConjunction(cond1, cond2, cond3)
self.assertEqual(
"(Or | input1 == 1 || Or | input2 == 1 || Or | "
"input3 == 1)", str(andconj3))
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConstrainedFeedNet.get_hyperparameter_search_space()
solver = Constant(name='solver', value='sgd')
cs.add_hyperparameter(solver)
# lr policy is fixed by default
return cs
def get_hyperparameter_search_space(dataset_properties=None):
#n_estimators = UniformIntegerHyperparameter(
# "n_estimators", 10, 100, default=10)
n_estimators = Constant("n_estimators", 100)
criterion = CategoricalHyperparameter("criterion", ["gini", "entropy"],
default="gini")
#max_features = UniformFloatHyperparameter(
# "max_features", 0.01, 0.5, default=0.2)
max_features = UniformFloatHyperparameter("max_features",
0.5,
5,
default=1)
max_depth = UnParametrizedHyperparameter("max_depth", "None")
min_samples_split = UniformIntegerHyperparameter("min_samples_split",
2,
20,
default=2)
min_samples_leaf = UniformIntegerHyperparameter("min_samples_leaf",
1,
20,
default=1)
max_leaf_nodes = UnParametrizedHyperparameter("max_leaf_nodes", "None")
bootstrap = CategoricalHyperparameter("bootstrap", ["True", "False"],
default="True")
cs = ConfigurationSpace()
cs.add_hyperparameter(n_estimators)
cs.add_hyperparameter(criterion)
cs.add_hyperparameter(max_features)
cs.add_hyperparameter(max_depth)
cs.add_hyperparameter(min_samples_split)
cs.add_hyperparameter(min_samples_leaf)
cs.add_hyperparameter(max_leaf_nodes)
cs.add_hyperparameter(bootstrap)
return cs
def _lr_policy_configuration_space(cs, policy=None):
if policy == 'inv':
lr_policy = Constant(name='lr_policy', value='inv')
gamma = UniformFloatHyperparameter(name="gamma",
lower=1e-2,
upper=1.0,
log=True,
default=1e-2)
power = UniformFloatHyperparameter("power", 0.0, 1.0, default=0.5)
cs.add_hyperparameter(lr_policy)
cs.add_hyperparameter(gamma)
cs.add_hyperparameter(power)
elif policy == 'exp':
lr_policy = Constant(name='lr_policy', value='exp')
gamma = UniformFloatHyperparameter(name="gamma",
lower=0.7,
upper=1.0,
default=0.79)
cs.add_hyperparameter(lr_policy)
cs.add_hyperparameter(gamma)
elif policy == 'step':
lr_policy = Constant(name='lr_policy', value='step')
gamma = UniformFloatHyperparameter(name="gamma",
lower=1e-2,
upper=1.0,
log=True,
default=1e-2)
epoch_step = CategoricalHyperparameter("epoch_step", [6, 8, 12],
default=8)
cs.add_hyperparameter(lr_policy)
cs.add_hyperparameter(gamma)
cs.add_hyperparameter(epoch_step)
else:
lr_policy = Constant(name='lr_policy', value='fixed')
cs.add_hyperparameter(lr_policy)
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
loss = cs.add_hyperparameter(Constant("loss", "deviance"))
learning_rate = cs.add_hyperparameter(
UniformFloatHyperparameter(name="learning_rate",
lower=0.0001,
upper=1,
default=0.1,
log=True))
n_estimators = cs.add_hyperparameter(Constant("n_estimators", 100))
max_depth = cs.add_hyperparameter(
UniformIntegerHyperparameter(name="max_depth",
lower=1,
upper=10,
default=3))
min_samples_split = cs.add_hyperparameter(
UniformIntegerHyperparameter(name="min_samples_split",
lower=2,
upper=20,
default=2,
log=False))
min_samples_leaf = cs.add_hyperparameter(
UniformIntegerHyperparameter(name="min_samples_leaf",
lower=1,
upper=20,
default=1,
log=False))
min_weight_fraction_leaf = cs.add_hyperparameter(
UnParametrizedHyperparameter("min_weight_fraction_leaf", 0.))
subsample = cs.add_hyperparameter(
UniformFloatHyperparameter(name="subsample",
lower=0.01,
upper=1.0,
default=1.0,
log=False))
max_features = cs.add_hyperparameter(
UniformFloatHyperparameter("max_features", 0.5, 5, default=1))
max_leaf_nodes = cs.add_hyperparameter(
UnParametrizedHyperparameter(name="max_leaf_nodes", value="None"))
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConstrainedFeedNet.get_hyperparameter_search_space()
solver = Constant(name='solver', value='momentum')
cs.add_hyperparameter(solver)
momentum = UniformFloatHyperparameter("momentum",
0.3,
0.999,
default=0.9)
cs.add_hyperparameter(momentum)
return cs
def test_sample_configuration(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
# This automatically checks the configuration!
Configuration(cs, dict(parent=0, child=5))
# and now for something more complicated
cs = ConfigurationSpace(seed=1)
hp1 = CategoricalHyperparameter("input1", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = CategoricalHyperparameter("input2", [0, 1])
cs.add_hyperparameter(hp2)
hp3 = CategoricalHyperparameter("input3", [0, 1])
cs.add_hyperparameter(hp3)
hp4 = CategoricalHyperparameter("input4", [0, 1])
cs.add_hyperparameter(hp4)
hp5 = CategoricalHyperparameter("input5", [0, 1])
cs.add_hyperparameter(hp5)
hp6 = Constant("AND", "True")
cs.add_hyperparameter(hp6)
cond1 = EqualsCondition(hp6, hp1, 1)
cond2 = NotEqualsCondition(hp6, hp2, 1)
cond3 = InCondition(hp6, hp3, [1])
cond4 = EqualsCondition(hp5, hp3, 1)
cond5 = EqualsCondition(hp4, hp5, 1)
cond6 = EqualsCondition(hp6, hp4, 1)
cond7 = EqualsCondition(hp6, hp5, 1)
conj1 = AndConjunction(cond1, cond2)
conj2 = OrConjunction(conj1, cond3)
conj3 = AndConjunction(conj2, cond6, cond7)
cs.add_condition(cond4)
cs.add_condition(cond5)
cs.add_condition(conj3)
samples = []
for i in range(5):
cs.seed(1)
samples.append([])
for j in range(100):
sample = cs.sample_configuration()
samples[-1].append(sample)
if i > 0:
for j in range(100):
self.assertEqual(samples[-1][j], samples[-2][j])
def test_get_hyperparameters_topological_sort(self):
for iteration in range(10):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
# This automatically checks the configuration!
Configuration(cs, dict(parent=0, child=5))
# and now for something more complicated
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("input1", [0, 1])
hp2 = CategoricalHyperparameter("input2", [0, 1])
hp3 = CategoricalHyperparameter("input3", [0, 1])
hp4 = CategoricalHyperparameter("input4", [0, 1])
hp5 = CategoricalHyperparameter("input5", [0, 1])
hp6 = Constant("AND", "True")
# More top-level hyperparameters
hp7 = CategoricalHyperparameter("input7", [0, 1])
hps = [hp1, hp2, hp3, hp4, hp5, hp6, hp7]
random.shuffle(hps)
for hp in hps:
cs.add_hyperparameter(hp)
cond1 = EqualsCondition(hp6, hp1, 1)
cond2 = NotEqualsCondition(hp6, hp2, 1)
cond3 = InCondition(hp6, hp3, [1])
cond4 = EqualsCondition(hp5, hp3, 1)
cond5 = EqualsCondition(hp4, hp5, 1)
cond6 = EqualsCondition(hp6, hp4, 1)
cond7 = EqualsCondition(hp6, hp5, 1)
conj1 = AndConjunction(cond1, cond2)
conj2 = OrConjunction(conj1, cond3)
conj3 = AndConjunction(conj2, cond6, cond7)
cs.add_condition(cond4)
cs.add_condition(cond5)
cs.add_condition(conj3)
hps = cs.get_hyperparameters()
self.assertEqual(hps.index(hp1), 0)
self.assertEqual(hps.index(hp2), 1)
self.assertEqual(hps.index(hp3), 2)
self.assertEqual(hps.index(hp7), 3)
self.assertEqual(hps.index(hp5), 4)
self.assertEqual(hps.index(hp4), 5)
self.assertEqual(hps.index(hp6), 6)
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConstrainedFeedNet.get_hyperparameter_search_space()
solver = Constant(name='solver', value='adam')
cs.add_hyperparameter(solver)
beta1 = UniformFloatHyperparameter("beta1",
1e-4,
0.1,
log=True,
default=0.1)
beta2 = UniformFloatHyperparameter("beta2",
1e-4,
0.1,
log=True,
default=0.1)
cs.add_hyperparameter(beta1)
cs.add_hyperparameter(beta2)
return cs
def get_hyperparameter_search_space(dataset_properties=None):
percentile = UniformFloatHyperparameter(
name="percentile", lower=10, upper=90, default=50)
score_func = CategoricalHyperparameter(
name="score_func", choices=["chi2", "f_classif"], default="chi2")
if dataset_properties is not None:
# Chi2 can handle sparse data, so we respect this
if 'sparse' in dataset_properties and dataset_properties['sparse']:
score_func = Constant(
name="score_func", value="chi2")
cs = ConfigurationSpace()
cs.add_hyperparameter(percentile)
cs.add_hyperparameter(score_func)
return cs
def test_all_components_have_the_same_child(self):
hp1 = CategoricalHyperparameter("input1", [0, 1])
hp2 = CategoricalHyperparameter("input2", [0, 1])
hp3 = CategoricalHyperparameter("input3", [0, 1])
hp4 = CategoricalHyperparameter("input4", [0, 1])
hp5 = CategoricalHyperparameter("input5", [0, 1])
hp6 = Constant("AND", "True")
cond1 = EqualsCondition(hp1, hp2, 1)
cond2 = EqualsCondition(hp1, hp3, 1)
cond3 = EqualsCondition(hp1, hp4, 1)
cond4 = EqualsCondition(hp6, hp4, 1)
cond5 = EqualsCondition(hp6, hp5, 1)
AndConjunction(cond1, cond2, cond3)
AndConjunction(cond4, cond5)
self.assertRaisesRegexp(
ValueError, "All Conjunctions and Conditions must have "
"the same child.", AndConjunction, cond1, cond4)
def test_add_second_condition_wo_conjunction(self):
hp1 = CategoricalHyperparameter("input1", [0, 1])
hp2 = CategoricalHyperparameter("input2", [0, 1])
hp3 = Constant("And", "True")
cond1 = EqualsCondition(hp3, hp1, 1)
cond2 = EqualsCondition(hp3, hp2, 1)
cs = ConfigurationSpace()
cs.add_hyperparameter(hp1)
cs.add_hyperparameter(hp2)
cs.add_hyperparameter(hp3)
cs.add_condition(cond1)
self.assertRaisesRegexp(
ValueError, "Adding a second condition \(different\) for a "
"hyperparameter is ambigouos and "
"therefore forbidden. Add a conjunction "
"instead!", cs.add_condition, cond2)
def get_hyperparameter_search_space(dataset_properties=None):
loss = CategoricalHyperparameter(
name="loss", choices=["ls", "lad"],
default='ls') #, "huber", "quantile"], default='ls')
learning_rate = UniformFloatHyperparameter(name="learning_rate",
lower=0.0001,
upper=1,
default=0.1,
log=True)
subsample = UniformFloatHyperparameter(name="subsample",
lower=0.01,
upper=1.0,
default=1.0,
log=False)
n_estimators = Constant("n_estimators", 100)
max_features = UniformFloatHyperparameter("max_features",
0.5,
5,
default=1)
max_depth = UniformIntegerHyperparameter(name="max_depth",
lower=1,
upper=10,
default=3)
min_samples_split = UniformIntegerHyperparameter(
name="min_samples_split", lower=2, upper=20, default=2, log=False)
min_samples_leaf = UniformIntegerHyperparameter(
name="min_samples_leaf", lower=1, upper=20, default=1, log=False)
cs = ConfigurationSpace()
cs.add_hyperparameter(n_estimators)
cs.add_hyperparameter(loss)
cs.add_hyperparameter(learning_rate)
cs.add_hyperparameter(max_features)
cs.add_hyperparameter(max_depth)
cs.add_hyperparameter(min_samples_split)
cs.add_hyperparameter(min_samples_leaf)
cs.add_hyperparameter(subsample)
return cs
def test_add_conjunction(self):
hp1 = CategoricalHyperparameter("input1", [0, 1])
hp2 = CategoricalHyperparameter("input2", [0, 1])
hp3 = CategoricalHyperparameter("input3", [0, 1])
hp4 = Constant("And", "True")
cond1 = EqualsCondition(hp4, hp1, 1)
cond2 = EqualsCondition(hp4, hp2, 1)
cond3 = EqualsCondition(hp4, hp3, 1)
andconj1 = AndConjunction(cond1, cond2, cond3)
cs = ConfigurationSpace()
cs.add_hyperparameter(hp1)
cs.add_hyperparameter(hp2)
cs.add_hyperparameter(hp3)
cs.add_hyperparameter(hp4)
cs.add_condition(andconj1)
self.assertNotIn(hp4, cs.get_all_uncoditional_hyperparameters())
def get_hyperparameter_search_space(dataset_properties=None):
alpha = UniformFloatHyperparameter(name="alpha",
lower=0.01,
upper=0.5,
default=0.1)
score_func = CategoricalHyperparameter(name="score_func",
choices=["chi2", "f_classif"],
default="chi2")
if dataset_properties is not None:
# Chi2 can handle sparse data, so we respect this
if 'sparse' in dataset_properties and dataset_properties['sparse']:
score_func = Constant(name="score_func", value="chi2")
mode = CategoricalHyperparameter('mode', ['fpr', 'fdr', 'fwe'], 'fpr')
cs = ConfigurationSpace()
cs.add_hyperparameter(alpha)
cs.add_hyperparameter(score_func)
cs.add_hyperparameter(mode)
return cs
def get_hyperparameter_search_space(dataset_properties=None):
n_estimators = UniformIntegerHyperparameter(name="n_estimators",
lower=10, upper=100,
default=10)
max_depth = UniformIntegerHyperparameter(name="max_depth",
lower=2, upper=10,
default=5)
min_samples_split = UniformIntegerHyperparameter(name="min_samples_split",
lower=2, upper=20,
default=2)
min_samples_leaf = UniformIntegerHyperparameter(name="min_samples_leaf",
lower=1, upper=20,
default=1)
min_weight_fraction_leaf = Constant('min_weight_fraction_leaf', 1.0)
max_leaf_nodes = UnParametrizedHyperparameter(name="max_leaf_nodes",
value="None")
cs = ConfigurationSpace()
cs.add_hyperparameter(n_estimators)
cs.add_hyperparameter(max_depth)
cs.add_hyperparameter(min_samples_split)
cs.add_hyperparameter(min_samples_leaf)
cs.add_hyperparameter(min_weight_fraction_leaf)
cs.add_hyperparameter(max_leaf_nodes)
return cs
def test_nested_conjunctions(self):
hp1 = CategoricalHyperparameter("input1", [0, 1])
hp2 = CategoricalHyperparameter("input2", [0, 1])
hp3 = CategoricalHyperparameter("input3", [0, 1])
hp4 = CategoricalHyperparameter("input4", [0, 1])
hp5 = CategoricalHyperparameter("input5", [0, 1])
hp6 = Constant("AND", "True")
cond1 = EqualsCondition(hp6, hp1, 1)
cond2 = EqualsCondition(hp6, hp2, 1)
cond3 = EqualsCondition(hp6, hp3, 1)
cond4 = EqualsCondition(hp6, hp4, 1)
cond5 = EqualsCondition(hp6, hp5, 1)
conj1 = AndConjunction(cond1, cond2)
conj2 = OrConjunction(conj1, cond3)
conj3 = AndConjunction(conj2, cond4, cond5)
# TODO: this does not look nice, And should depend on a large
# conjunction, there should not be many ANDs inside this string!
self.assertEqual(
"(((AND | input1 == 1 && AND | input2 == 1) || AND | "
"input3 == 1) && AND | input4 == 1 && AND | input5 "
"== 1)", str(conj3))
def get_hyperparameter_search_space(dataset_properties=None):
# Constrain the search space of the DeepFeedNet
# Fixed Architecture for MNIST
batch_size = Constant(name='batch_size', value=963)
# TODO: Decimal library to work around floating point issue
dropout_layer_1 = Constant(name='dropout_layer_1', value=0.39426633933)
dropout_output = Constant(name='dropout_output', value=0.085813712701)
num_layers = Constant(name='num_layers', value='c')
num_units_layer_1 = Constant(name='num_units_layer_1', value=1861)
number_updates = Constant(name='number_updates', value=1105)
std_layer_1 = Constant(name='std_layer_1', value=0.00351015701)
# To Optimize for all cases
l2 = UniformFloatHyperparameter("lambda2",
1e-6,
1e-2,
log=True,
default=1e-3)
lr = UniformFloatHyperparameter("learning_rate",
1e-4,
1e-1,
log=True,
default=1e-2)
cs = ConfigurationSpace()
cs.add_hyperparameter(batch_size)
cs.add_hyperparameter(number_updates)
cs.add_hyperparameter(num_layers)
cs.add_hyperparameter(num_units_layer_1)
cs.add_hyperparameter(dropout_layer_1)
cs.add_hyperparameter(dropout_output)
cs.add_hyperparameter(std_layer_1)
cs.add_hyperparameter(lr)
cs.add_hyperparameter(l2)
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConstrainedFeedNet.get_hyperparameter_search_space()
solver = Constant(name='solver', value='adagrad')
cs.add_hyperparameter(solver)
return cs