def test_normalint_to_uniform(self):
with pytest.warns(UserWarning, match="Setting quantization < 1 for Integer "
"Hyperparameter 'param' has no effect"):
f1 = NormalIntegerHyperparameter("param", 0, 10, q=0.1)
f1_expected = UniformIntegerHyperparameter("param", -30, 30)
f1_actual = f1.to_uniform()
self.assertEqual(f1_expected, f1_actual)
def test_normalint_to_uniform(self):
with pytest.warns(UserWarning, match="Setting quantization < 1 for Integer "
"Hyperparameter 'param' has no effect"):
f1 = NormalIntegerHyperparameter("param", 0, 10, q=0.1)
f1_expected = UniformIntegerHyperparameter("param", -30, 30)
f1_actual = f1.to_uniform()
self.assertEqual(f1_expected, f1_actual)
def test__sample_NormalIntegerHyperparameter(self):
# mean zero, std 1
hp = NormalIntegerHyperparameter("uihp", 0, 1)
values = []
rs = np.random.RandomState(1)
for i in range(100):
values.append(hp._sample(rs))
self.assertEqual(len(np.unique(values)), 5)
def test__sample_NormalIntegerHyperparameter(self):
# mean zero, std 1
hp = NormalIntegerHyperparameter("uihp", 0, 1)
values = []
rs = np.random.RandomState(1)
for i in range(100):
values.append(hp._sample(rs))
self.assertEqual(len(np.unique(values)), 5)
def get_complete_configspace():
"""Creates a configspace that includes all kinds of parameters with
complicated values. The idea is to provide a configspace that can be
used to check modules using ConfigSpace as a dependency to check
compatibility with e.g. Constants, log-scale, etc.
Returns
-------
cs: ConfigurationSpace
cs containing all kinds of parameters
"""
cs = ConfigurationSpace()
hp = {}
# Add Constants for all allowed types ('int', 'float', 'string')
hp['alpha'] = Constant("alpha", 0.0001)
hp['tol'] = Constant("tol", '0.0001')
hp['verbose'] = Constant("verbose", 1)
# Add numericals
# Add Floats
hp['beta1'] = UniformFloatHyperparameter("beta1", 0.85, 0.95, log=False)
hp['power_t'] = NormalFloatHyperparameter("power_t", mu=0.5, sigma=0.1, log=False)
# Add Ints
hp['momentum'] = UniformIntegerHyperparameter("momentum", 0, 100, False)
hp['beta2'] = NormalIntegerHyperparameter("beta2", mu=1, sigma=0.001, log=False)
# Add Floats (log)
hp['learning_rate_init'] = UniformFloatHyperparameter("learning_rate_init", 0.0001, 0.1, log=True)
hp['random1'] = NormalFloatHyperparameter("NormalFloat", mu=0, sigma=1, default_value=1, log=True)
# Add Ints (log)
hp['random2'] = UniformIntegerHyperparameter("UniformInt", 2, 100, log=True)
hp['random3'] = NormalIntegerHyperparameter("NormalInt", mu=0, sigma=1, default_value=1, log=True)
# Add Categorical for allowed types
hp['activation'] = CategoricalHyperparameter('activation', choices=['identity', 'logistic', 'tanh', 'relu'])
hp['solver'] = CategoricalHyperparameter('solver', choices=[-2, 0, 2]) # corrresponds to: ‘lbfgs’, ‘sgd’, ‘adam’
hp['batch_size_auto'] = CategoricalHyperparameter('batch_size_auto', choices=[True, False])
hp['learning_rate'] = CategoricalHyperparameter('learning_rate', choices=[-0.5, 0.0, 0.5]) # corresponds to {‘constant’, ‘invscaling’, ‘adaptive’}
# Add Ordinal
hp['batch_size'] = OrdinalHyperparameter('batch_size', sequence=[32, 64.0, '128'])
for k, v in hp.items():
cs.add_hyperparameter(v)
# learning_rate only with sgd
c = InCondition(hp['learning_rate'], hp['solver'], [0])
c = EqualsCondition(hp['momentum'], hp['solver'], 0)
# learning_rate_init only with sgd or adam
cs.add_condition(OrConjunction(EqualsCondition(hp['learning_rate'], hp['solver'], 0), # sgd
EqualsCondition(hp['learning_rate'], hp['solver'], 2))) # adam
# batch_size only with not batch_size_auto
cs.add_condition(NotEqualsCondition(hp['batch_size'], hp['batch_size_auto'], True))
# complicated way for solver == sgd
#cs.add_condition(AndConjunction(LessThanCondition(hp['power_t'], hp['solver'], 1),
# GreaterThanCondition(hp['power_t'], hp['solver'], -1)))
# betas with adam
cs.add_condition(EqualsCondition(hp['beta1'], hp['solver'], 2))
cs.add_condition(EqualsCondition(hp['beta2'], hp['solver'], 2))
return cs
def test_sample_NormalIntegerHyperparameter(self):
def sample(hp):
lower = -30
upper = 30
rs = np.random.RandomState(1)
counts_per_bin = [0 for i in range(21)]
for i in range(100000):
value = hp.sample(rs)
sample = float(value)
if sample < lower:
sample = lower
if sample > upper:
sample = upper
index = int((sample - lower) / (upper - lower) * 20)
counts_per_bin[index] += 1
self.assertIsInstance(value, int)
return counts_per_bin
hp = NormalIntegerHyperparameter("nihp", 0, 10)
self.assertEqual(sample(hp),
[305, 422, 835, 1596, 2682, 4531, 6572, 8670, 10649,
11510, 11854, 11223, 9309, 7244, 5155, 3406, 2025,
1079, 514, 249, 170])
self.assertEqual(sample(hp), sample(hp))
def _construct_hyperparameter(hyperparameter: Dict) -> Hyperparameter:
hp_type = hyperparameter['type']
name = hyperparameter['name']
if hp_type == 'constant':
return Constant(
name=name,
value=hyperparameter['value'],
)
elif hp_type == 'unparametrized':
return UnParametrizedHyperparameter(
name=name,
value=hyperparameter['value'],
)
elif hp_type == 'uniform_float':
return UniformFloatHyperparameter(
name=name,
log=hyperparameter['log'],
lower=hyperparameter['lower'],
upper=hyperparameter['upper'],
default_value=hyperparameter['default'],
)
elif hp_type == 'normal_float':
return NormalFloatHyperparameter(
name=name,
log=hyperparameter['log'],
mu=hyperparameter['mu'],
sigma=hyperparameter['sigma'],
default_value=hyperparameter['default'],
)
elif hp_type == 'uniform_int':
return UniformIntegerHyperparameter(
name=name,
log=hyperparameter['log'],
lower=hyperparameter['lower'],
upper=hyperparameter['upper'],
default_value=hyperparameter['default'],
)
elif hp_type == 'normal_int':
return NormalIntegerHyperparameter(
name=name,
log=hyperparameter['log'],
lower=hyperparameter['lower'],
upper=hyperparameter['upper'],
default_value=hyperparameter['default'],
)
elif hp_type == 'categorical':
return CategoricalHyperparameter(
name=name,
choices=hyperparameter['choices'],
default_value=hyperparameter['default'],
weights=hyperparameter.get('probabilities'),
)
elif hp_type == 'ordinal':
return OrdinalHyperparameter(
name=name,
sequence=hyperparameter['sequence'],
default_value=hyperparameter['default'],
)
else:
raise ValueError(hp_type)
def test_hyperparam_representation(self):
# Float
f1 = UniformFloatHyperparameter("param", 1, 100, log=True)
self.assertEqual(
"param, Type: UniformFloat, Range: [1.0, 100.0], Default: 10.0, on log-scale",
repr(f1)
)
f2 = NormalFloatHyperparameter("param", 8, 99.1, log=False)
self.assertEqual(
"param, Type: NormalFloat, Mu: 8.0 Sigma: 99.1, Default: 8.0",
repr(f2)
)
i1 = UniformIntegerHyperparameter("param", 0, 100)
self.assertEqual(
"param, Type: UniformInteger, Range: [0, 100], Default: 50",
repr(i1)
)
i2 = NormalIntegerHyperparameter("param", 5, 8)
self.assertEqual(
"param, Type: NormalInteger, Mu: 5 Sigma: 8, Default: 5",
repr(i2)
)
o1 = OrdinalHyperparameter("temp", ["freezing", "cold", "warm", "hot"])
self.assertEqual(
"temp, Type: Ordinal, Sequence: {freezing, cold, warm, hot}, Default: freezing",
repr(o1)
)
c1 = CategoricalHyperparameter("param", [True, False])
self.assertEqual(
"param, Type: Categorical, Choices: {True, False}, Default: True",
repr(c1)
)
def test_normalint_is_legal(self):
with pytest.warns(RuntimeWarning, match='divide by zero encountered in log'):
f1 = NormalIntegerHyperparameter("param", 0, 10, q=0.1, log=True)
self.assertFalse(f1.is_legal(3.1))
self.assertFalse(f1.is_legal(3.0)) # 3.0 behaves like an Integer
self.assertFalse(f1.is_legal("BlaBlaBla"))
self.assertTrue(f1.is_legal(2))
self.assertTrue(f1.is_legal(-15))
# Test is legal vector
self.assertTrue(f1.is_legal_vector(1.0))
self.assertTrue(f1.is_legal_vector(0.0))
self.assertTrue(f1.is_legal_vector(0))
self.assertTrue(f1.is_legal_vector(0.3))
self.assertTrue(f1.is_legal_vector(-0.1))
self.assertTrue(f1.is_legal_vector(1.1))
self.assertRaises(TypeError, f1.is_legal_vector, "Hahaha")
def test_normalint_legal_float_values(self):
n_iter = NormalIntegerHyperparameter("n_iter", 0, 1., default_value=2.0)
self.assertIsInstance(n_iter.default_value, int)
self.assertRaisesRegex(ValueError, r"For the Integer parameter n_iter, "
r"the value must be an Integer, too."
r" Right now it is a "
r"<(type|class) 'float'>"
r" with value 0.5.",
UniformIntegerHyperparameter, "n_iter", 0,
1., default_value=0.5)
def test_normalint_is_legal(self):
f1 = NormalIntegerHyperparameter("param", 0, 10, q=0.1, log=True)
self.assertFalse(f1.is_legal(3.1))
self.assertFalse(f1.is_legal(3.0)) # 3.0 behaves like an Integer
self.assertFalse(f1.is_legal("BlaBlaBla"))
self.assertTrue(f1.is_legal(2))
self.assertTrue(f1.is_legal(-15))
def test_meta_field(self):
cs = ConfigurationSpace()
cs.add_hyperparameter(
UniformIntegerHyperparameter("uihp",
lower=1,
upper=10,
meta=dict(uihp=True)))
cs.add_hyperparameter(
NormalIntegerHyperparameter("nihp",
mu=0,
sigma=1,
meta=dict(nihp=True)))
cs.add_hyperparameter(
UniformFloatHyperparameter("ufhp",
lower=1,
upper=10,
meta=dict(ufhp=True)))
cs.add_hyperparameter(
NormalFloatHyperparameter("nfhp",
mu=0,
sigma=1,
meta=dict(nfhp=True)))
cs.add_hyperparameter(
CategoricalHyperparameter("chp",
choices=['1', '2', '3'],
meta=dict(chp=True)))
cs.add_hyperparameter(
OrdinalHyperparameter("ohp",
sequence=['1', '2', '3'],
meta=dict(ohp=True)))
cs.add_hyperparameter(Constant("const", value=1,
meta=dict(const=True)))
parent = ConfigurationSpace()
parent.add_configuration_space("sub", cs, delimiter=':')
self.assertEqual(
parent.get_hyperparameter("sub:uihp").meta, dict(uihp=True))
self.assertEqual(
parent.get_hyperparameter("sub:nihp").meta, dict(nihp=True))
self.assertEqual(
parent.get_hyperparameter("sub:ufhp").meta, dict(ufhp=True))
self.assertEqual(
parent.get_hyperparameter("sub:nfhp").meta, dict(nfhp=True))
self.assertEqual(
parent.get_hyperparameter("sub:chp").meta, dict(chp=True))
self.assertEqual(
parent.get_hyperparameter("sub:ohp").meta, dict(ohp=True))
self.assertEqual(
parent.get_hyperparameter("sub:const").meta, dict(const=True))
def test_repr_roundtrip(self):
cs = ConfigurationSpace()
cs.add_hyperparameter(
UniformIntegerHyperparameter("uihp", lower=1, upper=10))
cs.add_hyperparameter(
NormalIntegerHyperparameter("nihp", mu=0, sigma=1))
cs.add_hyperparameter(
UniformFloatHyperparameter("ufhp", lower=1, upper=10))
cs.add_hyperparameter(NormalFloatHyperparameter("nfhp", mu=0, sigma=1))
cs.add_hyperparameter(
CategoricalHyperparameter("chp", choices=['1', '2', '3']))
cs.add_hyperparameter(
OrdinalHyperparameter("ohp", sequence=['1', '2', '3']))
cs.add_hyperparameter(Constant("const", value=1))
default = cs.get_default_configuration()
repr = default.__repr__()
repr = repr.replace('})', '}, configuration_space=cs)')
config = eval(repr)
self.assertEqual(default, config)
def test_normalint_to_uniform(self):
f1 = NormalIntegerHyperparameter("param", 0, 10, q=0.1)
f1_expected = UniformIntegerHyperparameter("param", -30, 30)
f1_actual = f1.to_uniform()
self.assertEqual(f1_expected, f1_actual)
def test_normalint(self):
# TODO test for unequal!
f1 = NormalIntegerHyperparameter("param", 0.5, 5.5)
f1_ = NormalIntegerHyperparameter("param", 0.5, 5.5)
self.assertEqual(f1, f1_)
self.assertEqual(
"param, Type: NormalInteger, Mu: 0.5 Sigma: 5.5, Default: 0.5",
str(f1))
f2 = NormalIntegerHyperparameter("param", 0, 10, q=0.1)
f2_ = NormalIntegerHyperparameter("param", 0, 10, q=0.1)
self.assertEqual(f2, f2_)
self.assertEqual(
"param, Type: NormalInteger, Mu: 0 Sigma: 10, Default: 0", str(f2))
f2_large_q = NormalIntegerHyperparameter("param", 0, 10, q=2)
f2_large_q_ = NormalIntegerHyperparameter("param", 0, 10, q=2)
self.assertEqual(f2_large_q, f2_large_q_)
self.assertEqual(
"param, Type: NormalInteger, Mu: 0 Sigma: 10, Default: 0, Q: 2",
str(f2_large_q))
f3 = NormalIntegerHyperparameter("param", 0, 10, log=True)
f3_ = NormalIntegerHyperparameter("param", 0, 10, log=True)
self.assertEqual(f3, f3_)
self.assertEqual(
"param, Type: NormalInteger, Mu: 0 Sigma: 10, Default: 0, "
"on log-scale", str(f3))
f4 = NormalIntegerHyperparameter("param", 0, 10, default=1, log=True)
f4_ = NormalIntegerHyperparameter("param", 0, 10, default=1, log=True)
self.assertEqual(f4, f4_)
self.assertEqual(
"param, Type: NormalInteger, Mu: 0 Sigma: 10, Default: 1, "
"on log-scale", str(f4))
f5 = NormalIntegerHyperparameter("param", 0, 10, q=0.1, log=True)
f5_ = NormalIntegerHyperparameter("param", 0, 10, q=0.1, log=True)
self.assertEqual(f5, f5_)
self.assertEqual(
"param, Type: NormalInteger, Mu: 0 Sigma: 10, Default: 0, "
"on log-scale", str(f5))
self.assertNotEqual(f1, f2)
self.assertNotEqual(f1, "UniformFloat")
def test_normalint_is_legal(self):
f1 = NormalIntegerHyperparameter("param", 0, 10, q=0.1, log=True)
self.assertFalse(f1.is_legal(3.1))
self.assertFalse(f1.is_legal(3.0)) # 3.0 behaves like an Integer
self.assertFalse(f1.is_legal("BlaBlaBla"))
self.assertTrue(f1.is_legal(2))
self.assertTrue(f1.is_legal(-15))
# Test is legal vector
self.assertTrue(f1.is_legal_vector(1.0))
self.assertTrue(f1.is_legal_vector(0.0))
self.assertTrue(f1.is_legal_vector(0))
self.assertTrue(f1.is_legal_vector(0.3))
self.assertTrue(f1.is_legal_vector(-0.1))
self.assertTrue(f1.is_legal_vector(1.1))
self.assertRaises(TypeError, f1.is_legal_vector, "Hahaha")
def test_normalint_to_uniform(self):
f1 = NormalIntegerHyperparameter("param", 0, 10, q=0.1)
f1_expected = UniformIntegerHyperparameter("param", -30, 30)
f1_actual = f1.to_uniform()
self.assertEqual(f1_expected, f1_actual)
def test_normalint_is_legal(self):
with pytest.warns(RuntimeWarning, match='divide by zero encountered in log'):
f1 = NormalIntegerHyperparameter("param", 0, 10, q=0.1, log=True)
self.assertFalse(f1.is_legal(3.1))
self.assertFalse(f1.is_legal(3.0)) # 3.0 behaves like an Integer
self.assertFalse(f1.is_legal("BlaBlaBla"))
self.assertTrue(f1.is_legal(2))
self.assertTrue(f1.is_legal(-15))
# Test is legal vector
self.assertTrue(f1.is_legal_vector(1.0))
self.assertTrue(f1.is_legal_vector(0.0))
self.assertTrue(f1.is_legal_vector(0))
self.assertTrue(f1.is_legal_vector(0.3))
self.assertTrue(f1.is_legal_vector(-0.1))
self.assertTrue(f1.is_legal_vector(1.1))
self.assertRaises(TypeError, f1.is_legal_vector, "Hahaha")
def test_normalint_is_legal(self):
f1 = NormalIntegerHyperparameter("param", 0, 10, q=0.1, log=True)
self.assertFalse(f1.is_legal(3.1))
self.assertFalse(f1.is_legal(3.0)) # 3.0 behaves like an Integer
self.assertFalse(f1.is_legal("BlaBlaBla"))
self.assertTrue(f1.is_legal(2))
self.assertTrue(f1.is_legal(-15))
# Test is legal vector
self.assertTrue(f1.is_legal_vector(1.0))
self.assertTrue(f1.is_legal_vector(0.0))
self.assertTrue(f1.is_legal_vector(0))
self.assertTrue(f1.is_legal_vector(0.3))
self.assertTrue(f1.is_legal_vector(-0.1))
self.assertTrue(f1.is_legal_vector(1.1))
self.assertFalse(f1.is_legal_vector("Hahaha"))
def test_normalfloat_to_integer(self):
f1 = NormalFloatHyperparameter("param", 0, 10)
f2_expected = NormalIntegerHyperparameter("param", 0, 10)
f2_actual = f1.to_integer()
self.assertEqual(f2_expected, f2_actual)
def test_normalint(self):
# TODO test for unequal!
f1 = NormalIntegerHyperparameter("param", 0.5, 5.5)
f1_ = NormalIntegerHyperparameter("param", 0.5, 5.5)
self.assertEqual(f1, f1_)
self.assertEqual(
"param, Type: NormalInteger, Mu: 0.5 Sigma: 5.5, Default: 0.5",
str(f1))
# Test attributes are accessible
self.assertEqual(f1.name, "param")
self.assertEqual(f1.mu, 0.5)
self.assertEqual(f1.sigma, 5.5)
self.assertEqual(f1.q, None)
self.assertEqual(f1.log, False)
self.assertAlmostEqual(f1.default_value, 0.5)
self.assertAlmostEqual(f1.normalized_default_value, 0.5)
with pytest.warns(UserWarning, match="Setting quantization < 1 for Integer "
"Hyperparameter 'param' has no effect"):
f2 = NormalIntegerHyperparameter("param", 0, 10, q=0.1)
with pytest.warns(UserWarning, match="Setting quantization < 1 for Integer "
"Hyperparameter 'param' has no effect"):
f2_ = NormalIntegerHyperparameter("param", 0, 10, q=0.1)
self.assertEqual(f2, f2_)
self.assertEqual(
"param, Type: NormalInteger, Mu: 0 Sigma: 10, Default: 0",
str(f2))
f2_large_q = NormalIntegerHyperparameter("param", 0, 10, q=2)
f2_large_q_ = NormalIntegerHyperparameter("param", 0, 10, q=2)
self.assertEqual(f2_large_q, f2_large_q_)
self.assertEqual(
"param, Type: NormalInteger, Mu: 0 Sigma: 10, Default: 0, Q: 2",
str(f2_large_q))
with pytest.warns(RuntimeWarning, match='divide by zero encountered in log'):
f3 = NormalIntegerHyperparameter("param", 0, 10, log=True)
with pytest.warns(RuntimeWarning, match='divide by zero encountered in log'):
f3_ = NormalIntegerHyperparameter("param", 0, 10, log=True)
self.assertEqual(f3, f3_)
self.assertEqual(
"param, Type: NormalInteger, Mu: 0 Sigma: 10, Default: 0, "
"on log-scale", str(f3))
f4 = NormalIntegerHyperparameter("param", 0, 10, default_value=1, log=True)
f4_ = NormalIntegerHyperparameter("param", 0, 10, default_value=1, log=True)
self.assertEqual(f4, f4_)
self.assertEqual(
"param, Type: NormalInteger, Mu: 0 Sigma: 10, Default: 1, "
"on log-scale", str(f4))
with pytest.warns(RuntimeWarning, match='divide by zero encountered in log'):
f5 = NormalIntegerHyperparameter("param", 0, 10, q=0.1, log=True)
with pytest.warns(RuntimeWarning, match='divide by zero encountered in log'):
f5_ = NormalIntegerHyperparameter("param", 0, 10, q=0.1, log=True)
self.assertEqual(f5, f5_)
self.assertEqual(
"param, Type: NormalInteger, Mu: 0 Sigma: 10, Default: 0, "
"on log-scale", str(f5))
self.assertNotEqual(f1, f2)
self.assertNotEqual(f1, "UniformFloat")
# test that meta-data is stored correctly
f_meta = NormalIntegerHyperparameter("param", 0, 10, default_value=1, log=True,
meta=dict(self.meta_data))
self.assertEqual(f_meta.meta, self.meta_data)
