def get_complete_configspace():
"""Creates a configspace that includes all kinds of parameters with
complicated values. The idea is to provide a configspace that can be
used to check modules using ConfigSpace as a dependency to check
compatibility with e.g. Constants, log-scale, etc.
Returns
-------
cs: ConfigurationSpace
cs containing all kinds of parameters
"""
cs = ConfigurationSpace()
hp = {}
# Add Constants for all allowed types ('int', 'float', 'string')
hp['alpha'] = Constant("alpha", 0.0001)
hp['tol'] = Constant("tol", '0.0001')
hp['verbose'] = Constant("verbose", 1)
# Add numericals
# Add Floats
hp['beta1'] = UniformFloatHyperparameter("beta1", 0.85, 0.95, log=False)
hp['power_t'] = NormalFloatHyperparameter("power_t", mu=0.5, sigma=0.1, log=False)
# Add Ints
hp['momentum'] = UniformIntegerHyperparameter("momentum", 0, 100, False)
hp['beta2'] = NormalIntegerHyperparameter("beta2", mu=1, sigma=0.001, log=False)
# Add Floats (log)
hp['learning_rate_init'] = UniformFloatHyperparameter("learning_rate_init", 0.0001, 0.1, log=True)
hp['random1'] = NormalFloatHyperparameter("NormalFloat", mu=0, sigma=1, default_value=1, log=True)
# Add Ints (log)
hp['random2'] = UniformIntegerHyperparameter("UniformInt", 2, 100, log=True)
hp['random3'] = NormalIntegerHyperparameter("NormalInt", mu=0, sigma=1, default_value=1, log=True)
# Add Categorical for allowed types
hp['activation'] = CategoricalHyperparameter('activation', choices=['identity', 'logistic', 'tanh', 'relu'])
hp['solver'] = CategoricalHyperparameter('solver', choices=[-2, 0, 2]) # corrresponds to: ‘lbfgs’, ‘sgd’, ‘adam’
hp['batch_size_auto'] = CategoricalHyperparameter('batch_size_auto', choices=[True, False])
hp['learning_rate'] = CategoricalHyperparameter('learning_rate', choices=[-0.5, 0.0, 0.5]) # corresponds to {‘constant’, ‘invscaling’, ‘adaptive’}
# Add Ordinal
hp['batch_size'] = OrdinalHyperparameter('batch_size', sequence=[32, 64.0, '128'])
for k, v in hp.items():
cs.add_hyperparameter(v)
# learning_rate only with sgd
c = InCondition(hp['learning_rate'], hp['solver'], [0])
c = EqualsCondition(hp['momentum'], hp['solver'], 0)
# learning_rate_init only with sgd or adam
cs.add_condition(OrConjunction(EqualsCondition(hp['learning_rate'], hp['solver'], 0), # sgd
EqualsCondition(hp['learning_rate'], hp['solver'], 2))) # adam
# batch_size only with not batch_size_auto
cs.add_condition(NotEqualsCondition(hp['batch_size'], hp['batch_size_auto'], True))
# complicated way for solver == sgd
#cs.add_condition(AndConjunction(LessThanCondition(hp['power_t'], hp['solver'], 1),
# GreaterThanCondition(hp['power_t'], hp['solver'], -1)))
# betas with adam
cs.add_condition(EqualsCondition(hp['beta1'], hp['solver'], 2))
cs.add_condition(EqualsCondition(hp['beta2'], hp['solver'], 2))
return cs
def test_normalfloat_is_legal(self):
f1 = NormalFloatHyperparameter("param", 0, 10)
self.assertTrue(f1.is_legal(3.0))
self.assertTrue(f1.is_legal(2))
self.assertFalse(f1.is_legal("Hahaha"))
# Test legal vector values
self.assertTrue(f1.is_legal_vector(1.0))
self.assertTrue(f1.is_legal_vector(0.0))
self.assertTrue(f1.is_legal_vector(0))
self.assertTrue(f1.is_legal_vector(0.3))
self.assertTrue(f1.is_legal_vector(-0.1))
self.assertTrue(f1.is_legal_vector(1.1))
self.assertRaises(TypeError, f1.is_legal_vector, "Hahaha")
def test_normalfloat_is_legal(self):
f1 = NormalFloatHyperparameter("param", 0, 10)
self.assertTrue(f1.is_legal(3.0))
self.assertTrue(f1.is_legal(2))
self.assertFalse(f1.is_legal("Hahaha"))
# Test legal vector values
self.assertTrue(f1.is_legal_vector(1.0))
self.assertTrue(f1.is_legal_vector(0.0))
self.assertTrue(f1.is_legal_vector(0))
self.assertTrue(f1.is_legal_vector(0.3))
self.assertTrue(f1.is_legal_vector(-0.1))
self.assertTrue(f1.is_legal_vector(1.1))
self.assertFalse(f1.is_legal_vector("Hahaha"))
def test_normalfloat(self):
# TODO test non-equality
f1 = NormalFloatHyperparameter("param", 0.5, 10.5)
f1_ = NormalFloatHyperparameter("param", 0.5, 10.5)
self.assertEqual(f1, f1_)
self.assertEqual(
"param, Type: NormalFloat, Mu: 0.5 Sigma: 10.5, Default: 0.5",
str(f1))
f2 = NormalFloatHyperparameter("param", 0, 10, q=0.1)
f2_ = NormalFloatHyperparameter("param", 0, 10, q=0.1)
self.assertEqual(f2, f2_)
self.assertEqual(
"param, Type: NormalFloat, Mu: 0.0 Sigma: 10.0, Default: 0.0, "
"Q: 0.1", str(f2))
f3 = NormalFloatHyperparameter("param", 0, 10, log=True)
f3_ = NormalFloatHyperparameter("param", 0, 10, log=True)
self.assertEqual(f3, f3_)
self.assertEqual(
"param, Type: NormalFloat, Mu: 0.0 Sigma: 10.0, Default: 0.0, "
"on log-scale", str(f3))
f4 = NormalFloatHyperparameter("param", 0, 10, default=1.0)
f4_ = NormalFloatHyperparameter("param", 0, 10, default=1.0)
self.assertEqual(f4, f4_)
self.assertEqual(
"param, Type: NormalFloat, Mu: 0.0 Sigma: 10.0, Default: 1.0",
str(f4))
f5 = NormalFloatHyperparameter("param",
0,
10,
default=1.0,
q=0.1,
log=True)
f5_ = NormalFloatHyperparameter("param",
0,
10,
default=1.0,
q=0.1,
log=True)
self.assertEqual(f5, f5_)
self.assertEqual(
"param, Type: NormalFloat, Mu: 0.0 Sigma: 10.0, Default: 1.0, "
"on log-scale, Q: 0.1", str(f5))
self.assertNotEqual(f1, f2)
self.assertNotEqual(f1, "UniformFloat")
def _construct_hyperparameter(hyperparameter: Dict) -> Hyperparameter:
hp_type = hyperparameter['type']
name = hyperparameter['name']
if hp_type == 'constant':
return Constant(
name=name,
value=hyperparameter['value'],
)
elif hp_type == 'unparametrized':
return UnParametrizedHyperparameter(
name=name,
value=hyperparameter['value'],
)
elif hp_type == 'uniform_float':
return UniformFloatHyperparameter(
name=name,
log=hyperparameter['log'],
lower=hyperparameter['lower'],
upper=hyperparameter['upper'],
default_value=hyperparameter['default'],
)
elif hp_type == 'normal_float':
return NormalFloatHyperparameter(
name=name,
log=hyperparameter['log'],
mu=hyperparameter['mu'],
sigma=hyperparameter['sigma'],
default_value=hyperparameter['default'],
)
elif hp_type == 'uniform_int':
return UniformIntegerHyperparameter(
name=name,
log=hyperparameter['log'],
lower=hyperparameter['lower'],
upper=hyperparameter['upper'],
default_value=hyperparameter['default'],
)
elif hp_type == 'normal_int':
return NormalIntegerHyperparameter(
name=name,
log=hyperparameter['log'],
lower=hyperparameter['lower'],
upper=hyperparameter['upper'],
default_value=hyperparameter['default'],
)
elif hp_type == 'categorical':
return CategoricalHyperparameter(
name=name,
choices=hyperparameter['choices'],
default_value=hyperparameter['default'],
weights=hyperparameter.get('probabilities'),
)
elif hp_type == 'ordinal':
return OrdinalHyperparameter(
name=name,
sequence=hyperparameter['sequence'],
default_value=hyperparameter['default'],
)
else:
raise ValueError(hp_type)
def test_hyperparam_representation(self):
# Float
f1 = UniformFloatHyperparameter("param", 1, 100, log=True)
self.assertEqual(
"param, Type: UniformFloat, Range: [1.0, 100.0], Default: 10.0, on log-scale",
repr(f1)
)
f2 = NormalFloatHyperparameter("param", 8, 99.1, log=False)
self.assertEqual(
"param, Type: NormalFloat, Mu: 8.0 Sigma: 99.1, Default: 8.0",
repr(f2)
)
i1 = UniformIntegerHyperparameter("param", 0, 100)
self.assertEqual(
"param, Type: UniformInteger, Range: [0, 100], Default: 50",
repr(i1)
)
i2 = NormalIntegerHyperparameter("param", 5, 8)
self.assertEqual(
"param, Type: NormalInteger, Mu: 5 Sigma: 8, Default: 5",
repr(i2)
)
o1 = OrdinalHyperparameter("temp", ["freezing", "cold", "warm", "hot"])
self.assertEqual(
"temp, Type: Ordinal, Sequence: {freezing, cold, warm, hot}, Default: freezing",
repr(o1)
)
c1 = CategoricalHyperparameter("param", [True, False])
self.assertEqual(
"param, Type: Categorical, Choices: {True, False}, Default: True",
repr(c1)
)
def test_uniformfloat_transform(self):
"""This checks whether a value sampled through the configuration
space (it does not happend when the variable is sampled alone) stays
equal when it is serialized via JSON and the deserialized again."""
cs = ConfigurationSpace()
a = cs.add_hyperparameter(UniformFloatHyperparameter('a', -5, 10))
b = cs.add_hyperparameter(
NormalFloatHyperparameter('b', 1, 2, log=True))
for i in range(100):
config = cs.sample_configuration()
value = OrderedDict(sorted(config.get_dictionary().items()))
string = json.dumps(value)
saved_value = json.loads(string)
saved_value = OrderedDict(sorted(byteify(saved_value).items()))
self.assertEqual(repr(value), repr(saved_value))
# Next, test whether the truncation also works when initializing the
# Configuration with a dictionary
for i in range(100):
rs = np.random.RandomState(1)
value_a = a.sample(rs)
value_b = b.sample(rs)
values_dict = {'a': value_a, 'b': value_b}
config = Configuration(cs, values=values_dict)
string = json.dumps(config.get_dictionary())
saved_value = json.loads(string)
saved_value = byteify(saved_value)
self.assertEqual(values_dict, saved_value)
def test_meta_field(self):
cs = ConfigurationSpace()
cs.add_hyperparameter(
UniformIntegerHyperparameter("uihp",
lower=1,
upper=10,
meta=dict(uihp=True)))
cs.add_hyperparameter(
NormalIntegerHyperparameter("nihp",
mu=0,
sigma=1,
meta=dict(nihp=True)))
cs.add_hyperparameter(
UniformFloatHyperparameter("ufhp",
lower=1,
upper=10,
meta=dict(ufhp=True)))
cs.add_hyperparameter(
NormalFloatHyperparameter("nfhp",
mu=0,
sigma=1,
meta=dict(nfhp=True)))
cs.add_hyperparameter(
CategoricalHyperparameter("chp",
choices=['1', '2', '3'],
meta=dict(chp=True)))
cs.add_hyperparameter(
OrdinalHyperparameter("ohp",
sequence=['1', '2', '3'],
meta=dict(ohp=True)))
cs.add_hyperparameter(Constant("const", value=1,
meta=dict(const=True)))
parent = ConfigurationSpace()
parent.add_configuration_space("sub", cs, delimiter=':')
self.assertEqual(
parent.get_hyperparameter("sub:uihp").meta, dict(uihp=True))
self.assertEqual(
parent.get_hyperparameter("sub:nihp").meta, dict(nihp=True))
self.assertEqual(
parent.get_hyperparameter("sub:ufhp").meta, dict(ufhp=True))
self.assertEqual(
parent.get_hyperparameter("sub:nfhp").meta, dict(nfhp=True))
self.assertEqual(
parent.get_hyperparameter("sub:chp").meta, dict(chp=True))
self.assertEqual(
parent.get_hyperparameter("sub:ohp").meta, dict(ohp=True))
self.assertEqual(
parent.get_hyperparameter("sub:const").meta, dict(const=True))
def test_sample_NormalFloatHyperparameter(self):
hp = NormalFloatHyperparameter("nfhp", 0, 1)
def actual_test():
rs = np.random.RandomState(1)
counts_per_bin = [0 for i in range(11)]
for i in range(100000):
value = hp.sample(rs)
index = min(max(int((round(value + 0.5)) + 5), 0), 9)
counts_per_bin[index] += 1
self.assertEqual([0, 4, 138, 2113, 13394, 34104, 34282, 13683,
2136, 146, 0], counts_per_bin)
return counts_per_bin
self.assertEqual(actual_test(), actual_test())
def test_repr_roundtrip(self):
cs = ConfigurationSpace()
cs.add_hyperparameter(
UniformIntegerHyperparameter("uihp", lower=1, upper=10))
cs.add_hyperparameter(
NormalIntegerHyperparameter("nihp", mu=0, sigma=1))
cs.add_hyperparameter(
UniformFloatHyperparameter("ufhp", lower=1, upper=10))
cs.add_hyperparameter(NormalFloatHyperparameter("nfhp", mu=0, sigma=1))
cs.add_hyperparameter(
CategoricalHyperparameter("chp", choices=['1', '2', '3']))
cs.add_hyperparameter(
OrdinalHyperparameter("ohp", sequence=['1', '2', '3']))
cs.add_hyperparameter(Constant("const", value=1))
default = cs.get_default_configuration()
repr = default.__repr__()
repr = repr.replace('})', '}, configuration_space=cs)')
config = eval(repr)
self.assertEqual(default, config)
def test_normalfloat_to_uniformfloat(self):
f1 = NormalFloatHyperparameter("param", 0, 10, q=0.1)
f1_expected = UniformFloatHyperparameter("param", -30, 30, q=0.1)
f1_actual = f1.to_uniform()
self.assertEqual(f1_expected, f1_actual)
def test_normalfloat_to_integer(self):
f1 = NormalFloatHyperparameter("param", 0, 10)
f2_expected = NormalIntegerHyperparameter("param", 0, 10)
f2_actual = f1.to_integer()
self.assertEqual(f2_expected, f2_actual)
def test_normalfloat_is_legal(self):
f1 = NormalFloatHyperparameter("param", 0, 10)
self.assertTrue(f1.is_legal(3.0))
self.assertTrue(f1.is_legal(2))
self.assertFalse(f1.is_legal("Hahaha"))
def test_normalfloat_to_uniformfloat(self):
f1 = NormalFloatHyperparameter("param", 0, 10, q=0.1)
f1_expected = UniformFloatHyperparameter("param", -30, 30, q=0.1)
f1_actual = f1.to_uniform()
self.assertEqual(f1_expected, f1_actual)
def test_normalfloat(self):
# TODO test non-equality
f1 = NormalFloatHyperparameter("param", 0.5, 10.5)
f1_ = NormalFloatHyperparameter("param", 0.5, 10.5)
self.assertEqual(f1, f1_)
self.assertEqual(
"param, Type: NormalFloat, Mu: 0.5 Sigma: 10.5, Default: 0.5",
str(f1))
# Test attributes are accessible
self.assertEqual(f1.name, "param")
self.assertAlmostEqual(f1.mu, 0.5)
self.assertAlmostEqual(f1.sigma, 10.5)
self.assertAlmostEqual(f1.q, None)
self.assertEqual(f1.log, False)
self.assertAlmostEqual(f1.default_value, 0.5)
self.assertAlmostEqual(f1.normalized_default_value, 0.5)
f2 = NormalFloatHyperparameter("param", 0, 10, q=0.1)
f2_ = NormalFloatHyperparameter("param", 0, 10, q=0.1)
self.assertEqual(f2, f2_)
self.assertEqual(
"param, Type: NormalFloat, Mu: 0.0 Sigma: 10.0, Default: 0.0, "
"Q: 0.1", str(f2))
with pytest.warns(RuntimeWarning, match='divide by zero encountered in log'):
f3 = NormalFloatHyperparameter("param", 0, 10, log=True)
with pytest.warns(RuntimeWarning, match='divide by zero encountered in log'):
f3_ = NormalFloatHyperparameter("param", 0, 10, log=True)
self.assertEqual(f3, f3_)
self.assertEqual(
"param, Type: NormalFloat, Mu: 0.0 Sigma: 10.0, Default: 0.0, "
"on log-scale", str(f3))
f4 = NormalFloatHyperparameter("param", 0, 10, default_value=1.0)
f4_ = NormalFloatHyperparameter("param", 0, 10, default_value=1.0)
self.assertEqual(f4, f4_)
self.assertEqual(
"param, Type: NormalFloat, Mu: 0.0 Sigma: 10.0, Default: 1.0",
str(f4))
f5 = NormalFloatHyperparameter("param", 0, 10, default_value=1.0,
q=0.1, log=True)
f5_ = NormalFloatHyperparameter("param", 0, 10, default_value=1.0,
q=0.1, log=True)
self.assertEqual(f5, f5_)
self.assertEqual(
"param, Type: NormalFloat, Mu: 0.0 Sigma: 10.0, Default: 1.0, "
"on log-scale, Q: 0.1", str(f5))
self.assertNotEqual(f1, f2)
self.assertNotEqual(f1, "UniformFloat")
# test that meta-data is stored correctly
f_meta = NormalFloatHyperparameter("param", 0.1, 10, q=0.1, log=True,
default_value=1.0, meta=dict(self.meta_data))
self.assertEqual(f_meta.meta, self.meta_data)
def test_normalfloat_to_integer(self):
f1 = NormalFloatHyperparameter("param", 0, 10)
f2_expected = NormalIntegerHyperparameter("param", 0, 10)
f2_actual = f1.to_integer()
self.assertEqual(f2_expected, f2_actual)
def test_normalfloat(self):
# TODO test non-equality
f1 = NormalFloatHyperparameter("param", 0.5, 10.5)
f1_ = NormalFloatHyperparameter("param", 0.5, 10.5)
self.assertEqual(f1, f1_)
self.assertEqual(
"param, Type: NormalFloat, Mu: 0.5 Sigma: 10.5, Default: 0.5",
str(f1))
f2 = NormalFloatHyperparameter("param", 0, 10, q=0.1)
f2_ = NormalFloatHyperparameter("param", 0, 10, q=0.1)
self.assertEqual(f2, f2_)
self.assertEqual(
"param, Type: NormalFloat, Mu: 0.0 Sigma: 10.0, Default: 0.0, "
"Q: 0.1", str(f2))
f3 = NormalFloatHyperparameter("param", 0, 10, log=True)
f3_ = NormalFloatHyperparameter("param", 0, 10, log=True)
self.assertEqual(f3, f3_)
self.assertEqual(
"param, Type: NormalFloat, Mu: 0.0 Sigma: 10.0, Default: 0.0, "
"on log-scale", str(f3))
f4 = NormalFloatHyperparameter("param", 0, 10, default_value=1.0)
f4_ = NormalFloatHyperparameter("param", 0, 10, default_value=1.0)
self.assertEqual(f4, f4_)
self.assertEqual(
"param, Type: NormalFloat, Mu: 0.0 Sigma: 10.0, Default: 1.0",
str(f4))
f5 = NormalFloatHyperparameter("param",
0,
10,
default_value=1.0,
q=0.1,
log=True)
f5_ = NormalFloatHyperparameter("param",
0,
10,
default_value=1.0,
q=0.1,
log=True)
self.assertEqual(f5, f5_)
self.assertEqual(
"param, Type: NormalFloat, Mu: 0.0 Sigma: 10.0, Default: 1.0, "
"on log-scale, Q: 0.1", str(f5))
self.assertNotEqual(f1, f2)
self.assertNotEqual(f1, "UniformFloat")
# test that meta-data is stored correctly
f_meta = NormalFloatHyperparameter("param",
0.1,
10,
q=0.1,
log=True,
default_value=1.0,
meta=dict(self.meta_data))
self.assertEqual(f_meta.meta, self.meta_data)
def cast_hyperparameter(hyperparameter, name):
# From D3M hyperparameters to ConfigSpace hyperparameters
# TODO: 'Choice' and 'Set' (D3M hyperparameters)
new_hyperparameter = None
try:
if isinstance(hyperparameter, Bounded):
lower = hyperparameter.lower
upper = hyperparameter.upper
default = hyperparameter.get_default()
if lower is None:
lower = default
if upper is None:
upper = default * 2 if default > 0 else 10
if hyperparameter.structural_type == int:
if not hyperparameter.lower_inclusive:
lower += 1
if not hyperparameter.upper_inclusive:
upper -= 1
new_hyperparameter = UniformIntegerHyperparameter(
name, lower, upper, default_value=default)
else:
if not hyperparameter.lower_inclusive:
lower += 1e-20
if not hyperparameter.upper_inclusive:
upper -= 1e-20
new_hyperparameter = UniformFloatHyperparameter(
name, lower, upper, default_value=default)
elif isinstance(hyperparameter, UniformBool):
default = hyperparameter.get_default()
new_hyperparameter = CategoricalHyperparameter(
name, [True, False], default_value=default)
elif isinstance(hyperparameter, UniformInt):
lower = hyperparameter.lower
upper = hyperparameter.upper
default = hyperparameter.get_default()
new_hyperparameter = UniformIntegerHyperparameter(
name, lower, upper, default_value=default)
elif isinstance(hyperparameter, Uniform):
lower = hyperparameter.lower
upper = hyperparameter.upper
default = hyperparameter.get_default()
new_hyperparameter = UniformFloatHyperparameter(
name, lower, upper, default_value=default)
elif isinstance(hyperparameter, Normal):
default = hyperparameter.get_default()
new_hyperparameter = NormalFloatHyperparameter(
name, default_value=default)
elif isinstance(hyperparameter, Enumeration):
values = hyperparameter.values
default = hyperparameter.get_default()
new_hyperparameter = CategoricalHyperparameter(
name, values, default_value=default)
elif isinstance(hyperparameter, ConstantD3M):
default = 'None' if hyperparameter.get_default(
) is None else hyperparameter.get_default()
new_hyperparameter = Constant(name, default)
except Exception as e:
logger.error(e)
return new_hyperparameter
