def get_complete_configspace(): """Creates a configspace that includes all kinds of parameters with complicated values. The idea is to provide a configspace that can be used to check modules using ConfigSpace as a dependency to check compatibility with e.g. Constants, log-scale, etc. Returns ------- cs: ConfigurationSpace cs containing all kinds of parameters """ cs = ConfigurationSpace() hp = {} # Add Constants for all allowed types ('int', 'float', 'string') hp['alpha'] = Constant("alpha", 0.0001) hp['tol'] = Constant("tol", '0.0001') hp['verbose'] = Constant("verbose", 1) # Add numericals # Add Floats hp['beta1'] = UniformFloatHyperparameter("beta1", 0.85, 0.95, log=False) hp['power_t'] = NormalFloatHyperparameter("power_t", mu=0.5, sigma=0.1, log=False) # Add Ints hp['momentum'] = UniformIntegerHyperparameter("momentum", 0, 100, False) hp['beta2'] = NormalIntegerHyperparameter("beta2", mu=1, sigma=0.001, log=False) # Add Floats (log) hp['learning_rate_init'] = UniformFloatHyperparameter("learning_rate_init", 0.0001, 0.1, log=True) hp['random1'] = NormalFloatHyperparameter("NormalFloat", mu=0, sigma=1, default_value=1, log=True) # Add Ints (log) hp['random2'] = UniformIntegerHyperparameter("UniformInt", 2, 100, log=True) hp['random3'] = NormalIntegerHyperparameter("NormalInt", mu=0, sigma=1, default_value=1, log=True) # Add Categorical for allowed types hp['activation'] = CategoricalHyperparameter('activation', choices=['identity', 'logistic', 'tanh', 'relu']) hp['solver'] = CategoricalHyperparameter('solver', choices=[-2, 0, 2]) # corrresponds to: ‘lbfgs’, ‘sgd’, ‘adam’ hp['batch_size_auto'] = CategoricalHyperparameter('batch_size_auto', choices=[True, False]) hp['learning_rate'] = CategoricalHyperparameter('learning_rate', choices=[-0.5, 0.0, 0.5]) # corresponds to {‘constant’, ‘invscaling’, ‘adaptive’} # Add Ordinal hp['batch_size'] = OrdinalHyperparameter('batch_size', sequence=[32, 64.0, '128']) for k, v in hp.items(): cs.add_hyperparameter(v) # learning_rate only with sgd c = InCondition(hp['learning_rate'], hp['solver'], [0]) c = EqualsCondition(hp['momentum'], hp['solver'], 0) # learning_rate_init only with sgd or adam cs.add_condition(OrConjunction(EqualsCondition(hp['learning_rate'], hp['solver'], 0), # sgd EqualsCondition(hp['learning_rate'], hp['solver'], 2))) # adam # batch_size only with not batch_size_auto cs.add_condition(NotEqualsCondition(hp['batch_size'], hp['batch_size_auto'], True)) # complicated way for solver == sgd #cs.add_condition(AndConjunction(LessThanCondition(hp['power_t'], hp['solver'], 1), # GreaterThanCondition(hp['power_t'], hp['solver'], -1))) # betas with adam cs.add_condition(EqualsCondition(hp['beta1'], hp['solver'], 2)) cs.add_condition(EqualsCondition(hp['beta2'], hp['solver'], 2)) return cs
def test_sample_NormalIntegerHyperparameter(self): def sample(hp): lower = -30 upper = 30 rs = np.random.RandomState(1) counts_per_bin = [0 for i in range(21)] for i in range(100000): value = hp.sample(rs) sample = float(value) if sample < lower: sample = lower if sample > upper: sample = upper index = int((sample - lower) / (upper - lower) * 20) counts_per_bin[index] += 1 self.assertIsInstance(value, int) return counts_per_bin hp = NormalIntegerHyperparameter("nihp", 0, 10) self.assertEqual(sample(hp), [305, 422, 835, 1596, 2682, 4531, 6572, 8670, 10649, 11510, 11854, 11223, 9309, 7244, 5155, 3406, 2025, 1079, 514, 249, 170]) self.assertEqual(sample(hp), sample(hp))
def test_normalint_to_uniform(self): with pytest.warns(UserWarning, match="Setting quantization < 1 for Integer " "Hyperparameter 'param' has no effect"): f1 = NormalIntegerHyperparameter("param", 0, 10, q=0.1) f1_expected = UniformIntegerHyperparameter("param", -30, 30) f1_actual = f1.to_uniform() self.assertEqual(f1_expected, f1_actual)
def test_normalint_is_legal(self): f1 = NormalIntegerHyperparameter("param", 0, 10, q=0.1, log=True) self.assertFalse(f1.is_legal(3.1)) self.assertFalse(f1.is_legal(3.0)) # 3.0 behaves like an Integer self.assertFalse(f1.is_legal("BlaBlaBla")) self.assertTrue(f1.is_legal(2)) self.assertTrue(f1.is_legal(-15))
def _construct_hyperparameter(hyperparameter: Dict) -> Hyperparameter: hp_type = hyperparameter['type'] name = hyperparameter['name'] if hp_type == 'constant': return Constant( name=name, value=hyperparameter['value'], ) elif hp_type == 'unparametrized': return UnParametrizedHyperparameter( name=name, value=hyperparameter['value'], ) elif hp_type == 'uniform_float': return UniformFloatHyperparameter( name=name, log=hyperparameter['log'], lower=hyperparameter['lower'], upper=hyperparameter['upper'], default_value=hyperparameter['default'], ) elif hp_type == 'normal_float': return NormalFloatHyperparameter( name=name, log=hyperparameter['log'], mu=hyperparameter['mu'], sigma=hyperparameter['sigma'], default_value=hyperparameter['default'], ) elif hp_type == 'uniform_int': return UniformIntegerHyperparameter( name=name, log=hyperparameter['log'], lower=hyperparameter['lower'], upper=hyperparameter['upper'], default_value=hyperparameter['default'], ) elif hp_type == 'normal_int': return NormalIntegerHyperparameter( name=name, log=hyperparameter['log'], lower=hyperparameter['lower'], upper=hyperparameter['upper'], default_value=hyperparameter['default'], ) elif hp_type == 'categorical': return CategoricalHyperparameter( name=name, choices=hyperparameter['choices'], default_value=hyperparameter['default'], weights=hyperparameter.get('probabilities'), ) elif hp_type == 'ordinal': return OrdinalHyperparameter( name=name, sequence=hyperparameter['sequence'], default_value=hyperparameter['default'], ) else: raise ValueError(hp_type)
def test_hyperparam_representation(self): # Float f1 = UniformFloatHyperparameter("param", 1, 100, log=True) self.assertEqual( "param, Type: UniformFloat, Range: [1.0, 100.0], Default: 10.0, on log-scale", repr(f1) ) f2 = NormalFloatHyperparameter("param", 8, 99.1, log=False) self.assertEqual( "param, Type: NormalFloat, Mu: 8.0 Sigma: 99.1, Default: 8.0", repr(f2) ) i1 = UniformIntegerHyperparameter("param", 0, 100) self.assertEqual( "param, Type: UniformInteger, Range: [0, 100], Default: 50", repr(i1) ) i2 = NormalIntegerHyperparameter("param", 5, 8) self.assertEqual( "param, Type: NormalInteger, Mu: 5 Sigma: 8, Default: 5", repr(i2) ) o1 = OrdinalHyperparameter("temp", ["freezing", "cold", "warm", "hot"]) self.assertEqual( "temp, Type: Ordinal, Sequence: {freezing, cold, warm, hot}, Default: freezing", repr(o1) ) c1 = CategoricalHyperparameter("param", [True, False]) self.assertEqual( "param, Type: Categorical, Choices: {True, False}, Default: True", repr(c1) )
def test__sample_NormalIntegerHyperparameter(self): # mean zero, std 1 hp = NormalIntegerHyperparameter("uihp", 0, 1) values = [] rs = np.random.RandomState(1) for i in range(100): values.append(hp._sample(rs)) self.assertEqual(len(np.unique(values)), 5)
def test_normalint_legal_float_values(self): n_iter = NormalIntegerHyperparameter("n_iter", 0, 1., default_value=2.0) self.assertIsInstance(n_iter.default_value, int) self.assertRaisesRegex(ValueError, r"For the Integer parameter n_iter, " r"the value must be an Integer, too." r" Right now it is a " r"<(type|class) 'float'>" r" with value 0.5.", UniformIntegerHyperparameter, "n_iter", 0, 1., default_value=0.5)
def test_meta_field(self): cs = ConfigurationSpace() cs.add_hyperparameter( UniformIntegerHyperparameter("uihp", lower=1, upper=10, meta=dict(uihp=True))) cs.add_hyperparameter( NormalIntegerHyperparameter("nihp", mu=0, sigma=1, meta=dict(nihp=True))) cs.add_hyperparameter( UniformFloatHyperparameter("ufhp", lower=1, upper=10, meta=dict(ufhp=True))) cs.add_hyperparameter( NormalFloatHyperparameter("nfhp", mu=0, sigma=1, meta=dict(nfhp=True))) cs.add_hyperparameter( CategoricalHyperparameter("chp", choices=['1', '2', '3'], meta=dict(chp=True))) cs.add_hyperparameter( OrdinalHyperparameter("ohp", sequence=['1', '2', '3'], meta=dict(ohp=True))) cs.add_hyperparameter(Constant("const", value=1, meta=dict(const=True))) parent = ConfigurationSpace() parent.add_configuration_space("sub", cs, delimiter=':') self.assertEqual( parent.get_hyperparameter("sub:uihp").meta, dict(uihp=True)) self.assertEqual( parent.get_hyperparameter("sub:nihp").meta, dict(nihp=True)) self.assertEqual( parent.get_hyperparameter("sub:ufhp").meta, dict(ufhp=True)) self.assertEqual( parent.get_hyperparameter("sub:nfhp").meta, dict(nfhp=True)) self.assertEqual( parent.get_hyperparameter("sub:chp").meta, dict(chp=True)) self.assertEqual( parent.get_hyperparameter("sub:ohp").meta, dict(ohp=True)) self.assertEqual( parent.get_hyperparameter("sub:const").meta, dict(const=True))
def test_normalint_is_legal(self): f1 = NormalIntegerHyperparameter("param", 0, 10, q=0.1, log=True) self.assertFalse(f1.is_legal(3.1)) self.assertFalse(f1.is_legal(3.0)) # 3.0 behaves like an Integer self.assertFalse(f1.is_legal("BlaBlaBla")) self.assertTrue(f1.is_legal(2)) self.assertTrue(f1.is_legal(-15)) # Test is legal vector self.assertTrue(f1.is_legal_vector(1.0)) self.assertTrue(f1.is_legal_vector(0.0)) self.assertTrue(f1.is_legal_vector(0)) self.assertTrue(f1.is_legal_vector(0.3)) self.assertTrue(f1.is_legal_vector(-0.1)) self.assertTrue(f1.is_legal_vector(1.1)) self.assertRaises(TypeError, f1.is_legal_vector, "Hahaha")
def test_normalint_is_legal(self): with pytest.warns(RuntimeWarning, match='divide by zero encountered in log'): f1 = NormalIntegerHyperparameter("param", 0, 10, q=0.1, log=True) self.assertFalse(f1.is_legal(3.1)) self.assertFalse(f1.is_legal(3.0)) # 3.0 behaves like an Integer self.assertFalse(f1.is_legal("BlaBlaBla")) self.assertTrue(f1.is_legal(2)) self.assertTrue(f1.is_legal(-15)) # Test is legal vector self.assertTrue(f1.is_legal_vector(1.0)) self.assertTrue(f1.is_legal_vector(0.0)) self.assertTrue(f1.is_legal_vector(0)) self.assertTrue(f1.is_legal_vector(0.3)) self.assertTrue(f1.is_legal_vector(-0.1)) self.assertTrue(f1.is_legal_vector(1.1)) self.assertRaises(TypeError, f1.is_legal_vector, "Hahaha")
def test_repr_roundtrip(self): cs = ConfigurationSpace() cs.add_hyperparameter( UniformIntegerHyperparameter("uihp", lower=1, upper=10)) cs.add_hyperparameter( NormalIntegerHyperparameter("nihp", mu=0, sigma=1)) cs.add_hyperparameter( UniformFloatHyperparameter("ufhp", lower=1, upper=10)) cs.add_hyperparameter(NormalFloatHyperparameter("nfhp", mu=0, sigma=1)) cs.add_hyperparameter( CategoricalHyperparameter("chp", choices=['1', '2', '3'])) cs.add_hyperparameter( OrdinalHyperparameter("ohp", sequence=['1', '2', '3'])) cs.add_hyperparameter(Constant("const", value=1)) default = cs.get_default_configuration() repr = default.__repr__() repr = repr.replace('})', '}, configuration_space=cs)') config = eval(repr) self.assertEqual(default, config)
def test_normalint(self): # TODO test for unequal! f1 = NormalIntegerHyperparameter("param", 0.5, 5.5) f1_ = NormalIntegerHyperparameter("param", 0.5, 5.5) self.assertEqual(f1, f1_) self.assertEqual( "param, Type: NormalInteger, Mu: 0.5 Sigma: 5.5, Default: 0.5", str(f1)) # Test attributes are accessible self.assertEqual(f1.name, "param") self.assertEqual(f1.mu, 0.5) self.assertEqual(f1.sigma, 5.5) self.assertEqual(f1.q, None) self.assertEqual(f1.log, False) self.assertAlmostEqual(f1.default_value, 0.5) self.assertAlmostEqual(f1.normalized_default_value, 0.5) with pytest.warns(UserWarning, match="Setting quantization < 1 for Integer " "Hyperparameter 'param' has no effect"): f2 = NormalIntegerHyperparameter("param", 0, 10, q=0.1) with pytest.warns(UserWarning, match="Setting quantization < 1 for Integer " "Hyperparameter 'param' has no effect"): f2_ = NormalIntegerHyperparameter("param", 0, 10, q=0.1) self.assertEqual(f2, f2_) self.assertEqual( "param, Type: NormalInteger, Mu: 0 Sigma: 10, Default: 0", str(f2)) f2_large_q = NormalIntegerHyperparameter("param", 0, 10, q=2) f2_large_q_ = NormalIntegerHyperparameter("param", 0, 10, q=2) self.assertEqual(f2_large_q, f2_large_q_) self.assertEqual( "param, Type: NormalInteger, Mu: 0 Sigma: 10, Default: 0, Q: 2", str(f2_large_q)) with pytest.warns(RuntimeWarning, match='divide by zero encountered in log'): f3 = NormalIntegerHyperparameter("param", 0, 10, log=True) with pytest.warns(RuntimeWarning, match='divide by zero encountered in log'): f3_ = NormalIntegerHyperparameter("param", 0, 10, log=True) self.assertEqual(f3, f3_) self.assertEqual( "param, Type: NormalInteger, Mu: 0 Sigma: 10, Default: 0, " "on log-scale", str(f3)) f4 = NormalIntegerHyperparameter("param", 0, 10, default_value=1, log=True) f4_ = NormalIntegerHyperparameter("param", 0, 10, default_value=1, log=True) self.assertEqual(f4, f4_) self.assertEqual( "param, Type: NormalInteger, Mu: 0 Sigma: 10, Default: 1, " "on log-scale", str(f4)) with pytest.warns(RuntimeWarning, match='divide by zero encountered in log'): f5 = NormalIntegerHyperparameter("param", 0, 10, q=0.1, log=True) with pytest.warns(RuntimeWarning, match='divide by zero encountered in log'): f5_ = NormalIntegerHyperparameter("param", 0, 10, q=0.1, log=True) self.assertEqual(f5, f5_) self.assertEqual( "param, Type: NormalInteger, Mu: 0 Sigma: 10, Default: 0, " "on log-scale", str(f5)) self.assertNotEqual(f1, f2) self.assertNotEqual(f1, "UniformFloat") # test that meta-data is stored correctly f_meta = NormalIntegerHyperparameter("param", 0, 10, default_value=1, log=True, meta=dict(self.meta_data)) self.assertEqual(f_meta.meta, self.meta_data)
def test_normalfloat_to_integer(self): f1 = NormalFloatHyperparameter("param", 0, 10) f2_expected = NormalIntegerHyperparameter("param", 0, 10) f2_actual = f1.to_integer() self.assertEqual(f2_expected, f2_actual)
def test_normalint_to_uniform(self): f1 = NormalIntegerHyperparameter("param", 0, 10, q=0.1) f1_expected = UniformIntegerHyperparameter("param", -30, 30) f1_actual = f1.to_uniform() self.assertEqual(f1_expected, f1_actual)
def test_normalint(self): # TODO test for unequal! f1 = NormalIntegerHyperparameter("param", 0.5, 5.5) f1_ = NormalIntegerHyperparameter("param", 0.5, 5.5) self.assertEqual(f1, f1_) self.assertEqual( "param, Type: NormalInteger, Mu: 0.5 Sigma: 5.5, Default: 0.5", str(f1)) f2 = NormalIntegerHyperparameter("param", 0, 10, q=0.1) f2_ = NormalIntegerHyperparameter("param", 0, 10, q=0.1) self.assertEqual(f2, f2_) self.assertEqual( "param, Type: NormalInteger, Mu: 0 Sigma: 10, Default: 0", str(f2)) f2_large_q = NormalIntegerHyperparameter("param", 0, 10, q=2) f2_large_q_ = NormalIntegerHyperparameter("param", 0, 10, q=2) self.assertEqual(f2_large_q, f2_large_q_) self.assertEqual( "param, Type: NormalInteger, Mu: 0 Sigma: 10, Default: 0, Q: 2", str(f2_large_q)) f3 = NormalIntegerHyperparameter("param", 0, 10, log=True) f3_ = NormalIntegerHyperparameter("param", 0, 10, log=True) self.assertEqual(f3, f3_) self.assertEqual( "param, Type: NormalInteger, Mu: 0 Sigma: 10, Default: 0, " "on log-scale", str(f3)) f4 = NormalIntegerHyperparameter("param", 0, 10, default=1, log=True) f4_ = NormalIntegerHyperparameter("param", 0, 10, default=1, log=True) self.assertEqual(f4, f4_) self.assertEqual( "param, Type: NormalInteger, Mu: 0 Sigma: 10, Default: 1, " "on log-scale", str(f4)) f5 = NormalIntegerHyperparameter("param", 0, 10, q=0.1, log=True) f5_ = NormalIntegerHyperparameter("param", 0, 10, q=0.1, log=True) self.assertEqual(f5, f5_) self.assertEqual( "param, Type: NormalInteger, Mu: 0 Sigma: 10, Default: 0, " "on log-scale", str(f5)) self.assertNotEqual(f1, f2) self.assertNotEqual(f1, "UniformFloat")