def get_complete_configspace(): """Creates a configspace that includes all kinds of parameters with complicated values. The idea is to provide a configspace that can be used to check modules using ConfigSpace as a dependency to check compatibility with e.g. Constants, log-scale, etc. Returns ------- cs: ConfigurationSpace cs containing all kinds of parameters """ cs = ConfigurationSpace() hp = {} # Add Constants for all allowed types ('int', 'float', 'string') hp['alpha'] = Constant("alpha", 0.0001) hp['tol'] = Constant("tol", '0.0001') hp['verbose'] = Constant("verbose", 1) # Add numericals # Add Floats hp['beta1'] = UniformFloatHyperparameter("beta1", 0.85, 0.95, log=False) hp['power_t'] = NormalFloatHyperparameter("power_t", mu=0.5, sigma=0.1, log=False) # Add Ints hp['momentum'] = UniformIntegerHyperparameter("momentum", 0, 100, False) hp['beta2'] = NormalIntegerHyperparameter("beta2", mu=1, sigma=0.001, log=False) # Add Floats (log) hp['learning_rate_init'] = UniformFloatHyperparameter("learning_rate_init", 0.0001, 0.1, log=True) hp['random1'] = NormalFloatHyperparameter("NormalFloat", mu=0, sigma=1, default_value=1, log=True) # Add Ints (log) hp['random2'] = UniformIntegerHyperparameter("UniformInt", 2, 100, log=True) hp['random3'] = NormalIntegerHyperparameter("NormalInt", mu=0, sigma=1, default_value=1, log=True) # Add Categorical for allowed types hp['activation'] = CategoricalHyperparameter('activation', choices=['identity', 'logistic', 'tanh', 'relu']) hp['solver'] = CategoricalHyperparameter('solver', choices=[-2, 0, 2]) # corrresponds to: ‘lbfgs’, ‘sgd’, ‘adam’ hp['batch_size_auto'] = CategoricalHyperparameter('batch_size_auto', choices=[True, False]) hp['learning_rate'] = CategoricalHyperparameter('learning_rate', choices=[-0.5, 0.0, 0.5]) # corresponds to {‘constant’, ‘invscaling’, ‘adaptive’} # Add Ordinal hp['batch_size'] = OrdinalHyperparameter('batch_size', sequence=[32, 64.0, '128']) for k, v in hp.items(): cs.add_hyperparameter(v) # learning_rate only with sgd c = InCondition(hp['learning_rate'], hp['solver'], [0]) c = EqualsCondition(hp['momentum'], hp['solver'], 0) # learning_rate_init only with sgd or adam cs.add_condition(OrConjunction(EqualsCondition(hp['learning_rate'], hp['solver'], 0), # sgd EqualsCondition(hp['learning_rate'], hp['solver'], 2))) # adam # batch_size only with not batch_size_auto cs.add_condition(NotEqualsCondition(hp['batch_size'], hp['batch_size_auto'], True)) # complicated way for solver == sgd #cs.add_condition(AndConjunction(LessThanCondition(hp['power_t'], hp['solver'], 1), # GreaterThanCondition(hp['power_t'], hp['solver'], -1))) # betas with adam cs.add_condition(EqualsCondition(hp['beta1'], hp['solver'], 2)) cs.add_condition(EqualsCondition(hp['beta2'], hp['solver'], 2)) return cs
def test_normalfloat_is_legal(self): f1 = NormalFloatHyperparameter("param", 0, 10) self.assertTrue(f1.is_legal(3.0)) self.assertTrue(f1.is_legal(2)) self.assertFalse(f1.is_legal("Hahaha")) # Test legal vector values self.assertTrue(f1.is_legal_vector(1.0)) self.assertTrue(f1.is_legal_vector(0.0)) self.assertTrue(f1.is_legal_vector(0)) self.assertTrue(f1.is_legal_vector(0.3)) self.assertTrue(f1.is_legal_vector(-0.1)) self.assertTrue(f1.is_legal_vector(1.1)) self.assertRaises(TypeError, f1.is_legal_vector, "Hahaha")
def test_normalfloat_is_legal(self): f1 = NormalFloatHyperparameter("param", 0, 10) self.assertTrue(f1.is_legal(3.0)) self.assertTrue(f1.is_legal(2)) self.assertFalse(f1.is_legal("Hahaha")) # Test legal vector values self.assertTrue(f1.is_legal_vector(1.0)) self.assertTrue(f1.is_legal_vector(0.0)) self.assertTrue(f1.is_legal_vector(0)) self.assertTrue(f1.is_legal_vector(0.3)) self.assertTrue(f1.is_legal_vector(-0.1)) self.assertTrue(f1.is_legal_vector(1.1)) self.assertFalse(f1.is_legal_vector("Hahaha"))
def test_normalfloat(self): # TODO test non-equality f1 = NormalFloatHyperparameter("param", 0.5, 10.5) f1_ = NormalFloatHyperparameter("param", 0.5, 10.5) self.assertEqual(f1, f1_) self.assertEqual( "param, Type: NormalFloat, Mu: 0.5 Sigma: 10.5, Default: 0.5", str(f1)) f2 = NormalFloatHyperparameter("param", 0, 10, q=0.1) f2_ = NormalFloatHyperparameter("param", 0, 10, q=0.1) self.assertEqual(f2, f2_) self.assertEqual( "param, Type: NormalFloat, Mu: 0.0 Sigma: 10.0, Default: 0.0, " "Q: 0.1", str(f2)) f3 = NormalFloatHyperparameter("param", 0, 10, log=True) f3_ = NormalFloatHyperparameter("param", 0, 10, log=True) self.assertEqual(f3, f3_) self.assertEqual( "param, Type: NormalFloat, Mu: 0.0 Sigma: 10.0, Default: 0.0, " "on log-scale", str(f3)) f4 = NormalFloatHyperparameter("param", 0, 10, default=1.0) f4_ = NormalFloatHyperparameter("param", 0, 10, default=1.0) self.assertEqual(f4, f4_) self.assertEqual( "param, Type: NormalFloat, Mu: 0.0 Sigma: 10.0, Default: 1.0", str(f4)) f5 = NormalFloatHyperparameter("param", 0, 10, default=1.0, q=0.1, log=True) f5_ = NormalFloatHyperparameter("param", 0, 10, default=1.0, q=0.1, log=True) self.assertEqual(f5, f5_) self.assertEqual( "param, Type: NormalFloat, Mu: 0.0 Sigma: 10.0, Default: 1.0, " "on log-scale, Q: 0.1", str(f5)) self.assertNotEqual(f1, f2) self.assertNotEqual(f1, "UniformFloat")
def _construct_hyperparameter(hyperparameter: Dict) -> Hyperparameter: hp_type = hyperparameter['type'] name = hyperparameter['name'] if hp_type == 'constant': return Constant( name=name, value=hyperparameter['value'], ) elif hp_type == 'unparametrized': return UnParametrizedHyperparameter( name=name, value=hyperparameter['value'], ) elif hp_type == 'uniform_float': return UniformFloatHyperparameter( name=name, log=hyperparameter['log'], lower=hyperparameter['lower'], upper=hyperparameter['upper'], default_value=hyperparameter['default'], ) elif hp_type == 'normal_float': return NormalFloatHyperparameter( name=name, log=hyperparameter['log'], mu=hyperparameter['mu'], sigma=hyperparameter['sigma'], default_value=hyperparameter['default'], ) elif hp_type == 'uniform_int': return UniformIntegerHyperparameter( name=name, log=hyperparameter['log'], lower=hyperparameter['lower'], upper=hyperparameter['upper'], default_value=hyperparameter['default'], ) elif hp_type == 'normal_int': return NormalIntegerHyperparameter( name=name, log=hyperparameter['log'], lower=hyperparameter['lower'], upper=hyperparameter['upper'], default_value=hyperparameter['default'], ) elif hp_type == 'categorical': return CategoricalHyperparameter( name=name, choices=hyperparameter['choices'], default_value=hyperparameter['default'], weights=hyperparameter.get('probabilities'), ) elif hp_type == 'ordinal': return OrdinalHyperparameter( name=name, sequence=hyperparameter['sequence'], default_value=hyperparameter['default'], ) else: raise ValueError(hp_type)
def test_hyperparam_representation(self): # Float f1 = UniformFloatHyperparameter("param", 1, 100, log=True) self.assertEqual( "param, Type: UniformFloat, Range: [1.0, 100.0], Default: 10.0, on log-scale", repr(f1) ) f2 = NormalFloatHyperparameter("param", 8, 99.1, log=False) self.assertEqual( "param, Type: NormalFloat, Mu: 8.0 Sigma: 99.1, Default: 8.0", repr(f2) ) i1 = UniformIntegerHyperparameter("param", 0, 100) self.assertEqual( "param, Type: UniformInteger, Range: [0, 100], Default: 50", repr(i1) ) i2 = NormalIntegerHyperparameter("param", 5, 8) self.assertEqual( "param, Type: NormalInteger, Mu: 5 Sigma: 8, Default: 5", repr(i2) ) o1 = OrdinalHyperparameter("temp", ["freezing", "cold", "warm", "hot"]) self.assertEqual( "temp, Type: Ordinal, Sequence: {freezing, cold, warm, hot}, Default: freezing", repr(o1) ) c1 = CategoricalHyperparameter("param", [True, False]) self.assertEqual( "param, Type: Categorical, Choices: {True, False}, Default: True", repr(c1) )
def test_uniformfloat_transform(self): """This checks whether a value sampled through the configuration space (it does not happend when the variable is sampled alone) stays equal when it is serialized via JSON and the deserialized again.""" cs = ConfigurationSpace() a = cs.add_hyperparameter(UniformFloatHyperparameter('a', -5, 10)) b = cs.add_hyperparameter( NormalFloatHyperparameter('b', 1, 2, log=True)) for i in range(100): config = cs.sample_configuration() value = OrderedDict(sorted(config.get_dictionary().items())) string = json.dumps(value) saved_value = json.loads(string) saved_value = OrderedDict(sorted(byteify(saved_value).items())) self.assertEqual(repr(value), repr(saved_value)) # Next, test whether the truncation also works when initializing the # Configuration with a dictionary for i in range(100): rs = np.random.RandomState(1) value_a = a.sample(rs) value_b = b.sample(rs) values_dict = {'a': value_a, 'b': value_b} config = Configuration(cs, values=values_dict) string = json.dumps(config.get_dictionary()) saved_value = json.loads(string) saved_value = byteify(saved_value) self.assertEqual(values_dict, saved_value)
def test_meta_field(self): cs = ConfigurationSpace() cs.add_hyperparameter( UniformIntegerHyperparameter("uihp", lower=1, upper=10, meta=dict(uihp=True))) cs.add_hyperparameter( NormalIntegerHyperparameter("nihp", mu=0, sigma=1, meta=dict(nihp=True))) cs.add_hyperparameter( UniformFloatHyperparameter("ufhp", lower=1, upper=10, meta=dict(ufhp=True))) cs.add_hyperparameter( NormalFloatHyperparameter("nfhp", mu=0, sigma=1, meta=dict(nfhp=True))) cs.add_hyperparameter( CategoricalHyperparameter("chp", choices=['1', '2', '3'], meta=dict(chp=True))) cs.add_hyperparameter( OrdinalHyperparameter("ohp", sequence=['1', '2', '3'], meta=dict(ohp=True))) cs.add_hyperparameter(Constant("const", value=1, meta=dict(const=True))) parent = ConfigurationSpace() parent.add_configuration_space("sub", cs, delimiter=':') self.assertEqual( parent.get_hyperparameter("sub:uihp").meta, dict(uihp=True)) self.assertEqual( parent.get_hyperparameter("sub:nihp").meta, dict(nihp=True)) self.assertEqual( parent.get_hyperparameter("sub:ufhp").meta, dict(ufhp=True)) self.assertEqual( parent.get_hyperparameter("sub:nfhp").meta, dict(nfhp=True)) self.assertEqual( parent.get_hyperparameter("sub:chp").meta, dict(chp=True)) self.assertEqual( parent.get_hyperparameter("sub:ohp").meta, dict(ohp=True)) self.assertEqual( parent.get_hyperparameter("sub:const").meta, dict(const=True))
def test_sample_NormalFloatHyperparameter(self): hp = NormalFloatHyperparameter("nfhp", 0, 1) def actual_test(): rs = np.random.RandomState(1) counts_per_bin = [0 for i in range(11)] for i in range(100000): value = hp.sample(rs) index = min(max(int((round(value + 0.5)) + 5), 0), 9) counts_per_bin[index] += 1 self.assertEqual([0, 4, 138, 2113, 13394, 34104, 34282, 13683, 2136, 146, 0], counts_per_bin) return counts_per_bin self.assertEqual(actual_test(), actual_test())
def test_repr_roundtrip(self): cs = ConfigurationSpace() cs.add_hyperparameter( UniformIntegerHyperparameter("uihp", lower=1, upper=10)) cs.add_hyperparameter( NormalIntegerHyperparameter("nihp", mu=0, sigma=1)) cs.add_hyperparameter( UniformFloatHyperparameter("ufhp", lower=1, upper=10)) cs.add_hyperparameter(NormalFloatHyperparameter("nfhp", mu=0, sigma=1)) cs.add_hyperparameter( CategoricalHyperparameter("chp", choices=['1', '2', '3'])) cs.add_hyperparameter( OrdinalHyperparameter("ohp", sequence=['1', '2', '3'])) cs.add_hyperparameter(Constant("const", value=1)) default = cs.get_default_configuration() repr = default.__repr__() repr = repr.replace('})', '}, configuration_space=cs)') config = eval(repr) self.assertEqual(default, config)
def test_normalfloat_to_uniformfloat(self): f1 = NormalFloatHyperparameter("param", 0, 10, q=0.1) f1_expected = UniformFloatHyperparameter("param", -30, 30, q=0.1) f1_actual = f1.to_uniform() self.assertEqual(f1_expected, f1_actual)
def test_normalfloat_to_integer(self): f1 = NormalFloatHyperparameter("param", 0, 10) f2_expected = NormalIntegerHyperparameter("param", 0, 10) f2_actual = f1.to_integer() self.assertEqual(f2_expected, f2_actual)
def test_normalfloat_is_legal(self): f1 = NormalFloatHyperparameter("param", 0, 10) self.assertTrue(f1.is_legal(3.0)) self.assertTrue(f1.is_legal(2)) self.assertFalse(f1.is_legal("Hahaha"))
def test_normalfloat(self): # TODO test non-equality f1 = NormalFloatHyperparameter("param", 0.5, 10.5) f1_ = NormalFloatHyperparameter("param", 0.5, 10.5) self.assertEqual(f1, f1_) self.assertEqual( "param, Type: NormalFloat, Mu: 0.5 Sigma: 10.5, Default: 0.5", str(f1)) # Test attributes are accessible self.assertEqual(f1.name, "param") self.assertAlmostEqual(f1.mu, 0.5) self.assertAlmostEqual(f1.sigma, 10.5) self.assertAlmostEqual(f1.q, None) self.assertEqual(f1.log, False) self.assertAlmostEqual(f1.default_value, 0.5) self.assertAlmostEqual(f1.normalized_default_value, 0.5) f2 = NormalFloatHyperparameter("param", 0, 10, q=0.1) f2_ = NormalFloatHyperparameter("param", 0, 10, q=0.1) self.assertEqual(f2, f2_) self.assertEqual( "param, Type: NormalFloat, Mu: 0.0 Sigma: 10.0, Default: 0.0, " "Q: 0.1", str(f2)) with pytest.warns(RuntimeWarning, match='divide by zero encountered in log'): f3 = NormalFloatHyperparameter("param", 0, 10, log=True) with pytest.warns(RuntimeWarning, match='divide by zero encountered in log'): f3_ = NormalFloatHyperparameter("param", 0, 10, log=True) self.assertEqual(f3, f3_) self.assertEqual( "param, Type: NormalFloat, Mu: 0.0 Sigma: 10.0, Default: 0.0, " "on log-scale", str(f3)) f4 = NormalFloatHyperparameter("param", 0, 10, default_value=1.0) f4_ = NormalFloatHyperparameter("param", 0, 10, default_value=1.0) self.assertEqual(f4, f4_) self.assertEqual( "param, Type: NormalFloat, Mu: 0.0 Sigma: 10.0, Default: 1.0", str(f4)) f5 = NormalFloatHyperparameter("param", 0, 10, default_value=1.0, q=0.1, log=True) f5_ = NormalFloatHyperparameter("param", 0, 10, default_value=1.0, q=0.1, log=True) self.assertEqual(f5, f5_) self.assertEqual( "param, Type: NormalFloat, Mu: 0.0 Sigma: 10.0, Default: 1.0, " "on log-scale, Q: 0.1", str(f5)) self.assertNotEqual(f1, f2) self.assertNotEqual(f1, "UniformFloat") # test that meta-data is stored correctly f_meta = NormalFloatHyperparameter("param", 0.1, 10, q=0.1, log=True, default_value=1.0, meta=dict(self.meta_data)) self.assertEqual(f_meta.meta, self.meta_data)
def test_normalfloat(self): # TODO test non-equality f1 = NormalFloatHyperparameter("param", 0.5, 10.5) f1_ = NormalFloatHyperparameter("param", 0.5, 10.5) self.assertEqual(f1, f1_) self.assertEqual( "param, Type: NormalFloat, Mu: 0.5 Sigma: 10.5, Default: 0.5", str(f1)) f2 = NormalFloatHyperparameter("param", 0, 10, q=0.1) f2_ = NormalFloatHyperparameter("param", 0, 10, q=0.1) self.assertEqual(f2, f2_) self.assertEqual( "param, Type: NormalFloat, Mu: 0.0 Sigma: 10.0, Default: 0.0, " "Q: 0.1", str(f2)) f3 = NormalFloatHyperparameter("param", 0, 10, log=True) f3_ = NormalFloatHyperparameter("param", 0, 10, log=True) self.assertEqual(f3, f3_) self.assertEqual( "param, Type: NormalFloat, Mu: 0.0 Sigma: 10.0, Default: 0.0, " "on log-scale", str(f3)) f4 = NormalFloatHyperparameter("param", 0, 10, default_value=1.0) f4_ = NormalFloatHyperparameter("param", 0, 10, default_value=1.0) self.assertEqual(f4, f4_) self.assertEqual( "param, Type: NormalFloat, Mu: 0.0 Sigma: 10.0, Default: 1.0", str(f4)) f5 = NormalFloatHyperparameter("param", 0, 10, default_value=1.0, q=0.1, log=True) f5_ = NormalFloatHyperparameter("param", 0, 10, default_value=1.0, q=0.1, log=True) self.assertEqual(f5, f5_) self.assertEqual( "param, Type: NormalFloat, Mu: 0.0 Sigma: 10.0, Default: 1.0, " "on log-scale, Q: 0.1", str(f5)) self.assertNotEqual(f1, f2) self.assertNotEqual(f1, "UniformFloat") # test that meta-data is stored correctly f_meta = NormalFloatHyperparameter("param", 0.1, 10, q=0.1, log=True, default_value=1.0, meta=dict(self.meta_data)) self.assertEqual(f_meta.meta, self.meta_data)
def cast_hyperparameter(hyperparameter, name): # From D3M hyperparameters to ConfigSpace hyperparameters # TODO: 'Choice' and 'Set' (D3M hyperparameters) new_hyperparameter = None try: if isinstance(hyperparameter, Bounded): lower = hyperparameter.lower upper = hyperparameter.upper default = hyperparameter.get_default() if lower is None: lower = default if upper is None: upper = default * 2 if default > 0 else 10 if hyperparameter.structural_type == int: if not hyperparameter.lower_inclusive: lower += 1 if not hyperparameter.upper_inclusive: upper -= 1 new_hyperparameter = UniformIntegerHyperparameter( name, lower, upper, default_value=default) else: if not hyperparameter.lower_inclusive: lower += 1e-20 if not hyperparameter.upper_inclusive: upper -= 1e-20 new_hyperparameter = UniformFloatHyperparameter( name, lower, upper, default_value=default) elif isinstance(hyperparameter, UniformBool): default = hyperparameter.get_default() new_hyperparameter = CategoricalHyperparameter( name, [True, False], default_value=default) elif isinstance(hyperparameter, UniformInt): lower = hyperparameter.lower upper = hyperparameter.upper default = hyperparameter.get_default() new_hyperparameter = UniformIntegerHyperparameter( name, lower, upper, default_value=default) elif isinstance(hyperparameter, Uniform): lower = hyperparameter.lower upper = hyperparameter.upper default = hyperparameter.get_default() new_hyperparameter = UniformFloatHyperparameter( name, lower, upper, default_value=default) elif isinstance(hyperparameter, Normal): default = hyperparameter.get_default() new_hyperparameter = NormalFloatHyperparameter( name, default_value=default) elif isinstance(hyperparameter, Enumeration): values = hyperparameter.values default = hyperparameter.get_default() new_hyperparameter = CategoricalHyperparameter( name, values, default_value=default) elif isinstance(hyperparameter, ConstantD3M): default = 'None' if hyperparameter.get_default( ) is None else hyperparameter.get_default() new_hyperparameter = Constant(name, default) except Exception as e: logger.error(e) return new_hyperparameter