def test_constant(self): # Test construction c1 = Constant("value", 1) c2 = Constant("value", 1) c3 = Constant("value", 2) c4 = Constant("valuee", 1) c5 = Constant("valueee", 2) # Test the representation self.assertEqual("value, Type: Constant, Value: 1", c1.__repr__()) # Test the equals operator (and the ne operator in the last line) self.assertFalse(c1 == 1) self.assertEqual(c1, c2) self.assertFalse(c1 == c3) self.assertFalse(c1 == c4) self.assertTrue(c1 != c5) # Test that only string, integers and floats are allowed self.assertRaises(TypeError, Constant, "value", dict()) self.assertRaises(TypeError, Constant, "value", None) self.assertRaises(TypeError, Constant, "value", True) # Test that only string names are allowed self.assertRaises(TypeError, Constant, 1, "value") self.assertRaises(TypeError, Constant, dict(), "value") self.assertRaises(TypeError, Constant, None, "value") self.assertRaises(TypeError, Constant, True, "value")
def get_hyperparameter_search_space(dataset_properties=None): n_neighbors = UniformIntegerHyperparameter(name="n_neighbors", lower=1, upper=100, default=1) weights = CategoricalHyperparameter(name="weights", choices=["uniform", "distance"], default="uniform") metric = UnParametrizedHyperparameter(name="metric", value="minkowski") algorithm = Constant(name='algorithm', value="auto") p = CategoricalHyperparameter(name="p", choices=[1, 2, 5], default=2) leaf_size = Constant(name="leaf_size", value=30) # Unparametrized # TODO: If we further parametrize 'metric' we need more metric params metric = UnParametrizedHyperparameter(name="metric", value="minkowski") cs = ConfigurationSpace() cs.add_hyperparameter(n_neighbors) cs.add_hyperparameter(weights) cs.add_hyperparameter(metric) cs.add_hyperparameter(algorithm) cs.add_hyperparameter(p) cs.add_hyperparameter(leaf_size) # Conditions metric_p = EqualsCondition(parent=metric, child=p, value="minkowski") cs.add_condition(metric_p) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() C = cs.add_hyperparameter(UniformFloatHyperparameter( "C", 0.03125, 32768, log=True, default=1.0)) loss = cs.add_hyperparameter(CategoricalHyperparameter( "loss", ["epsilon_insensitive", "squared_epsilon_insensitive"], default="squared_epsilon_insensitive")) # Random Guess epsilon = cs.add_hyperparameter(UniformFloatHyperparameter( name="epsilon", lower=0.001, upper=1, default=0.1, log=True)) dual = cs.add_hyperparameter(Constant("dual", "False")) # These are set ad-hoc tol = cs.add_hyperparameter(UniformFloatHyperparameter( "tol", 1e-5, 1e-1, default=1e-4, log=True)) fit_intercept = cs.add_hyperparameter(Constant("fit_intercept", "True")) intercept_scaling = cs.add_hyperparameter(Constant( "intercept_scaling", 1)) dual_and_loss = ForbiddenAndConjunction( ForbiddenEqualsClause(dual, "False"), ForbiddenEqualsClause(loss, "epsilon_insensitive") ) cs.add_forbidden_clause(dual_and_loss) return cs
def test_and_conjunction(self): self.assertRaises(TypeError, AndConjunction, "String1", "String2") hp1 = CategoricalHyperparameter("input1", [0, 1]) hp2 = CategoricalHyperparameter("input2", [0, 1]) hp3 = CategoricalHyperparameter("input3", [0, 1]) hp4 = Constant("And", "True") cond1 = EqualsCondition(hp4, hp1, 1) # Only one condition in an AndConjunction! self.assertRaises(ValueError, AndConjunction, cond1) cond2 = EqualsCondition(hp4, hp2, 1) cond3 = EqualsCondition(hp4, hp3, 1) andconj1 = AndConjunction(cond1, cond2) andconj1_ = AndConjunction(cond1, cond2) self.assertEqual(andconj1, andconj1_) andconj2 = AndConjunction(cond2, cond3) self.assertNotEqual(andconj1, andconj2) andconj3 = AndConjunction(cond1, cond2, cond3) self.assertEqual( "(And | input1 == 1 && And | input2 == 1 && And | " "input3 == 1)", str(andconj3)) # Test __eq__ self.assertNotEqual(andconj1, andconj3) self.assertNotEqual(andconj1, "String")
def test_or_conjunction(self): self.assertRaises(TypeError, AndConjunction, "String1", "String2") hp1 = CategoricalHyperparameter("input1", [0, 1]) hp2 = CategoricalHyperparameter("input2", [0, 1]) hp3 = CategoricalHyperparameter("input3", [0, 1]) hp4 = Constant("Or", "True") cond1 = EqualsCondition(hp4, hp1, 1) self.assertRaises(ValueError, OrConjunction, cond1) cond2 = EqualsCondition(hp4, hp2, 1) cond3 = EqualsCondition(hp4, hp3, 1) andconj1 = OrConjunction(cond1, cond2) andconj1_ = OrConjunction(cond1, cond2) self.assertEqual(andconj1, andconj1_) andconj2 = OrConjunction(cond2, cond3) self.assertNotEqual(andconj1, andconj2) andconj3 = OrConjunction(cond1, cond2, cond3) self.assertEqual( "(Or | input1 == 1 || Or | input2 == 1 || Or | " "input3 == 1)", str(andconj3))
def get_hyperparameter_search_space(dataset_properties=None): cs = ConstrainedFeedNet.get_hyperparameter_search_space() solver = Constant(name='solver', value='sgd') cs.add_hyperparameter(solver) # lr policy is fixed by default return cs
def get_hyperparameter_search_space(dataset_properties=None): #n_estimators = UniformIntegerHyperparameter( # "n_estimators", 10, 100, default=10) n_estimators = Constant("n_estimators", 100) criterion = CategoricalHyperparameter("criterion", ["gini", "entropy"], default="gini") #max_features = UniformFloatHyperparameter( # "max_features", 0.01, 0.5, default=0.2) max_features = UniformFloatHyperparameter("max_features", 0.5, 5, default=1) max_depth = UnParametrizedHyperparameter("max_depth", "None") min_samples_split = UniformIntegerHyperparameter("min_samples_split", 2, 20, default=2) min_samples_leaf = UniformIntegerHyperparameter("min_samples_leaf", 1, 20, default=1) max_leaf_nodes = UnParametrizedHyperparameter("max_leaf_nodes", "None") bootstrap = CategoricalHyperparameter("bootstrap", ["True", "False"], default="True") cs = ConfigurationSpace() cs.add_hyperparameter(n_estimators) cs.add_hyperparameter(criterion) cs.add_hyperparameter(max_features) cs.add_hyperparameter(max_depth) cs.add_hyperparameter(min_samples_split) cs.add_hyperparameter(min_samples_leaf) cs.add_hyperparameter(max_leaf_nodes) cs.add_hyperparameter(bootstrap) return cs
def _lr_policy_configuration_space(cs, policy=None): if policy == 'inv': lr_policy = Constant(name='lr_policy', value='inv') gamma = UniformFloatHyperparameter(name="gamma", lower=1e-2, upper=1.0, log=True, default=1e-2) power = UniformFloatHyperparameter("power", 0.0, 1.0, default=0.5) cs.add_hyperparameter(lr_policy) cs.add_hyperparameter(gamma) cs.add_hyperparameter(power) elif policy == 'exp': lr_policy = Constant(name='lr_policy', value='exp') gamma = UniformFloatHyperparameter(name="gamma", lower=0.7, upper=1.0, default=0.79) cs.add_hyperparameter(lr_policy) cs.add_hyperparameter(gamma) elif policy == 'step': lr_policy = Constant(name='lr_policy', value='step') gamma = UniformFloatHyperparameter(name="gamma", lower=1e-2, upper=1.0, log=True, default=1e-2) epoch_step = CategoricalHyperparameter("epoch_step", [6, 8, 12], default=8) cs.add_hyperparameter(lr_policy) cs.add_hyperparameter(gamma) cs.add_hyperparameter(epoch_step) else: lr_policy = Constant(name='lr_policy', value='fixed') cs.add_hyperparameter(lr_policy) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() loss = cs.add_hyperparameter(Constant("loss", "deviance")) learning_rate = cs.add_hyperparameter( UniformFloatHyperparameter(name="learning_rate", lower=0.0001, upper=1, default=0.1, log=True)) n_estimators = cs.add_hyperparameter(Constant("n_estimators", 100)) max_depth = cs.add_hyperparameter( UniformIntegerHyperparameter(name="max_depth", lower=1, upper=10, default=3)) min_samples_split = cs.add_hyperparameter( UniformIntegerHyperparameter(name="min_samples_split", lower=2, upper=20, default=2, log=False)) min_samples_leaf = cs.add_hyperparameter( UniformIntegerHyperparameter(name="min_samples_leaf", lower=1, upper=20, default=1, log=False)) min_weight_fraction_leaf = cs.add_hyperparameter( UnParametrizedHyperparameter("min_weight_fraction_leaf", 0.)) subsample = cs.add_hyperparameter( UniformFloatHyperparameter(name="subsample", lower=0.01, upper=1.0, default=1.0, log=False)) max_features = cs.add_hyperparameter( UniformFloatHyperparameter("max_features", 0.5, 5, default=1)) max_leaf_nodes = cs.add_hyperparameter( UnParametrizedHyperparameter(name="max_leaf_nodes", value="None")) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConstrainedFeedNet.get_hyperparameter_search_space() solver = Constant(name='solver', value='momentum') cs.add_hyperparameter(solver) momentum = UniformFloatHyperparameter("momentum", 0.3, 0.999, default=0.9) cs.add_hyperparameter(momentum) return cs
def test_sample_configuration(self): cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("parent", [0, 1]) cs.add_hyperparameter(hp1) hp2 = UniformIntegerHyperparameter("child", 0, 10) cs.add_hyperparameter(hp2) cond1 = EqualsCondition(hp2, hp1, 0) cs.add_condition(cond1) # This automatically checks the configuration! Configuration(cs, dict(parent=0, child=5)) # and now for something more complicated cs = ConfigurationSpace(seed=1) hp1 = CategoricalHyperparameter("input1", [0, 1]) cs.add_hyperparameter(hp1) hp2 = CategoricalHyperparameter("input2", [0, 1]) cs.add_hyperparameter(hp2) hp3 = CategoricalHyperparameter("input3", [0, 1]) cs.add_hyperparameter(hp3) hp4 = CategoricalHyperparameter("input4", [0, 1]) cs.add_hyperparameter(hp4) hp5 = CategoricalHyperparameter("input5", [0, 1]) cs.add_hyperparameter(hp5) hp6 = Constant("AND", "True") cs.add_hyperparameter(hp6) cond1 = EqualsCondition(hp6, hp1, 1) cond2 = NotEqualsCondition(hp6, hp2, 1) cond3 = InCondition(hp6, hp3, [1]) cond4 = EqualsCondition(hp5, hp3, 1) cond5 = EqualsCondition(hp4, hp5, 1) cond6 = EqualsCondition(hp6, hp4, 1) cond7 = EqualsCondition(hp6, hp5, 1) conj1 = AndConjunction(cond1, cond2) conj2 = OrConjunction(conj1, cond3) conj3 = AndConjunction(conj2, cond6, cond7) cs.add_condition(cond4) cs.add_condition(cond5) cs.add_condition(conj3) samples = [] for i in range(5): cs.seed(1) samples.append([]) for j in range(100): sample = cs.sample_configuration() samples[-1].append(sample) if i > 0: for j in range(100): self.assertEqual(samples[-1][j], samples[-2][j])
def test_get_hyperparameters_topological_sort(self): for iteration in range(10): cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("parent", [0, 1]) cs.add_hyperparameter(hp1) hp2 = UniformIntegerHyperparameter("child", 0, 10) cs.add_hyperparameter(hp2) cond1 = EqualsCondition(hp2, hp1, 0) cs.add_condition(cond1) # This automatically checks the configuration! Configuration(cs, dict(parent=0, child=5)) # and now for something more complicated cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("input1", [0, 1]) hp2 = CategoricalHyperparameter("input2", [0, 1]) hp3 = CategoricalHyperparameter("input3", [0, 1]) hp4 = CategoricalHyperparameter("input4", [0, 1]) hp5 = CategoricalHyperparameter("input5", [0, 1]) hp6 = Constant("AND", "True") # More top-level hyperparameters hp7 = CategoricalHyperparameter("input7", [0, 1]) hps = [hp1, hp2, hp3, hp4, hp5, hp6, hp7] random.shuffle(hps) for hp in hps: cs.add_hyperparameter(hp) cond1 = EqualsCondition(hp6, hp1, 1) cond2 = NotEqualsCondition(hp6, hp2, 1) cond3 = InCondition(hp6, hp3, [1]) cond4 = EqualsCondition(hp5, hp3, 1) cond5 = EqualsCondition(hp4, hp5, 1) cond6 = EqualsCondition(hp6, hp4, 1) cond7 = EqualsCondition(hp6, hp5, 1) conj1 = AndConjunction(cond1, cond2) conj2 = OrConjunction(conj1, cond3) conj3 = AndConjunction(conj2, cond6, cond7) cs.add_condition(cond4) cs.add_condition(cond5) cs.add_condition(conj3) hps = cs.get_hyperparameters() self.assertEqual(hps.index(hp1), 0) self.assertEqual(hps.index(hp2), 1) self.assertEqual(hps.index(hp3), 2) self.assertEqual(hps.index(hp7), 3) self.assertEqual(hps.index(hp5), 4) self.assertEqual(hps.index(hp4), 5) self.assertEqual(hps.index(hp6), 6)
def get_hyperparameter_search_space(dataset_properties=None): cs = ConstrainedFeedNet.get_hyperparameter_search_space() solver = Constant(name='solver', value='adam') cs.add_hyperparameter(solver) beta1 = UniformFloatHyperparameter("beta1", 1e-4, 0.1, log=True, default=0.1) beta2 = UniformFloatHyperparameter("beta2", 1e-4, 0.1, log=True, default=0.1) cs.add_hyperparameter(beta1) cs.add_hyperparameter(beta2) return cs
def get_hyperparameter_search_space(dataset_properties=None): percentile = UniformFloatHyperparameter( name="percentile", lower=10, upper=90, default=50) score_func = CategoricalHyperparameter( name="score_func", choices=["chi2", "f_classif"], default="chi2") if dataset_properties is not None: # Chi2 can handle sparse data, so we respect this if 'sparse' in dataset_properties and dataset_properties['sparse']: score_func = Constant( name="score_func", value="chi2") cs = ConfigurationSpace() cs.add_hyperparameter(percentile) cs.add_hyperparameter(score_func) return cs
def test_all_components_have_the_same_child(self): hp1 = CategoricalHyperparameter("input1", [0, 1]) hp2 = CategoricalHyperparameter("input2", [0, 1]) hp3 = CategoricalHyperparameter("input3", [0, 1]) hp4 = CategoricalHyperparameter("input4", [0, 1]) hp5 = CategoricalHyperparameter("input5", [0, 1]) hp6 = Constant("AND", "True") cond1 = EqualsCondition(hp1, hp2, 1) cond2 = EqualsCondition(hp1, hp3, 1) cond3 = EqualsCondition(hp1, hp4, 1) cond4 = EqualsCondition(hp6, hp4, 1) cond5 = EqualsCondition(hp6, hp5, 1) AndConjunction(cond1, cond2, cond3) AndConjunction(cond4, cond5) self.assertRaisesRegexp( ValueError, "All Conjunctions and Conditions must have " "the same child.", AndConjunction, cond1, cond4)
def test_add_second_condition_wo_conjunction(self): hp1 = CategoricalHyperparameter("input1", [0, 1]) hp2 = CategoricalHyperparameter("input2", [0, 1]) hp3 = Constant("And", "True") cond1 = EqualsCondition(hp3, hp1, 1) cond2 = EqualsCondition(hp3, hp2, 1) cs = ConfigurationSpace() cs.add_hyperparameter(hp1) cs.add_hyperparameter(hp2) cs.add_hyperparameter(hp3) cs.add_condition(cond1) self.assertRaisesRegexp( ValueError, "Adding a second condition \(different\) for a " "hyperparameter is ambigouos and " "therefore forbidden. Add a conjunction " "instead!", cs.add_condition, cond2)
def get_hyperparameter_search_space(dataset_properties=None): loss = CategoricalHyperparameter( name="loss", choices=["ls", "lad"], default='ls') #, "huber", "quantile"], default='ls') learning_rate = UniformFloatHyperparameter(name="learning_rate", lower=0.0001, upper=1, default=0.1, log=True) subsample = UniformFloatHyperparameter(name="subsample", lower=0.01, upper=1.0, default=1.0, log=False) n_estimators = Constant("n_estimators", 100) max_features = UniformFloatHyperparameter("max_features", 0.5, 5, default=1) max_depth = UniformIntegerHyperparameter(name="max_depth", lower=1, upper=10, default=3) min_samples_split = UniformIntegerHyperparameter( name="min_samples_split", lower=2, upper=20, default=2, log=False) min_samples_leaf = UniformIntegerHyperparameter( name="min_samples_leaf", lower=1, upper=20, default=1, log=False) cs = ConfigurationSpace() cs.add_hyperparameter(n_estimators) cs.add_hyperparameter(loss) cs.add_hyperparameter(learning_rate) cs.add_hyperparameter(max_features) cs.add_hyperparameter(max_depth) cs.add_hyperparameter(min_samples_split) cs.add_hyperparameter(min_samples_leaf) cs.add_hyperparameter(subsample) return cs
def test_add_conjunction(self): hp1 = CategoricalHyperparameter("input1", [0, 1]) hp2 = CategoricalHyperparameter("input2", [0, 1]) hp3 = CategoricalHyperparameter("input3", [0, 1]) hp4 = Constant("And", "True") cond1 = EqualsCondition(hp4, hp1, 1) cond2 = EqualsCondition(hp4, hp2, 1) cond3 = EqualsCondition(hp4, hp3, 1) andconj1 = AndConjunction(cond1, cond2, cond3) cs = ConfigurationSpace() cs.add_hyperparameter(hp1) cs.add_hyperparameter(hp2) cs.add_hyperparameter(hp3) cs.add_hyperparameter(hp4) cs.add_condition(andconj1) self.assertNotIn(hp4, cs.get_all_uncoditional_hyperparameters())
def get_hyperparameter_search_space(dataset_properties=None): alpha = UniformFloatHyperparameter(name="alpha", lower=0.01, upper=0.5, default=0.1) score_func = CategoricalHyperparameter(name="score_func", choices=["chi2", "f_classif"], default="chi2") if dataset_properties is not None: # Chi2 can handle sparse data, so we respect this if 'sparse' in dataset_properties and dataset_properties['sparse']: score_func = Constant(name="score_func", value="chi2") mode = CategoricalHyperparameter('mode', ['fpr', 'fdr', 'fwe'], 'fpr') cs = ConfigurationSpace() cs.add_hyperparameter(alpha) cs.add_hyperparameter(score_func) cs.add_hyperparameter(mode) return cs
def get_hyperparameter_search_space(dataset_properties=None): n_estimators = UniformIntegerHyperparameter(name="n_estimators", lower=10, upper=100, default=10) max_depth = UniformIntegerHyperparameter(name="max_depth", lower=2, upper=10, default=5) min_samples_split = UniformIntegerHyperparameter(name="min_samples_split", lower=2, upper=20, default=2) min_samples_leaf = UniformIntegerHyperparameter(name="min_samples_leaf", lower=1, upper=20, default=1) min_weight_fraction_leaf = Constant('min_weight_fraction_leaf', 1.0) max_leaf_nodes = UnParametrizedHyperparameter(name="max_leaf_nodes", value="None") cs = ConfigurationSpace() cs.add_hyperparameter(n_estimators) cs.add_hyperparameter(max_depth) cs.add_hyperparameter(min_samples_split) cs.add_hyperparameter(min_samples_leaf) cs.add_hyperparameter(min_weight_fraction_leaf) cs.add_hyperparameter(max_leaf_nodes) return cs
def test_nested_conjunctions(self): hp1 = CategoricalHyperparameter("input1", [0, 1]) hp2 = CategoricalHyperparameter("input2", [0, 1]) hp3 = CategoricalHyperparameter("input3", [0, 1]) hp4 = CategoricalHyperparameter("input4", [0, 1]) hp5 = CategoricalHyperparameter("input5", [0, 1]) hp6 = Constant("AND", "True") cond1 = EqualsCondition(hp6, hp1, 1) cond2 = EqualsCondition(hp6, hp2, 1) cond3 = EqualsCondition(hp6, hp3, 1) cond4 = EqualsCondition(hp6, hp4, 1) cond5 = EqualsCondition(hp6, hp5, 1) conj1 = AndConjunction(cond1, cond2) conj2 = OrConjunction(conj1, cond3) conj3 = AndConjunction(conj2, cond4, cond5) # TODO: this does not look nice, And should depend on a large # conjunction, there should not be many ANDs inside this string! self.assertEqual( "(((AND | input1 == 1 && AND | input2 == 1) || AND | " "input3 == 1) && AND | input4 == 1 && AND | input5 " "== 1)", str(conj3))
def get_hyperparameter_search_space(dataset_properties=None): # Constrain the search space of the DeepFeedNet # Fixed Architecture for MNIST batch_size = Constant(name='batch_size', value=963) # TODO: Decimal library to work around floating point issue dropout_layer_1 = Constant(name='dropout_layer_1', value=0.39426633933) dropout_output = Constant(name='dropout_output', value=0.085813712701) num_layers = Constant(name='num_layers', value='c') num_units_layer_1 = Constant(name='num_units_layer_1', value=1861) number_updates = Constant(name='number_updates', value=1105) std_layer_1 = Constant(name='std_layer_1', value=0.00351015701) # To Optimize for all cases l2 = UniformFloatHyperparameter("lambda2", 1e-6, 1e-2, log=True, default=1e-3) lr = UniformFloatHyperparameter("learning_rate", 1e-4, 1e-1, log=True, default=1e-2) cs = ConfigurationSpace() cs.add_hyperparameter(batch_size) cs.add_hyperparameter(number_updates) cs.add_hyperparameter(num_layers) cs.add_hyperparameter(num_units_layer_1) cs.add_hyperparameter(dropout_layer_1) cs.add_hyperparameter(dropout_output) cs.add_hyperparameter(std_layer_1) cs.add_hyperparameter(lr) cs.add_hyperparameter(l2) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConstrainedFeedNet.get_hyperparameter_search_space() solver = Constant(name='solver', value='adagrad') cs.add_hyperparameter(solver) return cs