def get_hyperparameter_search_space(dataset_properties=None): n_components = UniformIntegerHyperparameter("n_components", 10, 2000, default=100) kernel = CategoricalHyperparameter( 'kernel', ['poly', 'rbf', 'sigmoid', 'cosine'], 'rbf') degree = UniformIntegerHyperparameter('degree', 2, 5, 3) gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8, log=True, default=1.0) coef0 = UniformFloatHyperparameter("coef0", -1, 1, default=0) cs = ConfigurationSpace() cs.add_hyperparameter(n_components) cs.add_hyperparameter(kernel) cs.add_hyperparameter(degree) cs.add_hyperparameter(gamma) cs.add_hyperparameter(coef0) degree_depends_on_poly = EqualsCondition(degree, kernel, "poly") coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"]) gamma_condition = InCondition(gamma, kernel, ["poly", "rbf"]) cs.add_condition(degree_depends_on_poly) cs.add_condition(coef0_condition) cs.add_condition(gamma_condition) return cs
def get_hyperparameter_search_space(dataset_properties=None): loss = CategoricalHyperparameter( "loss", ["hinge", "log", "modified_huber", "squared_hinge", "perceptron"], default="hinge") penalty = CategoricalHyperparameter("penalty", ["l1", "l2", "elasticnet"], default="l2") alpha = UniformFloatHyperparameter("alpha", 10**-7, 10**-1, log=True, default=0.0001) l1_ratio = UniformFloatHyperparameter("l1_ratio", 0, 1, default=0.15) fit_intercept = UnParametrizedHyperparameter("fit_intercept", "True") n_iter = UniformIntegerHyperparameter("n_iter", 5, 1000, default=20) epsilon = UniformFloatHyperparameter("epsilon", 1e-5, 1e-1, default=1e-4, log=True) learning_rate = CategoricalHyperparameter( "learning_rate", ["optimal", "invscaling", "constant"], default="optimal") eta0 = UniformFloatHyperparameter("eta0", 10**-7, 0.1, default=0.01) power_t = UniformFloatHyperparameter("power_t", 1e-5, 1, default=0.5) # This does not allow for other resampling methods! class_weight = CategoricalHyperparameter("class_weight", ["None", "auto"], default="None") cs = ConfigurationSpace() cs.add_hyperparameter(loss) cs.add_hyperparameter(penalty) cs.add_hyperparameter(alpha) cs.add_hyperparameter(l1_ratio) cs.add_hyperparameter(fit_intercept) cs.add_hyperparameter(n_iter) cs.add_hyperparameter(epsilon) cs.add_hyperparameter(learning_rate) cs.add_hyperparameter(eta0) cs.add_hyperparameter(power_t) cs.add_hyperparameter(class_weight) # TODO add passive/aggressive here, although not properly documented? elasticnet = EqualsCondition(l1_ratio, penalty, "elasticnet") epsilon_condition = EqualsCondition(epsilon, loss, "modified_huber") # eta0 seems to be always active according to the source code; when # learning_rate is set to optimial, eta0 is the starting value: # https://github.com/scikit-learn/scikit-learn/blob/0.15.X/sklearn/linear_model/sgd_fast.pyx #eta0_and_inv = EqualsCondition(eta0, learning_rate, "invscaling") #eta0_and_constant = EqualsCondition(eta0, learning_rate, "constant") #eta0_condition = OrConjunction(eta0_and_inv, eta0_and_constant) power_t_condition = EqualsCondition(power_t, learning_rate, "invscaling") cs.add_condition(elasticnet) cs.add_condition(epsilon_condition) cs.add_condition(power_t_condition) return cs
def get_hyperparameter_search_space(dataset_properties=None): n_neighbors = UniformIntegerHyperparameter(name="n_neighbors", lower=1, upper=100, default=1) weights = CategoricalHyperparameter(name="weights", choices=["uniform", "distance"], default="uniform") metric = UnParametrizedHyperparameter(name="metric", value="minkowski") algorithm = Constant(name='algorithm', value="auto") p = CategoricalHyperparameter(name="p", choices=[1, 2, 5], default=2) leaf_size = Constant(name="leaf_size", value=30) # Unparametrized # TODO: If we further parametrize 'metric' we need more metric params metric = UnParametrizedHyperparameter(name="metric", value="minkowski") cs = ConfigurationSpace() cs.add_hyperparameter(n_neighbors) cs.add_hyperparameter(weights) cs.add_hyperparameter(metric) cs.add_hyperparameter(algorithm) cs.add_hyperparameter(p) cs.add_hyperparameter(leaf_size) # Conditions metric_p = EqualsCondition(parent=metric, child=p, value="minkowski") cs.add_condition(metric_p) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() loss = cs.add_hyperparameter(CategoricalHyperparameter( "loss", ["ls", "lad", "huber", "quantile"], default="ls")) learning_rate = cs.add_hyperparameter(UniformFloatHyperparameter( name="learning_rate", lower=0.0001, upper=1, default=0.1, log=True)) n_estimators = cs.add_hyperparameter(Constant("n_estimators", 100)) max_depth = cs.add_hyperparameter(UniformIntegerHyperparameter( name="max_depth", lower=1, upper=10, default=3)) min_samples_split = cs.add_hyperparameter(UniformIntegerHyperparameter( name="min_samples_split", lower=2, upper=20, default=2, log=False)) min_samples_leaf = cs.add_hyperparameter(UniformIntegerHyperparameter( name="min_samples_leaf", lower=1, upper=20, default=1, log=False)) min_weight_fraction_leaf = cs.add_hyperparameter( UnParametrizedHyperparameter("min_weight_fraction_leaf", 0.)) subsample = cs.add_hyperparameter(UniformFloatHyperparameter( name="subsample", lower=0.01, upper=1.0, default=1.0, log=False)) max_features = cs.add_hyperparameter(UniformFloatHyperparameter( "max_features", 0.5, 5, default=1)) max_leaf_nodes = cs.add_hyperparameter(UnParametrizedHyperparameter( name="max_leaf_nodes", value="None")) alpha = cs.add_hyperparameter(UniformFloatHyperparameter( "alpha", lower=0.75, upper=0.99, default=0.9)) cs.add_condition(InCondition(alpha, loss, ['huber', 'quantile'])) return cs
def get_hyperparameter_search_space(dataset_properties=None): C = UniformFloatHyperparameter("C", 0.03125, 32768, log=True, default=1.0) # No linear kernel here, because we have liblinear kernel = CategoricalHyperparameter(name="kernel", choices=["rbf", "poly", "sigmoid"], default="rbf") degree = UniformIntegerHyperparameter("degree", 1, 5, default=3) gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8, log=True, default=0.1) # TODO this is totally ad-hoc coef0 = UniformFloatHyperparameter("coef0", -1, 1, default=0) # probability is no hyperparameter, but an argument to the SVM algo shrinking = CategoricalHyperparameter("shrinking", ["True", "False"], default="True") tol = UniformFloatHyperparameter("tol", 1e-5, 1e-1, default=1e-4, log=True) # cache size is not a hyperparameter, but an argument to the program! max_iter = UnParametrizedHyperparameter("max_iter", -1) cs = ConfigurationSpace() cs.add_hyperparameter(C) cs.add_hyperparameter(kernel) cs.add_hyperparameter(degree) cs.add_hyperparameter(gamma) cs.add_hyperparameter(coef0) cs.add_hyperparameter(shrinking) cs.add_hyperparameter(tol) cs.add_hyperparameter(max_iter) degree_depends_on_poly = EqualsCondition(degree, kernel, "poly") coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"]) cs.add_condition(degree_depends_on_poly) cs.add_condition(coef0_condition) return cs
def test_hyperparameters_with_valid_condition(self): cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("parent", [0, 1]) cs.add_hyperparameter(hp1) hp2 = UniformIntegerHyperparameter("child", 0, 10) cs.add_hyperparameter(hp2) cond = EqualsCondition(hp2, hp1, 0) cs.add_condition(cond) self.assertEqual(len(cs._hyperparameters), 2)
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() use_minimum_fraction = cs.add_hyperparameter(CategoricalHyperparameter( "use_minimum_fraction", ["True", "False"], default="True")) minimum_fraction = cs.add_hyperparameter(UniformFloatHyperparameter( "minimum_fraction", lower=.0001, upper=0.5, default=0.01, log=True)) cs.add_condition(EqualsCondition(minimum_fraction, use_minimum_fraction, 'True')) return cs
def test_get_conditions(self): cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("parent", [0, 1]) cs.add_hyperparameter(hp1) hp2 = UniformIntegerHyperparameter("child", 0, 10) cs.add_hyperparameter(hp2) self.assertEqual([], cs.get_conditions()) cond1 = EqualsCondition(hp2, hp1, 0) cs.add_condition(cond1) self.assertEqual([cond1], cs.get_conditions())
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() shrinkage = cs.add_hyperparameter( CategoricalHyperparameter("shrinkage", ["None", "auto", "manual"], default="None") ) shrinkage_factor = cs.add_hyperparameter(UniformFloatHyperparameter("shrinkage_factor", 0.0, 1.0, 0.5)) n_components = cs.add_hyperparameter(UniformIntegerHyperparameter("n_components", 1, 250, default=10)) tol = cs.add_hyperparameter(UniformFloatHyperparameter("tol", 1e-5, 1e-1, default=1e-4, log=True)) cs.add_condition(EqualsCondition(shrinkage_factor, shrinkage, "manual")) return cs
def test_condition_with_cycles(self): cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("parent", [0, 1]) cs.add_hyperparameter(hp1) hp2 = UniformIntegerHyperparameter("child", 0, 10) cs.add_hyperparameter(hp2) cond1 = EqualsCondition(hp2, hp1, 0) cs.add_condition(cond1) cond2 = EqualsCondition(hp1, hp2, 0) self.assertRaisesRegexp(ValueError, "Hyperparameter configuration " "contains a cycle \[\['child', 'parent'\]\]", cs.add_condition, cond2)
def test_get_hyperparameters(self): cs = ConfigurationSpace() self.assertEqual(0, len(cs.get_hyperparameters())) hp1 = CategoricalHyperparameter("parent", [0, 1]) cs.add_hyperparameter(hp1) self.assertEqual([hp1], cs.get_hyperparameters()) hp2 = UniformIntegerHyperparameter("child", 0, 10) cs.add_hyperparameter(hp2) cond1 = EqualsCondition(hp2, hp1, 1) cs.add_condition(cond1) self.assertEqual([hp1, hp2], cs.get_hyperparameters()) # TODO: I need more tests for the topological sort! self.assertEqual([hp1, hp2], cs.get_hyperparameters())
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() shrinkage = cs.add_hyperparameter(CategoricalHyperparameter( "shrinkage", ["None", "auto", "manual"], default="None")) shrinkage_factor = cs.add_hyperparameter(UniformFloatHyperparameter( "shrinkage_factor", 0., 1., 0.5)) n_components = cs.add_hyperparameter(UniformIntegerHyperparameter( 'n_components', 1, 250, default=10)) tol = cs.add_hyperparameter(UniformFloatHyperparameter( "tol", 1e-5, 1e-1, default=1e-4, log=True)) cs.add_condition(EqualsCondition(shrinkage_factor, shrinkage, "manual")) return cs
def test_condition_with_cycles(self): cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("parent", [0, 1]) cs.add_hyperparameter(hp1) hp2 = UniformIntegerHyperparameter("child", 0, 10) cs.add_hyperparameter(hp2) cond1 = EqualsCondition(hp2, hp1, 0) cs.add_condition(cond1) cond2 = EqualsCondition(hp1, hp2, 0) self.assertRaisesRegexp( ValueError, "Hyperparameter configuration " "contains a cycle \[\['child', 'parent'\]\]", cs.add_condition, cond2)
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() loss = cs.add_hyperparameter( CategoricalHyperparameter("loss", ["ls", "lad", "huber", "quantile"], default="ls")) learning_rate = cs.add_hyperparameter( UniformFloatHyperparameter(name="learning_rate", lower=0.0001, upper=1, default=0.1, log=True)) n_estimators = cs.add_hyperparameter(Constant("n_estimators", 100)) max_depth = cs.add_hyperparameter( UniformIntegerHyperparameter(name="max_depth", lower=1, upper=10, default=3)) min_samples_split = cs.add_hyperparameter( UniformIntegerHyperparameter(name="min_samples_split", lower=2, upper=20, default=2, log=False)) min_samples_leaf = cs.add_hyperparameter( UniformIntegerHyperparameter(name="min_samples_leaf", lower=1, upper=20, default=1, log=False)) min_weight_fraction_leaf = cs.add_hyperparameter( UnParametrizedHyperparameter("min_weight_fraction_leaf", 0.)) subsample = cs.add_hyperparameter( UniformFloatHyperparameter(name="subsample", lower=0.01, upper=1.0, default=1.0, log=False)) max_features = cs.add_hyperparameter( UniformFloatHyperparameter("max_features", 0.5, 5, default=1)) max_leaf_nodes = cs.add_hyperparameter( UnParametrizedHyperparameter(name="max_leaf_nodes", value="None")) alpha = cs.add_hyperparameter( UniformFloatHyperparameter("alpha", lower=0.75, upper=0.99, default=0.9)) cs.add_condition(InCondition(alpha, loss, ['huber', 'quantile'])) return cs
def get_hyperparameter_search_space(dataset_properties=None): # Copied from libsvm_c C = UniformFloatHyperparameter( name="C", lower=0.03125, upper=32768, log=True, default=1.0) kernel = CategoricalHyperparameter( name="kernel", choices=['linear', 'poly', 'rbf', 'sigmoid'], default="rbf") degree = UniformIntegerHyperparameter( name="degree", lower=1, upper=5, default=3) # Changed the gamma value to 0.0 (is 0.1 for classification) gamma = UniformFloatHyperparameter( name="gamma", lower=3.0517578125e-05, upper=8, log=True, default=0.1) # TODO this is totally ad-hoc coef0 = UniformFloatHyperparameter( name="coef0", lower=-1, upper=1, default=0) # probability is no hyperparameter, but an argument to the SVM algo shrinking = CategoricalHyperparameter( name="shrinking", choices=["True", "False"], default="True") tol = UniformFloatHyperparameter( name="tol", lower=1e-5, upper=1e-1, default=1e-3, log=True) max_iter = UnParametrizedHyperparameter("max_iter", -1) # Random Guess epsilon = UniformFloatHyperparameter(name="epsilon", lower=0.001, upper=1, default=0.1, log=True) cs = ConfigurationSpace() cs.add_hyperparameter(C) cs.add_hyperparameter(kernel) cs.add_hyperparameter(degree) cs.add_hyperparameter(gamma) cs.add_hyperparameter(coef0) cs.add_hyperparameter(shrinking) cs.add_hyperparameter(tol) cs.add_hyperparameter(max_iter) cs.add_hyperparameter(epsilon) degree_depends_on_kernel = InCondition(child=degree, parent=kernel, values=('poly', 'rbf', 'sigmoid')) gamma_depends_on_kernel = InCondition(child=gamma, parent=kernel, values=('poly', 'rbf')) coef0_depends_on_kernel = InCondition(child=coef0, parent=kernel, values=('poly', 'sigmoid')) cs.add_condition(degree_depends_on_kernel) cs.add_condition(gamma_depends_on_kernel) cs.add_condition(coef0_depends_on_kernel) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() n_components = cs.add_hyperparameter(UniformIntegerHyperparameter( "n_components", 10, 2000, default=100)) algorithm = cs.add_hyperparameter(CategoricalHyperparameter('algorithm', ['parallel', 'deflation'], 'parallel')) whiten = cs.add_hyperparameter(CategoricalHyperparameter('whiten', ['False', 'True'], 'False')) fun = cs.add_hyperparameter(CategoricalHyperparameter( 'fun', ['logcosh', 'exp', 'cube'], 'logcosh')) cs.add_condition(EqualsCondition(n_components, whiten, "True")) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() use_minimum_fraction = cs.add_hyperparameter( CategoricalHyperparameter("use_minimum_fraction", ["True", "False"], default="True")) minimum_fraction = cs.add_hyperparameter( UniformFloatHyperparameter("minimum_fraction", lower=.0001, upper=0.5, default=0.01, log=True)) cs.add_condition( EqualsCondition(minimum_fraction, use_minimum_fraction, 'True')) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() loss = cs.add_hyperparameter(CategoricalHyperparameter("loss", ["squared_loss", "huber", "epsilon_insensitive", "squared_epsilon_insensitive"], default="squared_loss")) penalty = cs.add_hyperparameter(CategoricalHyperparameter( "penalty", ["l1", "l2", "elasticnet"], default="l2")) alpha = cs.add_hyperparameter(UniformFloatHyperparameter( "alpha", 10e-7, 1e-1, log=True, default=0.01)) l1_ratio = cs.add_hyperparameter(UniformFloatHyperparameter( "l1_ratio", 1e-9, 1., log=True, default=0.15)) fit_intercept = cs.add_hyperparameter(UnParametrizedHyperparameter( "fit_intercept", "True")) n_iter = cs.add_hyperparameter(UniformIntegerHyperparameter( "n_iter", 5, 1000, log=True, default=20)) epsilon = cs.add_hyperparameter(UniformFloatHyperparameter( "epsilon", 1e-5, 1e-1, default=1e-4, log=True)) learning_rate = cs.add_hyperparameter(CategoricalHyperparameter( "learning_rate", ["optimal", "invscaling", "constant"], default="optimal")) eta0 = cs.add_hyperparameter(UniformFloatHyperparameter( "eta0", 10 ** -7, 0.1, default=0.01)) power_t = cs.add_hyperparameter(UniformFloatHyperparameter( "power_t", 1e-5, 1, default=0.5)) average = cs.add_hyperparameter(CategoricalHyperparameter( "average", ["False", "True"], default="False")) # TODO add passive/aggressive here, although not properly documented? elasticnet = EqualsCondition(l1_ratio, penalty, "elasticnet") epsilon_condition = InCondition(epsilon, loss, ["huber", "epsilon_insensitive", "squared_epsilon_insensitive"]) # eta0 seems to be always active according to the source code; when # learning_rate is set to optimial, eta0 is the starting value: # https://github.com/scikit-learn/scikit-learn/blob/0.15.X/sklearn/linear_model/sgd_fast.pyx # eta0_and_inv = EqualsCondition(eta0, learning_rate, "invscaling") #eta0_and_constant = EqualsCondition(eta0, learning_rate, "constant") #eta0_condition = OrConjunction(eta0_and_inv, eta0_and_constant) power_t_condition = EqualsCondition(power_t, learning_rate, "invscaling") cs.add_condition(elasticnet) cs.add_condition(epsilon_condition) cs.add_condition(power_t_condition) return cs
def get_hyperparameter_search_space(dataset_properties=None): if dataset_properties is not None and \ (dataset_properties.get("sparse") is True or dataset_properties.get("signed") is False): allow_chi2 = False else: allow_chi2 = True possible_kernels = ['poly', 'rbf', 'sigmoid', 'cosine'] if allow_chi2: possible_kernels.append("chi2") kernel = CategoricalHyperparameter('kernel', possible_kernels, 'rbf') degree = UniformIntegerHyperparameter('degree', 2, 5, 3) gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8, log=True, default=0.1) coef0 = UniformFloatHyperparameter("coef0", -1, 1, default=0) n_components = UniformIntegerHyperparameter("n_components", 50, 10000, default=100, log=True) cs = ConfigurationSpace() cs.add_hyperparameter(kernel) cs.add_hyperparameter(degree) cs.add_hyperparameter(gamma) cs.add_hyperparameter(coef0) cs.add_hyperparameter(n_components) degree_depends_on_poly = EqualsCondition(degree, kernel, "poly") coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"]) gamma_kernels = ["poly", "rbf", "sigmoid"] if allow_chi2: gamma_kernels.append("chi2") gamma_condition = InCondition(gamma, kernel, gamma_kernels) cs.add_condition(degree_depends_on_poly) cs.add_condition(coef0_condition) cs.add_condition(gamma_condition) return cs
def test_add_second_condition_wo_conjunction(self): hp1 = CategoricalHyperparameter("input1", [0, 1]) hp2 = CategoricalHyperparameter("input2", [0, 1]) hp3 = Constant("And", "True") cond1 = EqualsCondition(hp3, hp1, 1) cond2 = EqualsCondition(hp3, hp2, 1) cs = ConfigurationSpace() cs.add_hyperparameter(hp1) cs.add_hyperparameter(hp2) cs.add_hyperparameter(hp3) cs.add_condition(cond1) self.assertRaisesRegexp( ValueError, "Adding a second condition \(different\) for a " "hyperparameter is ambigouos and " "therefore forbidden. Add a conjunction " "instead!", cs.add_condition, cond2)
def test_get_parent_and_children_of(self): cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("parent", [0, 1]) cs.add_hyperparameter(hp1) hp2 = UniformIntegerHyperparameter("child", 0, 10) cs.add_hyperparameter(hp2) cond1 = EqualsCondition(hp2, hp1, 0) cs.add_condition(cond1) self.assertEqual([hp1], cs.get_parents_of(hp2.name)) self.assertEqual([hp1], cs.get_parents_of(hp2)) self.assertEqual([hp2], cs.get_children_of(hp1.name)) self.assertEqual([hp2], cs.get_children_of(hp1)) self.assertRaisesRegexp( KeyError, "Hyperparameter 'Foo' does not exist in this " "configuration space.", cs.get_parents_of, "Foo") self.assertRaisesRegexp( KeyError, "Hyperparameter 'Foo' does not exist in this " "configuration space.", cs.get_children_of, "Foo")
def test_repr(self): cs1 = ConfigurationSpace() retval = cs1.__str__() self.assertEqual("Configuration space object:\n Hyperparameters:\n", retval) hp1 = CategoricalHyperparameter("parent", [0, 1]) cs1.add_hyperparameter(hp1) retval = cs1.__str__() self.assertEqual("Configuration space object:\n Hyperparameters:\n" " %s\n" % str(hp1), retval) hp2 = UniformIntegerHyperparameter("child", 0, 10) cond1 = EqualsCondition(hp2, hp1, 0) cs1.add_hyperparameter(hp2) cs1.add_condition(cond1) retval = cs1.__str__() self.assertEqual("Configuration space object:\n Hyperparameters:\n" " %s\n %s\n Conditions:\n %s\n" % (str(hp2), str(hp1), str(cond1)), retval)
def test_add_conjunction(self): hp1 = CategoricalHyperparameter("input1", [0, 1]) hp2 = CategoricalHyperparameter("input2", [0, 1]) hp3 = CategoricalHyperparameter("input3", [0, 1]) hp4 = Constant("And", "True") cond1 = EqualsCondition(hp4, hp1, 1) cond2 = EqualsCondition(hp4, hp2, 1) cond3 = EqualsCondition(hp4, hp3, 1) andconj1 = AndConjunction(cond1, cond2, cond3) cs = ConfigurationSpace() cs.add_hyperparameter(hp1) cs.add_hyperparameter(hp2) cs.add_hyperparameter(hp3) cs.add_hyperparameter(hp4) cs.add_condition(andconj1) self.assertNotIn(hp4, cs.get_all_uncoditional_hyperparameters())
def test_add_second_condition_wo_conjunction(self): hp1 = CategoricalHyperparameter("input1", [0, 1]) hp2 = CategoricalHyperparameter("input2", [0, 1]) hp3 = Constant("And", "True") cond1 = EqualsCondition(hp3, hp1, 1) cond2 = EqualsCondition(hp3, hp2, 1) cs = ConfigurationSpace() cs.add_hyperparameter(hp1) cs.add_hyperparameter(hp2) cs.add_hyperparameter(hp3) cs.add_condition(cond1) self.assertRaisesRegexp(ValueError, "Adding a second condition \(different\) for a " "hyperparameter is ambigouos and " "therefore forbidden. Add a conjunction " "instead!", cs.add_condition, cond2)
def test_repr(self): cs1 = ConfigurationSpace() retval = cs1.__str__() self.assertEqual("Configuration space object:\n Hyperparameters:\n", retval) hp1 = CategoricalHyperparameter("parent", [0, 1]) cs1.add_hyperparameter(hp1) retval = cs1.__str__() self.assertEqual( "Configuration space object:\n Hyperparameters:\n" " %s\n" % str(hp1), retval) hp2 = UniformIntegerHyperparameter("child", 0, 10) cond1 = EqualsCondition(hp2, hp1, 0) cs1.add_hyperparameter(hp2) cs1.add_condition(cond1) retval = cs1.__str__() self.assertEqual( "Configuration space object:\n Hyperparameters:\n" " %s\n %s\n Conditions:\n %s\n" % (str(hp2), str(hp1), str(cond1)), retval)
def test_check_configuration2(self): # Test that hyperparameters which are not active must not be set and # that evaluating forbidden clauses does not choke on missing # hyperparameters cs = ConfigurationSpace() classifier = CategoricalHyperparameter("classifier", ["k_nearest_neighbors", "extra_trees"]) metric = CategoricalHyperparameter("metric", ["minkowski", "other"]) p = CategoricalHyperparameter("k_nearest_neighbors:p", [1, 2]) metric_depends_on_classifier = EqualsCondition(metric, classifier, "k_nearest_neighbors") p_depends_on_metric = EqualsCondition(p, metric, "minkowski") cs.add_hyperparameter(metric) cs.add_hyperparameter(p) cs.add_hyperparameter(classifier) cs.add_condition(metric_depends_on_classifier) cs.add_condition(p_depends_on_metric) forbidden = ForbiddenEqualsClause(metric, "other") cs.add_forbidden_clause(forbidden) configuration = Configuration(cs, dict(classifier="extra_trees"))
def test_get_parent_and_children_of(self): cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("parent", [0, 1]) cs.add_hyperparameter(hp1) hp2 = UniformIntegerHyperparameter("child", 0, 10) cs.add_hyperparameter(hp2) cond1 = EqualsCondition(hp2, hp1, 0) cs.add_condition(cond1) self.assertEqual([hp1], cs.get_parents_of(hp2.name)) self.assertEqual([hp1], cs.get_parents_of(hp2)) self.assertEqual([hp2], cs.get_children_of(hp1.name)) self.assertEqual([hp2], cs.get_children_of(hp1)) self.assertRaisesRegexp(KeyError, "Hyperparameter 'Foo' does not exist in this " "configuration space.", cs.get_parents_of, "Foo") self.assertRaisesRegexp(KeyError, "Hyperparameter 'Foo' does not exist in this " "configuration space.", cs.get_children_of, "Foo")
def test_check_configuration2(self): # Test that hyperparameters which are not active must not be set and # that evaluating forbidden clauses does not choke on missing # hyperparameters cs = ConfigurationSpace() classifier = CategoricalHyperparameter( "classifier", ["k_nearest_neighbors", "extra_trees"]) metric = CategoricalHyperparameter("metric", ["minkowski", "other"]) p = CategoricalHyperparameter("k_nearest_neighbors:p", [1, 2]) metric_depends_on_classifier = EqualsCondition(metric, classifier, "k_nearest_neighbors") p_depends_on_metric = EqualsCondition(p, metric, "minkowski") cs.add_hyperparameter(metric) cs.add_hyperparameter(p) cs.add_hyperparameter(classifier) cs.add_condition(metric_depends_on_classifier) cs.add_condition(p_depends_on_metric) forbidden = ForbiddenEqualsClause(metric, "other") cs.add_forbidden_clause(forbidden) configuration = Configuration(cs, dict(classifier="extra_trees"))
def get_hyperparameter_search_space(dataset_properties=None): n_components = UniformIntegerHyperparameter( "n_components", 10, 2000, default=100) kernel = CategoricalHyperparameter('kernel', ['poly', 'rbf', 'sigmoid', 'cosine'], 'rbf') degree = UniformIntegerHyperparameter('degree', 2, 5, 3) gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8, log=True, default=1.0) coef0 = UniformFloatHyperparameter("coef0", -1, 1, default=0) cs = ConfigurationSpace() cs.add_hyperparameter(n_components) cs.add_hyperparameter(kernel) cs.add_hyperparameter(degree) cs.add_hyperparameter(gamma) cs.add_hyperparameter(coef0) degree_depends_on_poly = EqualsCondition(degree, kernel, "poly") coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"]) gamma_condition = InCondition(gamma, kernel, ["poly", "rbf"]) cs.add_condition(degree_depends_on_poly) cs.add_condition(coef0_condition) cs.add_condition(gamma_condition) return cs
def test_eq(self): # Compare empty configuration spaces cs1 = ConfigurationSpace() cs2 = ConfigurationSpace() self.assertEqual(cs1, cs2) # Compare to something which isn't a configuration space self.assertTrue(not (cs1 == "ConfigurationSpace")) # Compare to equal configuration spaces hp1 = CategoricalHyperparameter("parent", [0, 1]) hp2 = UniformIntegerHyperparameter("child", 0, 10) hp3 = UniformIntegerHyperparameter("friend", 0, 5) cond1 = EqualsCondition(hp2, hp1, 0) cs1.add_hyperparameter(hp1) cs1.add_hyperparameter(hp2) cs1.add_condition(cond1) cs2.add_hyperparameter(hp1) cs2.add_hyperparameter(hp2) cs2.add_condition(cond1) self.assertEqual(cs1, cs2) cs1.add_hyperparameter(hp3) self.assertFalse(cs1 == cs2)
def get_hyperparameter_search_space(dataset_properties=None): if dataset_properties is not None and \ (dataset_properties.get("sparse") is True or dataset_properties.get("signed") is False): allow_chi2 = False else: allow_chi2 = True possible_kernels = ['poly', 'rbf', 'sigmoid', 'cosine'] if allow_chi2: possible_kernels.append("chi2") kernel = CategoricalHyperparameter('kernel', possible_kernels, 'rbf') degree = UniformIntegerHyperparameter('degree', 2, 5, 3) gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8, log=True, default=0.1) coef0 = UniformFloatHyperparameter("coef0", -1, 1, default=0) n_components = UniformIntegerHyperparameter( "n_components", 50, 10000, default=100, log=True) cs = ConfigurationSpace() cs.add_hyperparameter(kernel) cs.add_hyperparameter(degree) cs.add_hyperparameter(gamma) cs.add_hyperparameter(coef0) cs.add_hyperparameter(n_components) degree_depends_on_poly = EqualsCondition(degree, kernel, "poly") coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"]) gamma_kernels = ["poly", "rbf", "sigmoid"] if allow_chi2: gamma_kernels.append("chi2") gamma_condition = InCondition(gamma, kernel, gamma_kernels) cs.add_condition(degree_depends_on_poly) cs.add_condition(coef0_condition) cs.add_condition(gamma_condition) return cs
def test_add_configuration_space(self): cs = ConfigurationSpace() hp1 = cs.add_hyperparameter(CategoricalHyperparameter("input1", [0, 1])) forb1 = cs.add_forbidden_clause(ForbiddenEqualsClause(hp1, 1)) hp2 = cs.add_hyperparameter(UniformIntegerHyperparameter("child", 0, 10)) cond = cs.add_condition(EqualsCondition(hp2, hp1, 0)) cs2 = ConfigurationSpace() cs2.add_configuration_space('prefix', cs, delimiter='__') self.assertEqual(str(cs2), '''Configuration space object: Hyperparameters: prefix__child, Type: UniformInteger, Range: [0, 10], Default: 5 prefix__input1, Type: Categorical, Choices: {0, 1}, Default: 0 Conditions: prefix__child | prefix__input1 == 0 Forbidden Clauses: Forbidden: prefix__input1 == 1 ''')
def test_sample_configuration(self): cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("parent", [0, 1]) cs.add_hyperparameter(hp1) hp2 = UniformIntegerHyperparameter("child", 0, 10) cs.add_hyperparameter(hp2) cond1 = EqualsCondition(hp2, hp1, 0) cs.add_condition(cond1) # This automatically checks the configuration! Configuration(cs, dict(parent=0, child=5)) # and now for something more complicated cs = ConfigurationSpace(seed=1) hp1 = CategoricalHyperparameter("input1", [0, 1]) cs.add_hyperparameter(hp1) hp2 = CategoricalHyperparameter("input2", [0, 1]) cs.add_hyperparameter(hp2) hp3 = CategoricalHyperparameter("input3", [0, 1]) cs.add_hyperparameter(hp3) hp4 = CategoricalHyperparameter("input4", [0, 1]) cs.add_hyperparameter(hp4) hp5 = CategoricalHyperparameter("input5", [0, 1]) cs.add_hyperparameter(hp5) hp6 = Constant("AND", "True") cs.add_hyperparameter(hp6) cond1 = EqualsCondition(hp6, hp1, 1) cond2 = NotEqualsCondition(hp6, hp2, 1) cond3 = InCondition(hp6, hp3, [1]) cond4 = EqualsCondition(hp5, hp3, 1) cond5 = EqualsCondition(hp4, hp5, 1) cond6 = EqualsCondition(hp6, hp4, 1) cond7 = EqualsCondition(hp6, hp5, 1) conj1 = AndConjunction(cond1, cond2) conj2 = OrConjunction(conj1, cond3) conj3 = AndConjunction(conj2, cond6, cond7) cs.add_condition(cond4) cs.add_condition(cond5) cs.add_condition(conj3) samples = [] for i in range(5): cs.seed(1) samples.append([]) for j in range(100): sample = cs.sample_configuration() samples[-1].append(sample) if i > 0: for j in range(100): self.assertEqual(samples[-1][j], samples[-2][j])
def test_get_hyperparameters_topological_sort(self): for iteration in range(10): cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("parent", [0, 1]) cs.add_hyperparameter(hp1) hp2 = UniformIntegerHyperparameter("child", 0, 10) cs.add_hyperparameter(hp2) cond1 = EqualsCondition(hp2, hp1, 0) cs.add_condition(cond1) # This automatically checks the configuration! Configuration(cs, dict(parent=0, child=5)) # and now for something more complicated cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("input1", [0, 1]) hp2 = CategoricalHyperparameter("input2", [0, 1]) hp3 = CategoricalHyperparameter("input3", [0, 1]) hp4 = CategoricalHyperparameter("input4", [0, 1]) hp5 = CategoricalHyperparameter("input5", [0, 1]) hp6 = Constant("AND", "True") # More top-level hyperparameters hp7 = CategoricalHyperparameter("input7", [0, 1]) hps = [hp1, hp2, hp3, hp4, hp5, hp6, hp7] random.shuffle(hps) for hp in hps: cs.add_hyperparameter(hp) cond1 = EqualsCondition(hp6, hp1, 1) cond2 = NotEqualsCondition(hp6, hp2, 1) cond3 = InCondition(hp6, hp3, [1]) cond4 = EqualsCondition(hp5, hp3, 1) cond5 = EqualsCondition(hp4, hp5, 1) cond6 = EqualsCondition(hp6, hp4, 1) cond7 = EqualsCondition(hp6, hp5, 1) conj1 = AndConjunction(cond1, cond2) conj2 = OrConjunction(conj1, cond3) conj3 = AndConjunction(conj2, cond6, cond7) cs.add_condition(cond4) cs.add_condition(cond5) cs.add_condition(conj3) hps = cs.get_hyperparameters() self.assertEqual(hps.index(hp1), 0) self.assertEqual(hps.index(hp2), 1) self.assertEqual(hps.index(hp3), 2) self.assertEqual(hps.index(hp7), 3) self.assertEqual(hps.index(hp5), 4) self.assertEqual(hps.index(hp4), 5) self.assertEqual(hps.index(hp6), 6)
def test_add_configuration_space(self): cs = ConfigurationSpace() hp1 = cs.add_hyperparameter(CategoricalHyperparameter( "input1", [0, 1])) forb1 = cs.add_forbidden_clause(ForbiddenEqualsClause(hp1, 1)) hp2 = cs.add_hyperparameter( UniformIntegerHyperparameter("child", 0, 10)) cond = cs.add_condition(EqualsCondition(hp2, hp1, 0)) cs2 = ConfigurationSpace() cs2.add_configuration_space('prefix', cs, delimiter='__') self.assertEqual( str(cs2), '''Configuration space object: Hyperparameters: prefix__child, Type: UniformInteger, Range: [0, 10], Default: 5 prefix__input1, Type: Categorical, Choices: {0, 1}, Default: 0 Conditions: prefix__child | prefix__input1 == 0 Forbidden Clauses: Forbidden: prefix__input1 == 1 ''')
degree_condition = InCondition(degree, kernel, ["poly", "sigmoid"]) neurons = UniformIntegerHyperparameter("neurons", 16, 1024) neurons_condition = EqualsCondition(neurons, classifier, "nn") lr = UniformFloatHyperparameter("lr", 0.0001, 1.0) lr_condition = EqualsCondition(lr, classifier, "nn") preprocessing = CategoricalHyperparameter("preprocessing", ["None", "pca"]) conditional_space = ConfigurationSpace() conditional_space.add_hyperparameter(classifier) conditional_space.add_hyperparameter(kernel) conditional_space.add_hyperparameter(C) conditional_space.add_hyperparameter(gamma) conditional_space.add_hyperparameter(degree) conditional_space.add_hyperparameter(neurons) conditional_space.add_hyperparameter(lr) conditional_space.add_hyperparameter(preprocessing) conditional_space.add_condition(kernel_condition) conditional_space.add_condition(C_condition) conditional_space.add_condition(gamma_condition) conditional_space.add_condition(degree_condition) conditional_space.add_condition(neurons_condition) conditional_space.add_condition(lr_condition) float_a = UniformFloatHyperparameter("float_a", -1.23, 6.45) e_float_a = UniformFloatHyperparameter("e_float_a", .5E-2, 4.5e+06) int_a = UniformIntegerHyperparameter("int_a", -1, 6) log_a = UniformFloatHyperparameter("log_a", 4e-1, 6.45, log=True) int_log_a = UniformIntegerHyperparameter("int_log_a", 1, 6, log=True) cat_a = CategoricalHyperparameter("cat_a", ["a", "b", "c", "d"]) crazy = CategoricalHyperparameter("@.:;/\?!$%&_-<>*+1234567890", ["const"]) easy_space = ConfigurationSpace() easy_space.add_hyperparameter(float_a)
def get_hyperparameter_search_space(cls, dataset_properties, default=None, include=None, exclude=None): cs = ConfigurationSpace() # Compile a list of legal preprocessors for this problem available_preprocessors = cls.get_available_components( data_prop=dataset_properties, include=include, exclude=exclude) if len(available_preprocessors) == 0: raise ValueError( "No preprocessors found, please add NoPreprocessing") if default is None: defaults = ['no_preprocessing', 'select_percentile', 'pca', 'truncatedSVD'] for default_ in defaults: if default_ in available_preprocessors: default = default_ break preprocessor = CategoricalHyperparameter('__choice__', list( available_preprocessors.keys()), default=default) cs.add_hyperparameter(preprocessor) for name in available_preprocessors: preprocessor_configuration_space = available_preprocessors[name]. \ get_hyperparameter_search_space(dataset_properties) for parameter in preprocessor_configuration_space.get_hyperparameters(): new_parameter = copy.deepcopy(parameter) new_parameter.name = "%s:%s" % (name, new_parameter.name) cs.add_hyperparameter(new_parameter) # We must only add a condition if the hyperparameter is not # conditional on something else if len(preprocessor_configuration_space. get_parents_of(parameter)) == 0: condition = EqualsCondition(new_parameter, preprocessor, name) cs.add_condition(condition) for condition in available_preprocessors[name]. \ get_hyperparameter_search_space( dataset_properties).get_conditions(): if not isinstance(condition, AbstractConjunction): dlcs = [condition] else: dlcs = condition.get_descendent_literal_conditions() for dlc in dlcs: if not dlc.child.name.startswith(name): dlc.child.name = "%s:%s" % (name, dlc.child.name) if not dlc.parent.name.startswith(name): dlc.parent.name = "%s:%s" % (name, dlc.parent.name) cs.add_condition(condition) for forbidden_clause in available_preprocessors[name]. \ get_hyperparameter_search_space( dataset_properties).forbidden_clauses: dlcs = forbidden_clause.get_descendant_literal_clauses() for dlc in dlcs: if not dlc.hyperparameter.name.startswith(name): dlc.hyperparameter.name = "%s:%s" % (name, dlc.hyperparameter.name) cs.add_forbidden_clause(forbidden_clause) return cs
def test_check_configuration(self): # TODO this is only a smoke test # TODO actually, this rather tests the evaluate methods in the # conditions module! cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("parent", [0, 1]) cs.add_hyperparameter(hp1) hp2 = UniformIntegerHyperparameter("child", 0, 10) cs.add_hyperparameter(hp2) cond1 = EqualsCondition(hp2, hp1, 0) cs.add_condition(cond1) # This automatically checks the configuration! Configuration(cs, dict(parent=0, child=5)) # and now for something more complicated cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("input1", [0, 1]) cs.add_hyperparameter(hp1) hp2 = CategoricalHyperparameter("input2", [0, 1]) cs.add_hyperparameter(hp2) hp3 = CategoricalHyperparameter("input3", [0, 1]) cs.add_hyperparameter(hp3) hp4 = CategoricalHyperparameter("input4", [0, 1]) cs.add_hyperparameter(hp4) hp5 = CategoricalHyperparameter("input5", [0, 1]) cs.add_hyperparameter(hp5) hp6 = Constant("AND", "True") cs.add_hyperparameter(hp6) cond1 = EqualsCondition(hp6, hp1, 1) cond2 = NotEqualsCondition(hp6, hp2, 1) cond3 = InCondition(hp6, hp3, [1]) cond4 = EqualsCondition(hp6, hp4, 1) cond5 = EqualsCondition(hp6, hp5, 1) conj1 = AndConjunction(cond1, cond2) conj2 = OrConjunction(conj1, cond3) conj3 = AndConjunction(conj2, cond4, cond5) cs.add_condition(conj3) expected_outcomes = [False, False, False, False, False, False, False, True, False, False, False, False, False, False, False, True, False, False, False, True, False, False, False, True, False, False, False, False, False, False, False, True] for idx, values in enumerate(product([0, 1], repeat=5)): # The hyperparameters aren't sorted, but the test assumes them to # be sorted. hyperparameters = sorted(cs.get_hyperparameters(), key=lambda t: t.name) instantiations = {hyperparameters[jdx+1].name: values[jdx] for jdx in range(len(values))} evaluation = conj3.evaluate(instantiations) self.assertEqual(expected_outcomes[idx], evaluation) if evaluation == False: self.assertRaisesRegexp(ValueError, "Inactive hyperparameter 'AND' must " "not be specified, but has the value: " "'True'.", Configuration, cs, values={ "input1": values[0], "input2": values[1], "input3": values[2], "input4": values[3], "input5": values[4], "AND": "True"}) else: Configuration(cs, values={"input1": values[0], "input2": values[1], "input3": values[2], "input4": values[3], "input5": values[4], "AND": "True"})
def get_hyperparameter_search_space(dataset_properties=None): # Copied from libsvm_c C = UniformFloatHyperparameter(name="C", lower=0.03125, upper=32768, log=True, default=1.0) kernel = CategoricalHyperparameter( name="kernel", choices=['linear', 'poly', 'rbf', 'sigmoid'], default="rbf") degree = UniformIntegerHyperparameter(name="degree", lower=1, upper=5, default=3) # Changed the gamma value to 0.0 (is 0.1 for classification) gamma = UniformFloatHyperparameter(name="gamma", lower=3.0517578125e-05, upper=8, log=True, default=0.1) # TODO this is totally ad-hoc coef0 = UniformFloatHyperparameter(name="coef0", lower=-1, upper=1, default=0) # probability is no hyperparameter, but an argument to the SVM algo shrinking = CategoricalHyperparameter(name="shrinking", choices=["True", "False"], default="True") tol = UniformFloatHyperparameter(name="tol", lower=1e-5, upper=1e-1, default=1e-3, log=True) max_iter = UnParametrizedHyperparameter("max_iter", -1) # Random Guess epsilon = UniformFloatHyperparameter(name="epsilon", lower=0.001, upper=1, default=0.1, log=True) cs = ConfigurationSpace() cs.add_hyperparameter(C) cs.add_hyperparameter(kernel) cs.add_hyperparameter(degree) cs.add_hyperparameter(gamma) cs.add_hyperparameter(coef0) cs.add_hyperparameter(shrinking) cs.add_hyperparameter(tol) cs.add_hyperparameter(max_iter) cs.add_hyperparameter(epsilon) degree_depends_on_kernel = InCondition(child=degree, parent=kernel, values=('poly', 'rbf', 'sigmoid')) gamma_depends_on_kernel = InCondition(child=gamma, parent=kernel, values=('poly', 'rbf')) coef0_depends_on_kernel = InCondition(child=coef0, parent=kernel, values=('poly', 'sigmoid')) cs.add_condition(degree_depends_on_kernel) cs.add_condition(gamma_depends_on_kernel) cs.add_condition(coef0_depends_on_kernel) return cs
def get_hyperparameter_search_space(dataset_properties=None): max_num_layers = 7 # Maximum number of layers coded # Hacky way to condition layers params based on the number of layers # 'c'=1, 'd'=2, 'e'=3 ,'f'=4', g ='5', h='6' + output_layer layer_choices = [chr(i) for i in range(ord('c'), ord('b') + max_num_layers)] batch_size = UniformIntegerHyperparameter("batch_size", 32, 4096, log=True, default=32) number_epochs = UniformIntegerHyperparameter("number_epochs", 2, 80, default=5) num_layers = CategoricalHyperparameter("num_layers", choices=layer_choices, default='c') lr = UniformFloatHyperparameter("learning_rate", 1e-6, 1.0, log=True, default=0.01) l2 = UniformFloatHyperparameter("lambda2", 1e-7, 1e-2, log=True, default=1e-4) dropout_output = UniformFloatHyperparameter("dropout_output", 0.0, 0.99, default=0.5) # Define basic hyperparameters and define the config space # basic means that are independent from the number of layers cs = ConfigurationSpace() cs.add_hyperparameter(number_epochs) cs.add_hyperparameter(batch_size) cs.add_hyperparameter(num_layers) cs.add_hyperparameter(lr) cs.add_hyperparameter(l2) cs.add_hyperparameter(dropout_output) # Define parameters with different child parameters and conditions solver_choices = ["adam", "adadelta", "adagrad", "sgd", "momentum", "nesterov", "smorm3s"] solver = CategoricalHyperparameter(name="solver", choices=solver_choices, default="smorm3s") beta1 = UniformFloatHyperparameter("beta1", 1e-4, 0.1, log=True, default=0.1) beta2 = UniformFloatHyperparameter("beta2", 1e-4, 0.1, log=True, default=0.01) rho = UniformFloatHyperparameter("rho", 0.05, 0.99, log=True, default=0.95) momentum = UniformFloatHyperparameter("momentum", 0.3, 0.999, default=0.9) # TODO: Add policy based on this sklearn sgd policy_choices = ['fixed', 'inv', 'exp', 'step'] lr_policy = CategoricalHyperparameter(name="lr_policy", choices=policy_choices, default='fixed') gamma = UniformFloatHyperparameter(name="gamma", lower=1e-3, upper=1e-1, default=1e-2) power = UniformFloatHyperparameter("power", 0.0, 1.0, default=0.5) epoch_step = UniformIntegerHyperparameter("epoch_step", 2, 20, default=5) cs.add_hyperparameter(solver) cs.add_hyperparameter(beta1) cs.add_hyperparameter(beta2) cs.add_hyperparameter(momentum) cs.add_hyperparameter(rho) cs.add_hyperparameter(lr_policy) cs.add_hyperparameter(gamma) cs.add_hyperparameter(power) cs.add_hyperparameter(epoch_step) # Define parameters that are needed it for each layer output_activation_choices = ['softmax', 'sigmoid', 'softplus', 'tanh'] activations_choices = ['sigmoid', 'tanh', 'scaledTanh', 'elu', 'relu', 'leaky', 'linear'] weight_choices = ['constant', 'normal', 'uniform', 'glorot_normal', 'glorot_uniform', 'he_normal', 'he_uniform', 'ortogonal', 'sparse'] # Iterate over parameters that are used in each layer for i in range(1, max_num_layers): layer_units = UniformIntegerHyperparameter("num_units_layer_" + str(i), 64, 4096, log=True, default=128) cs.add_hyperparameter(layer_units) layer_dropout = UniformFloatHyperparameter("dropout_layer_" + str(i), 0.0, 0.99, default=0.5) cs.add_hyperparameter(layer_dropout) weight_initialization = CategoricalHyperparameter('weight_init_' + str(i), choices=weight_choices, default='he_normal') cs.add_hyperparameter(weight_initialization) layer_std = UniformFloatHyperparameter("std_layer_" + str(i), 1e-6, 0.1, log=True, default=0.005) cs.add_hyperparameter(layer_std) layer_activation = CategoricalHyperparameter("activation_layer_" + str(i), choices=activations_choices, default="relu") cs.add_hyperparameter(layer_activation) layer_leakiness = UniformFloatHyperparameter('leakiness_layer_' + str(i), 0.01, 0.99, default=0.3) cs.add_hyperparameter(layer_leakiness) layer_tanh_alpha = UniformFloatHyperparameter('tanh_alpha_layer_' + str(i), 0.5, 1.0, default=2. / 3.) cs.add_hyperparameter(layer_tanh_alpha) layer_tanh_beta = UniformFloatHyperparameter('tanh_beta_layer_' + str(i), 1.1, 3.0, log=True, default=1.7159) cs.add_hyperparameter(layer_tanh_beta) # TODO: Could be in a function in a new module for i in range(2, max_num_layers): # Condition layers parameter on layer choice layer_unit_param = cs.get_hyperparameter("num_units_layer_" + str(i)) layer_cond = InCondition(child=layer_unit_param, parent=num_layers, values=[l for l in layer_choices[i - 1:]]) cs.add_condition(layer_cond) # Condition dropout parameter on layer choice layer_dropout_param = cs.get_hyperparameter("dropout_layer_" + str(i)) layer_cond = InCondition(child=layer_dropout_param, parent=num_layers, values=[l for l in layer_choices[i - 1:]]) cs.add_condition(layer_cond) # Condition weight initialization on layer choice layer_weight_param = cs.get_hyperparameter("weight_init_" + str(i)) layer_cond = InCondition(child=layer_weight_param, parent=num_layers, values=[l for l in layer_choices[i - 1:]]) cs.add_condition(layer_cond) # Condition std parameter on weight layer initialization choice layer_std_param = cs.get_hyperparameter("std_layer_" + str(i)) weight_cond = EqualsCondition(child=layer_std_param, parent=layer_weight_param, value='normal') cs.add_condition(weight_cond) # Condition activation parameter on layer choice layer_activation_param = cs.get_hyperparameter("activation_layer_" + str(i)) layer_cond = InCondition(child=layer_activation_param, parent=num_layers, values=[l for l in layer_choices[i - 1:]]) cs.add_condition(layer_cond) # Condition leakiness on activation choice layer_leakiness_param = cs.get_hyperparameter("leakiness_layer_" + str(i)) activation_cond = EqualsCondition(child=layer_leakiness_param, parent=layer_activation_param, value='leaky') cs.add_condition(activation_cond) # Condition tanh on activation choice layer_tanh_alpha_param = cs.get_hyperparameter("tanh_alpha_layer_" + str(i)) activation_cond = EqualsCondition(child=layer_tanh_alpha_param, parent=layer_activation_param, value='scaledTanh') cs.add_condition(activation_cond) layer_tanh_beta_param = cs.get_hyperparameter("tanh_beta_layer_" + str(i)) activation_cond = EqualsCondition(child=layer_tanh_beta_param, parent=layer_activation_param, value='scaledTanh') cs.add_condition(activation_cond) # Conditioning on solver momentum_depends_on_solver = InCondition(momentum, solver, values=["momentum", "nesterov"]) beta1_depends_on_solver = EqualsCondition(beta1, solver, "adam") beta2_depends_on_solver = EqualsCondition(beta2, solver, "adam") rho_depends_on_solver = EqualsCondition(rho, solver, "adadelta") cs.add_condition(momentum_depends_on_solver) cs.add_condition(beta1_depends_on_solver) cs.add_condition(beta2_depends_on_solver) cs.add_condition(rho_depends_on_solver) # Conditioning on learning rate policy lr_policy_depends_on_solver = InCondition(lr_policy, solver, ["adadelta", "adagrad", "sgd", "momentum", "nesterov"]) gamma_depends_on_policy = InCondition(child=gamma, parent=lr_policy, values=["inv", "exp", "step"]) power_depends_on_policy = EqualsCondition(power, lr_policy, "inv") epoch_step_depends_on_policy = EqualsCondition(epoch_step, lr_policy, "step") cs.add_condition(lr_policy_depends_on_solver) cs.add_condition(gamma_depends_on_policy) cs.add_condition(power_depends_on_policy) cs.add_condition(epoch_step_depends_on_policy) return cs
def get_hyperparameter_search_space(cls, dataset_properties=None, default=None, include=None, exclude=None): cs = ConfigurationSpace() # Compile a list of legal preprocessors for this problem available_preprocessors = cls.get_available_components( data_prop=dataset_properties, include=include, exclude=exclude) if len(available_preprocessors) == 0: raise ValueError("No rescaling algorithm found.") if default is None: defaults = ['min/max', 'standardize', 'none', 'normalize'] for default_ in defaults: if default_ in available_preprocessors: default = default_ break preprocessor = CategoricalHyperparameter( '__choice__', list(available_preprocessors.keys()), default=default) cs.add_hyperparameter(preprocessor) for name in available_preprocessors: preprocessor_configuration_space = available_preprocessors[name]. \ get_hyperparameter_search_space(dataset_properties) for parameter in preprocessor_configuration_space.get_hyperparameters( ): new_parameter = copy.deepcopy(parameter) new_parameter.name = "%s:%s" % (name, new_parameter.name) cs.add_hyperparameter(new_parameter) # We must only add a condition if the hyperparameter is not # conditional on something else if len( preprocessor_configuration_space.get_parents_of( parameter)) == 0: condition = EqualsCondition(new_parameter, preprocessor, name) cs.add_condition(condition) for condition in available_preprocessors[name]. \ get_hyperparameter_search_space( dataset_properties).get_conditions(): if not isinstance(condition, AbstractConjunction): dlcs = [condition] else: dlcs = condition.get_descendent_literal_conditions() for dlc in dlcs: if not dlc.child.name.startswith(name): dlc.child.name = "%s:%s" % (name, dlc.child.name) if not dlc.parent.name.startswith(name): dlc.parent.name = "%s:%s" % (name, dlc.parent.name) cs.add_condition(condition) for forbidden_clause in available_preprocessors[name]. \ get_hyperparameter_search_space( dataset_properties).forbidden_clauses: dlcs = forbidden_clause.get_descendant_literal_clauses() for dlc in dlcs: if not dlc.hyperparameter.name.startswith(name): dlc.hyperparameter.name = "%s:%s" % ( name, dlc.hyperparameter.name) cs.add_forbidden_clause(forbidden_clause) return cs
def get_hyperparameter_search_space(cls, dataset_properties, default=None, include=None, exclude=None): if include is not None and exclude is not None: raise ValueError("The argument include and exclude cannot be used together.") cs = ConfigurationSpace() # Compile a list of all estimator objects for this problem available_estimators = cls.get_available_components( data_prop=dataset_properties, include=include, exclude=exclude) if len(available_estimators) == 0: raise ValueError("No regressors found") if default is None: defaults = ['random_forest', 'support_vector_regression'] + \ list(available_estimators.keys()) for default_ in defaults: if default_ in available_estimators: if include is not None and default_ not in include: continue if exclude is not None and default_ in exclude: continue default = default_ break estimator = CategoricalHyperparameter('__choice__', list(available_estimators.keys()), default=default) cs.add_hyperparameter(estimator) for estimator_name in available_estimators.keys(): # We have to retrieve the configuration space every time because # we change the objects it returns. If we reused it, we could not # retrieve the conditions further down # TODO implement copy for hyperparameters and forbidden and # conditions! estimator_configuration_space = available_estimators[ estimator_name]. \ get_hyperparameter_search_space(dataset_properties) for parameter in estimator_configuration_space.get_hyperparameters(): new_parameter = copy.deepcopy(parameter) new_parameter.name = "%s:%s" % ( estimator_name, new_parameter.name) cs.add_hyperparameter(new_parameter) # We must only add a condition if the hyperparameter is not # conditional on something else if len(estimator_configuration_space. get_parents_of(parameter)) == 0: condition = EqualsCondition(new_parameter, estimator, estimator_name) cs.add_condition(condition) for condition in available_estimators[estimator_name]. \ get_hyperparameter_search_space( dataset_properties).get_conditions(): dlcs = condition.get_descendant_literal_conditions() for dlc in dlcs: if not dlc.child.name.startswith(estimator_name): dlc.child.name = "%s:%s" % ( estimator_name, dlc.child.name) if not dlc.parent.name.startswith(estimator_name): dlc.parent.name = "%s:%s" % ( estimator_name, dlc.parent.name) cs.add_condition(condition) for forbidden_clause in available_estimators[estimator_name]. \ get_hyperparameter_search_space( dataset_properties).forbidden_clauses: dlcs = forbidden_clause.get_descendant_literal_clauses() for dlc in dlcs: if not dlc.hyperparameter.name.startswith(estimator_name): dlc.hyperparameter.name = "%s:%s" % (estimator_name, dlc.hyperparameter.name) cs.add_forbidden_clause(forbidden_clause) return cs
def get_hyperparameter_search_space(cls, estimator_name, default_estimator, estimator_components, preprocessor_components, dataset_properties, always_active): """Return the configuration space for the CASH problem. This method should be called by the method get_hyperparameter_search_space of a subclass. After the subclass assembles a list of available estimators and preprocessor components, _get_hyperparameter_search_space can be called to do the work of creating the actual HPOlibConfigSpace.configuration_space.ConfigurationSpace object. Parameters ---------- estimator_name : str Name of the estimator hyperparameter which will be used in the configuration space. For a classification task, this would be 'classifier'. estimator_components : dict {name: component} Dictionary with all estimator components to be included in the configuration space. preprocessor_components : dict {name: component} Dictionary with all preprocessor components to be included in the configuration space. . always_active : list of str A list of components which will always be active in the pipeline. This is useful for components like imputation which have hyperparameters to be configured, but which do not have any parent. default_estimator : str Default value for the estimator hyperparameter. Returns ------- cs : HPOlibConfigSpace.configuration_space.Configuration The configuration space describing the AutoSklearnClassifier. """ cs = ConfigurationSpace() available_estimators = estimator_components available_preprocessors = preprocessor_components if default_estimator is None: default_estimator = available_estimators.keys()[0] estimator = CategoricalHyperparameter(estimator_name, available_estimators.keys(), default=default_estimator) cs.add_hyperparameter(estimator) for name in available_estimators.keys(): # We have to retrieve the configuration space every time because # we change the objects it returns. If we reused it, we could not # retrieve the conditions further down # TODO implement copy for hyperparameters and forbidden and # conditions! estimator_configuration_space = available_estimators[name]. \ get_hyperparameter_search_space(dataset_properties) for parameter in estimator_configuration_space.get_hyperparameters(): new_parameter = copy.deepcopy(parameter) new_parameter.name = "%s:%s" % (name, new_parameter.name) cs.add_hyperparameter(new_parameter) # We must only add a condition if the hyperparameter is not # conditional on something else if len(estimator_configuration_space. get_parents_of(parameter)) == 0: condition = EqualsCondition(new_parameter, estimator, name) cs.add_condition(condition) for condition in available_estimators[name]. \ get_hyperparameter_search_space(dataset_properties).get_conditions(): dlcs = condition.get_descendant_literal_conditions() for dlc in dlcs: if not dlc.child.name.startswith(name): dlc.child.name = "%s:%s" % (name, dlc.child.name) if not dlc.parent.name.startswith(name): dlc.parent.name = "%s:%s" % (name, dlc.parent.name) cs.add_condition(condition) for forbidden_clause in available_estimators[name]. \ get_hyperparameter_search_space(dataset_properties).forbidden_clauses: dlcs = forbidden_clause.get_descendant_literal_clauses() for dlc in dlcs: if not dlc.hyperparameter.name.startswith(name): dlc.hyperparameter.name = "%s:%s" % (name, dlc.hyperparameter.name) cs.add_forbidden_clause(forbidden_clause) preprocessor_choices = filter(lambda app: app not in always_active, available_preprocessors.keys()) preprocessor = CategoricalHyperparameter("preprocessor", ["None"] + preprocessor_choices, default='None') cs.add_hyperparameter(preprocessor) for name in available_preprocessors.keys(): preprocessor_configuration_space = available_preprocessors[name]. \ get_hyperparameter_search_space(dataset_properties) for parameter in preprocessor_configuration_space.get_hyperparameters(): new_parameter = copy.deepcopy(parameter) new_parameter.name = "%s:%s" % (name, new_parameter.name) cs.add_hyperparameter(new_parameter) # We must only add a condition if the hyperparameter is not # conditional on something else if len(preprocessor_configuration_space. get_parents_of( parameter)) == 0 and name not in always_active: condition = EqualsCondition(new_parameter, preprocessor, name) cs.add_condition(condition) for condition in available_preprocessors[name]. \ get_hyperparameter_search_space(dataset_properties).get_conditions(): dlcs = condition.get_descendent_literal_conditions() for dlc in dlcs: if not dlc.child.name.startswith(name): dlc.child.name = "%s:%s" % (name, dlc.child.name) if not dlc.parent.name.startswith(name): dlc.parent.name = "%s:%s" % (name, dlc.parent.name) cs.add_condition(condition) for forbidden_clause in available_preprocessors[name]. \ get_hyperparameter_search_space(dataset_properties).forbidden_clauses: dlcs = forbidden_clause.get_descendant_literal_clauses() for dlc in dlcs: if not dlc.hyperparameter.startwith(name): dlc.hyperparameter.name = "%s:%s" % (name, dlc.hyperparameter.name) cs.add_forbidden_clause(forbidden_clause) # Now try to add things for which we know that they don't work try: cs.add_forbidden_clause(ForbiddenAndConjunction( ForbiddenEqualsClause(cs.get_hyperparameter( "select_percentile_classification:score_func"), "chi2"), ForbiddenEqualsClause(cs.get_hyperparameter( "rescaling:strategy"), "standard") )) except: pass return cs
def get_hyperparameter_search_space(dataset_properties=None): policy_choices = ['fixed', 'inv', 'exp', 'step'] batch_size = UniformIntegerHyperparameter("batch_size", 32, 2048, log=True, default=100) number_updates = UniformIntegerHyperparameter("number_updates", 500, 10500, log=True, default=1050) dropout_output = UniformFloatHyperparameter("dropout_output", 0.0, 0.99, default=0.5) lr = UniformFloatHyperparameter("learning_rate", 1e-6, 0.1, log=True, default=1e-6) l2 = UniformFloatHyperparameter("lambda2", 1e-6, 1e-2, log=True, default=1e-6) solver = CategoricalHyperparameter(name="solver", choices=["sgd", "adam"], default="adam") beta1 = UniformFloatHyperparameter("beta1", 1e-4, 0.1, log=True, default=0.1) beta2 = UniformFloatHyperparameter("beta2", 1e-4, 0.1, log=True, default=0.01) lr_policy = CategoricalHyperparameter(name="lr_policy", choices=policy_choices, default='fixed') gamma = UniformFloatHyperparameter(name="gamma", lower=1e-3, upper=1e-1, default=1e-2) power = UniformFloatHyperparameter("power", 0.0, 1.0, default=0.5) epoch_step = UniformIntegerHyperparameter("epoch_step", 2, 20, default=5) cs = ConfigurationSpace() cs.add_hyperparameter(number_updates) cs.add_hyperparameter(batch_size) cs.add_hyperparameter(dropout_output) cs.add_hyperparameter(lr) cs.add_hyperparameter(l2) cs.add_hyperparameter(solver) cs.add_hyperparameter(beta1) cs.add_hyperparameter(beta2) cs.add_hyperparameter(lr_policy) cs.add_hyperparameter(gamma) cs.add_hyperparameter(power) cs.add_hyperparameter(epoch_step) beta1_depends_on_solver = EqualsCondition(beta1, solver, "adam") beta2_depends_on_solver = EqualsCondition(beta2, solver, "adam") gamma_depends_on_policy = InCondition(child=gamma, parent=lr_policy, values=['inv', 'exp', 'step']) power_depends_on_policy = EqualsCondition(power, lr_policy, 'inv') epoch_step_depends_on_policy = EqualsCondition(epoch_step, lr_policy, 'step') cs.add_condition(beta1_depends_on_solver) cs.add_condition(beta2_depends_on_solver) cs.add_condition(gamma_depends_on_policy) cs.add_condition(power_depends_on_policy) cs.add_condition(epoch_step_depends_on_policy) return cs
def read(pcs_string, debug=False): configuration_space = ConfigurationSpace() conditions = [] forbidden = [] # some statistics ct = 0 cont_ct = 0 cat_ct = 0 line_ct = 0 for line in pcs_string: line_ct += 1 if "#" in line: # It contains a comment pos = line.find("#") line = line[:pos] # Remove quotes and whitespaces at beginning and end line = line.replace('"', "").replace("'", "") line = line.strip() if "|" in line: # It's a condition try: c = pp_condition.parseString(line) conditions.append(c) except pyparsing.ParseException: raise NotImplementedError("Could not parse condition: %s" % line) continue if "}" not in line and "]" not in line: print("Skipping: %s" % line) continue if line.startswith("{") and line.endswith("}"): forbidden.append(line) continue if len(line.strip()) == 0: continue ct += 1 param = None # print "Parsing: " + line create = { "int": UniformIntegerHyperparameter, "float": UniformFloatHyperparameter, "categorical": CategoricalHyperparameter } try: param_list = pp_cont_param.parseString(line) il = param_list[9:] if len(il) > 0: il = il[0] param_list = param_list[:9] name = param_list[0] lower = float(param_list[2]) upper = float(param_list[4]) paramtype = "int" if "i" in il else "float" log = True if "l" in il else False default = float(param_list[7]) param = create[paramtype](name=name, lower=lower, upper=upper, q=None, log=log, default=default) cont_ct += 1 except pyparsing.ParseException: pass try: param_list = pp_cat_param.parseString(line) name = param_list[0] choices = [c for c in param_list[2:-4:2]] default = param_list[-2] param = create["categorical"](name=name, choices=choices, default=default) cat_ct += 1 except pyparsing.ParseException: pass if param is None: raise NotImplementedError("Could not parse: %s" % line) configuration_space.add_hyperparameter(param) for clause in forbidden: # TODO test this properly! # TODO Add a try/catch here! # noinspection PyUnusedLocal param_list = pp_forbidden_clause.parseString(clause) tmp_list = [] clause_list = [] for value in param_list[1:]: if len(tmp_list) < 3: tmp_list.append(value) else: # So far, only equals is supported by SMAC if tmp_list[1] == '=': # TODO maybe add a check if the hyperparameter is # actually in the configuration space clause_list.append( ForbiddenEqualsClause( configuration_space.get_hyperparameter( tmp_list[0]), tmp_list[2])) else: raise NotImplementedError() tmp_list = [] configuration_space.add_forbidden_clause( ForbiddenAndConjunction(*clause_list)) #Now handle conditions # If there are two conditions for one child, these two conditions are an # AND-conjunction of conditions, thus we have to connect them conditions_per_child = defaultdict(list) for condition in conditions: child_name = condition[0] conditions_per_child[child_name].append(condition) for child_name in conditions_per_child: condition_objects = [] for condition in conditions_per_child[child_name]: child = configuration_space.get_hyperparameter(child_name) parent_name = condition[2] parent = configuration_space.get_hyperparameter(parent_name) restrictions = condition[5:-1:2] # TODO: cast the type of the restriction! if len(restrictions) == 1: condition = EqualsCondition(child, parent, restrictions[0]) else: condition = InCondition(child, parent, values=restrictions) condition_objects.append(condition) # Now we have all condition objects for this child, so we can build a # giant AND-conjunction of them (if number of conditions >= 2)! if len(condition_objects) > 1: and_conjunction = AndConjunction(*condition_objects) configuration_space.add_condition(and_conjunction) else: configuration_space.add_condition(condition_objects[0]) return configuration_space
def read(pcs_string, debug=False): configuration_space = ConfigurationSpace() conditions = [] forbidden = [] # some statistics ct = 0 cont_ct = 0 cat_ct = 0 line_ct = 0 for line in pcs_string: line_ct += 1 if "#" in line: # It contains a comment pos = line.find("#") line = line[:pos] # Remove quotes and whitespaces at beginning and end line = line.replace('"', "").replace("'", "") line = line.strip() if "|" in line: # It's a condition try: c = pp_condition.parseString(line) conditions.append(c) except pyparsing.ParseException: raise NotImplementedError("Could not parse condition: %s" % line) continue if "}" not in line and "]" not in line: print("Skipping: %s" % line) continue if line.startswith("{") and line.endswith("}"): forbidden.append(line) continue if len(line.strip()) == 0: continue ct += 1 param = None # print "Parsing: " + line create = {"int": UniformIntegerHyperparameter, "float": UniformFloatHyperparameter, "categorical": CategoricalHyperparameter} try: param_list = pp_cont_param.parseString(line) il = param_list[9:] if len(il) > 0: il = il[0] param_list = param_list[:9] name = param_list[0] lower = float(param_list[2]) upper = float(param_list[4]) paramtype = "int" if "i" in il else "float" log = True if "l" in il else False default = float(param_list[7]) param = create[paramtype](name=name, lower=lower, upper=upper, q=None, log=log, default=default) cont_ct += 1 except pyparsing.ParseException: pass try: param_list = pp_cat_param.parseString(line) name = param_list[0] choices = [c for c in param_list[2:-4:2]] default = param_list[-2] param = create["categorical"](name=name, choices=choices, default=default) cat_ct += 1 except pyparsing.ParseException: pass if param is None: raise NotImplementedError("Could not parse: %s" % line) configuration_space.add_hyperparameter(param) for clause in forbidden: # TODO test this properly! # TODO Add a try/catch here! # noinspection PyUnusedLocal param_list = pp_forbidden_clause.parseString(clause) tmp_list = [] clause_list = [] for value in param_list[1:]: if len(tmp_list) < 3: tmp_list.append(value) else: # So far, only equals is supported by SMAC if tmp_list[1] == '=': # TODO maybe add a check if the hyperparameter is # actually in the configuration space clause_list.append(ForbiddenEqualsClause( configuration_space.get_hyperparameter(tmp_list[0]), tmp_list[2])) else: raise NotImplementedError() tmp_list = [] configuration_space.add_forbidden_clause(ForbiddenAndConjunction( *clause_list)) #Now handle conditions # If there are two conditions for one child, these two conditions are an # AND-conjunction of conditions, thus we have to connect them conditions_per_child = defaultdict(list) for condition in conditions: child_name = condition[0] conditions_per_child[child_name].append(condition) for child_name in conditions_per_child: condition_objects = [] for condition in conditions_per_child[child_name]: child = configuration_space.get_hyperparameter(child_name) parent_name = condition[2] parent = configuration_space.get_hyperparameter(parent_name) restrictions = condition[5:-1:2] # TODO: cast the type of the restriction! if len(restrictions) == 1: condition = EqualsCondition(child, parent, restrictions[0]) else: condition = InCondition(child, parent, values=restrictions) condition_objects.append(condition) # Now we have all condition objects for this child, so we can build a # giant AND-conjunction of them (if number of conditions >= 2)! if len(condition_objects) > 1: and_conjunction = AndConjunction(*condition_objects) configuration_space.add_condition(and_conjunction) else: configuration_space.add_condition(condition_objects[0]) return configuration_space
def get_hyperparameter_search_space(dataset_properties=None): max_num_layers = 7 # Maximum number of layers coded # Hacky way to condition layers params based on the number of layers # 'c'=1, 'd'=2, 'e'=3 ,'f'=4', g ='5', h='6' + output_layer layer_choices = [ chr(i) for i in range(ord('c'), ord('b') + max_num_layers) ] batch_size = UniformIntegerHyperparameter("batch_size", 32, 4096, log=True, default=32) number_epochs = UniformIntegerHyperparameter("number_epochs", 2, 80, default=5) num_layers = CategoricalHyperparameter("num_layers", choices=layer_choices, default='c') lr = UniformFloatHyperparameter("learning_rate", 1e-6, 1.0, log=True, default=0.01) l2 = UniformFloatHyperparameter("lambda2", 1e-7, 1e-2, log=True, default=1e-4) dropout_output = UniformFloatHyperparameter("dropout_output", 0.0, 0.99, default=0.5) # Define basic hyperparameters and define the config space # basic means that are independent from the number of layers cs = ConfigurationSpace() cs.add_hyperparameter(number_epochs) cs.add_hyperparameter(batch_size) cs.add_hyperparameter(num_layers) cs.add_hyperparameter(lr) cs.add_hyperparameter(l2) cs.add_hyperparameter(dropout_output) # Define parameters with different child parameters and conditions solver_choices = [ "adam", "adadelta", "adagrad", "sgd", "momentum", "nesterov", "smorm3s" ] solver = CategoricalHyperparameter(name="solver", choices=solver_choices, default="smorm3s") beta1 = UniformFloatHyperparameter("beta1", 1e-4, 0.1, log=True, default=0.1) beta2 = UniformFloatHyperparameter("beta2", 1e-4, 0.1, log=True, default=0.01) rho = UniformFloatHyperparameter("rho", 0.05, 0.99, log=True, default=0.95) momentum = UniformFloatHyperparameter("momentum", 0.3, 0.999, default=0.9) # TODO: Add policy based on this sklearn sgd policy_choices = ['fixed', 'inv', 'exp', 'step'] lr_policy = CategoricalHyperparameter(name="lr_policy", choices=policy_choices, default='fixed') gamma = UniformFloatHyperparameter(name="gamma", lower=1e-3, upper=1e-1, default=1e-2) power = UniformFloatHyperparameter("power", 0.0, 1.0, default=0.5) epoch_step = UniformIntegerHyperparameter("epoch_step", 2, 20, default=5) cs.add_hyperparameter(solver) cs.add_hyperparameter(beta1) cs.add_hyperparameter(beta2) cs.add_hyperparameter(momentum) cs.add_hyperparameter(rho) cs.add_hyperparameter(lr_policy) cs.add_hyperparameter(gamma) cs.add_hyperparameter(power) cs.add_hyperparameter(epoch_step) # Define parameters that are needed it for each layer output_activation_choices = ['softmax', 'sigmoid', 'softplus', 'tanh'] activations_choices = [ 'sigmoid', 'tanh', 'scaledTanh', 'elu', 'relu', 'leaky', 'linear' ] weight_choices = [ 'constant', 'normal', 'uniform', 'glorot_normal', 'glorot_uniform', 'he_normal', 'he_uniform', 'ortogonal', 'sparse' ] # Iterate over parameters that are used in each layer for i in range(1, max_num_layers): layer_units = UniformIntegerHyperparameter("num_units_layer_" + str(i), 64, 4096, log=True, default=128) cs.add_hyperparameter(layer_units) layer_dropout = UniformFloatHyperparameter("dropout_layer_" + str(i), 0.0, 0.99, default=0.5) cs.add_hyperparameter(layer_dropout) weight_initialization = CategoricalHyperparameter( 'weight_init_' + str(i), choices=weight_choices, default='he_normal') cs.add_hyperparameter(weight_initialization) layer_std = UniformFloatHyperparameter("std_layer_" + str(i), 1e-6, 0.1, log=True, default=0.005) cs.add_hyperparameter(layer_std) layer_activation = CategoricalHyperparameter( "activation_layer_" + str(i), choices=activations_choices, default="relu") cs.add_hyperparameter(layer_activation) layer_leakiness = UniformFloatHyperparameter('leakiness_layer_' + str(i), 0.01, 0.99, default=0.3) cs.add_hyperparameter(layer_leakiness) layer_tanh_alpha = UniformFloatHyperparameter('tanh_alpha_layer_' + str(i), 0.5, 1.0, default=2. / 3.) cs.add_hyperparameter(layer_tanh_alpha) layer_tanh_beta = UniformFloatHyperparameter('tanh_beta_layer_' + str(i), 1.1, 3.0, log=True, default=1.7159) cs.add_hyperparameter(layer_tanh_beta) # TODO: Could be in a function in a new module for i in range(2, max_num_layers): # Condition layers parameter on layer choice layer_unit_param = cs.get_hyperparameter("num_units_layer_" + str(i)) layer_cond = InCondition(child=layer_unit_param, parent=num_layers, values=[l for l in layer_choices[i - 1:]]) cs.add_condition(layer_cond) # Condition dropout parameter on layer choice layer_dropout_param = cs.get_hyperparameter("dropout_layer_" + str(i)) layer_cond = InCondition(child=layer_dropout_param, parent=num_layers, values=[l for l in layer_choices[i - 1:]]) cs.add_condition(layer_cond) # Condition weight initialization on layer choice layer_weight_param = cs.get_hyperparameter("weight_init_" + str(i)) layer_cond = InCondition(child=layer_weight_param, parent=num_layers, values=[l for l in layer_choices[i - 1:]]) cs.add_condition(layer_cond) # Condition std parameter on weight layer initialization choice layer_std_param = cs.get_hyperparameter("std_layer_" + str(i)) weight_cond = EqualsCondition(child=layer_std_param, parent=layer_weight_param, value='normal') cs.add_condition(weight_cond) # Condition activation parameter on layer choice layer_activation_param = cs.get_hyperparameter( "activation_layer_" + str(i)) layer_cond = InCondition(child=layer_activation_param, parent=num_layers, values=[l for l in layer_choices[i - 1:]]) cs.add_condition(layer_cond) # Condition leakiness on activation choice layer_leakiness_param = cs.get_hyperparameter("leakiness_layer_" + str(i)) activation_cond = EqualsCondition(child=layer_leakiness_param, parent=layer_activation_param, value='leaky') cs.add_condition(activation_cond) # Condition tanh on activation choice layer_tanh_alpha_param = cs.get_hyperparameter( "tanh_alpha_layer_" + str(i)) activation_cond = EqualsCondition(child=layer_tanh_alpha_param, parent=layer_activation_param, value='scaledTanh') cs.add_condition(activation_cond) layer_tanh_beta_param = cs.get_hyperparameter("tanh_beta_layer_" + str(i)) activation_cond = EqualsCondition(child=layer_tanh_beta_param, parent=layer_activation_param, value='scaledTanh') cs.add_condition(activation_cond) # Conditioning on solver momentum_depends_on_solver = InCondition( momentum, solver, values=["momentum", "nesterov"]) beta1_depends_on_solver = EqualsCondition(beta1, solver, "adam") beta2_depends_on_solver = EqualsCondition(beta2, solver, "adam") rho_depends_on_solver = EqualsCondition(rho, solver, "adadelta") cs.add_condition(momentum_depends_on_solver) cs.add_condition(beta1_depends_on_solver) cs.add_condition(beta2_depends_on_solver) cs.add_condition(rho_depends_on_solver) # Conditioning on learning rate policy lr_policy_depends_on_solver = InCondition( lr_policy, solver, ["adadelta", "adagrad", "sgd", "momentum", "nesterov"]) gamma_depends_on_policy = InCondition(child=gamma, parent=lr_policy, values=["inv", "exp", "step"]) power_depends_on_policy = EqualsCondition(power, lr_policy, "inv") epoch_step_depends_on_policy = EqualsCondition(epoch_step, lr_policy, "step") cs.add_condition(lr_policy_depends_on_solver) cs.add_condition(gamma_depends_on_policy) cs.add_condition(power_depends_on_policy) cs.add_condition(epoch_step_depends_on_policy) return cs
def get_hyperparameter_search_space(cls, dataset_properties, default=None, include=None, exclude=None): if include is not None and exclude is not None: raise ValueError("The arguments include_estimators and " "exclude_estimators cannot be used together.") cs = ConfigurationSpace() # Compile a list of all estimator objects for this problem available_estimators = cls.get_available_components( data_prop=dataset_properties, include=include, exclude=exclude) if len(available_estimators) == 0: raise ValueError("No classifiers found") if default is None: defaults = ['random_forest', 'liblinear_svc', 'sgd', 'libsvm_svc' ] + list(available_estimators.keys()) for default_ in defaults: if default_ in available_estimators: if include is not None and default_ not in include: continue if exclude is not None and default_ in exclude: continue default = default_ break estimator = CategoricalHyperparameter('__choice__', list( available_estimators.keys()), default=default) cs.add_hyperparameter(estimator) for estimator_name in available_estimators.keys(): # We have to retrieve the configuration space every time because # we change the objects it returns. If we reused it, we could not # retrieve the conditions further down # TODO implement copy for hyperparameters and forbidden and # conditions! estimator_configuration_space = available_estimators[ estimator_name]. \ get_hyperparameter_search_space(dataset_properties) for parameter in estimator_configuration_space.get_hyperparameters( ): new_parameter = copy.deepcopy(parameter) new_parameter.name = "%s:%s" % (estimator_name, new_parameter.name) cs.add_hyperparameter(new_parameter) # We must only add a condition if the hyperparameter is not # conditional on something else if len(estimator_configuration_space.get_parents_of( parameter)) == 0: condition = EqualsCondition(new_parameter, estimator, estimator_name) cs.add_condition(condition) for condition in available_estimators[estimator_name]. \ get_hyperparameter_search_space( dataset_properties).get_conditions(): dlcs = condition.get_descendant_literal_conditions() for dlc in dlcs: if not dlc.child.name.startswith(estimator_name): dlc.child.name = "%s:%s" % (estimator_name, dlc.child.name) if not dlc.parent.name.startswith(estimator_name): dlc.parent.name = "%s:%s" % (estimator_name, dlc.parent.name) cs.add_condition(condition) for forbidden_clause in available_estimators[estimator_name]. \ get_hyperparameter_search_space( dataset_properties).forbidden_clauses: dlcs = forbidden_clause.get_descendant_literal_clauses() for dlc in dlcs: if not dlc.hyperparameter.name.startswith(estimator_name): dlc.hyperparameter.name = "%s:%s" % ( estimator_name, dlc.hyperparameter.name) cs.add_forbidden_clause(forbidden_clause) return cs