def get_hyperparameter_search_space(dataset_properties=None): n_components = UniformIntegerHyperparameter("n_components", 10, 2000, default=100) kernel = CategoricalHyperparameter( 'kernel', ['poly', 'rbf', 'sigmoid', 'cosine'], 'rbf') degree = UniformIntegerHyperparameter('degree', 2, 5, 3) gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8, log=True, default=1.0) coef0 = UniformFloatHyperparameter("coef0", -1, 1, default=0) cs = ConfigurationSpace() cs.add_hyperparameter(n_components) cs.add_hyperparameter(kernel) cs.add_hyperparameter(degree) cs.add_hyperparameter(gamma) cs.add_hyperparameter(coef0) degree_depends_on_poly = EqualsCondition(degree, kernel, "poly") coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"]) gamma_condition = InCondition(gamma, kernel, ["poly", "rbf"]) cs.add_condition(degree_depends_on_poly) cs.add_condition(coef0_condition) cs.add_condition(gamma_condition) return cs
def test_in_condition(self): hp1 = CategoricalHyperparameter("parent", range(0, 11)) hp2 = UniformIntegerHyperparameter("child", 0, 10) cond = InCondition(hp2, hp1, [0, 1, 2, 3, 4, 5]) cond_ = InCondition(hp2, hp1, [0, 1, 2, 3, 4, 5]) self.assertEqual(cond, cond_) cond_reverse = InCondition(hp1, hp2, [0, 1, 2, 3, 4, 5]) self.assertNotEqual(cond, cond_reverse) self.assertNotEqual(cond, dict()) self.assertEqual("child | parent in {0, 1, 2, 3, 4, 5}", str(cond))
def get_hyperparameter_search_space(dataset_properties=None): if dataset_properties is not None and \ (dataset_properties.get("sparse") is True or dataset_properties.get("signed") is False): allow_chi2 = False else: allow_chi2 = True possible_kernels = ['poly', 'rbf', 'sigmoid', 'cosine'] if allow_chi2: possible_kernels.append("chi2") kernel = CategoricalHyperparameter('kernel', possible_kernels, 'rbf') degree = UniformIntegerHyperparameter('degree', 2, 5, 3) gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8, log=True, default=0.1) coef0 = UniformFloatHyperparameter("coef0", -1, 1, default=0) n_components = UniformIntegerHyperparameter("n_components", 50, 10000, default=100, log=True) cs = ConfigurationSpace() cs.add_hyperparameter(kernel) cs.add_hyperparameter(degree) cs.add_hyperparameter(gamma) cs.add_hyperparameter(coef0) cs.add_hyperparameter(n_components) degree_depends_on_poly = EqualsCondition(degree, kernel, "poly") coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"]) gamma_kernels = ["poly", "rbf", "sigmoid"] if allow_chi2: gamma_kernels.append("chi2") gamma_condition = InCondition(gamma, kernel, gamma_kernels) cs.add_condition(degree_depends_on_poly) cs.add_condition(coef0_condition) cs.add_condition(gamma_condition) return cs
def test_sample_configuration(self): cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("parent", [0, 1]) cs.add_hyperparameter(hp1) hp2 = UniformIntegerHyperparameter("child", 0, 10) cs.add_hyperparameter(hp2) cond1 = EqualsCondition(hp2, hp1, 0) cs.add_condition(cond1) # This automatically checks the configuration! Configuration(cs, dict(parent=0, child=5)) # and now for something more complicated cs = ConfigurationSpace(seed=1) hp1 = CategoricalHyperparameter("input1", [0, 1]) cs.add_hyperparameter(hp1) hp2 = CategoricalHyperparameter("input2", [0, 1]) cs.add_hyperparameter(hp2) hp3 = CategoricalHyperparameter("input3", [0, 1]) cs.add_hyperparameter(hp3) hp4 = CategoricalHyperparameter("input4", [0, 1]) cs.add_hyperparameter(hp4) hp5 = CategoricalHyperparameter("input5", [0, 1]) cs.add_hyperparameter(hp5) hp6 = Constant("AND", "True") cs.add_hyperparameter(hp6) cond1 = EqualsCondition(hp6, hp1, 1) cond2 = NotEqualsCondition(hp6, hp2, 1) cond3 = InCondition(hp6, hp3, [1]) cond4 = EqualsCondition(hp5, hp3, 1) cond5 = EqualsCondition(hp4, hp5, 1) cond6 = EqualsCondition(hp6, hp4, 1) cond7 = EqualsCondition(hp6, hp5, 1) conj1 = AndConjunction(cond1, cond2) conj2 = OrConjunction(conj1, cond3) conj3 = AndConjunction(conj2, cond6, cond7) cs.add_condition(cond4) cs.add_condition(cond5) cs.add_condition(conj3) samples = [] for i in range(5): cs.seed(1) samples.append([]) for j in range(100): sample = cs.sample_configuration() samples[-1].append(sample) if i > 0: for j in range(100): self.assertEqual(samples[-1][j], samples[-2][j])
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() loss = cs.add_hyperparameter( CategoricalHyperparameter("loss", ["ls", "lad", "huber", "quantile"], default="ls")) learning_rate = cs.add_hyperparameter( UniformFloatHyperparameter(name="learning_rate", lower=0.0001, upper=1, default=0.1, log=True)) n_estimators = cs.add_hyperparameter(Constant("n_estimators", 100)) max_depth = cs.add_hyperparameter( UniformIntegerHyperparameter(name="max_depth", lower=1, upper=10, default=3)) min_samples_split = cs.add_hyperparameter( UniformIntegerHyperparameter(name="min_samples_split", lower=2, upper=20, default=2, log=False)) min_samples_leaf = cs.add_hyperparameter( UniformIntegerHyperparameter(name="min_samples_leaf", lower=1, upper=20, default=1, log=False)) min_weight_fraction_leaf = cs.add_hyperparameter( UnParametrizedHyperparameter("min_weight_fraction_leaf", 0.)) subsample = cs.add_hyperparameter( UniformFloatHyperparameter(name="subsample", lower=0.01, upper=1.0, default=1.0, log=False)) max_features = cs.add_hyperparameter( UniformFloatHyperparameter("max_features", 0.5, 5, default=1)) max_leaf_nodes = cs.add_hyperparameter( UnParametrizedHyperparameter(name="max_leaf_nodes", value="None")) alpha = cs.add_hyperparameter( UniformFloatHyperparameter("alpha", lower=0.75, upper=0.99, default=0.9)) cs.add_condition(InCondition(alpha, loss, ['huber', 'quantile'])) return cs
def test_get_hyperparameters_topological_sort(self): for iteration in range(10): cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("parent", [0, 1]) cs.add_hyperparameter(hp1) hp2 = UniformIntegerHyperparameter("child", 0, 10) cs.add_hyperparameter(hp2) cond1 = EqualsCondition(hp2, hp1, 0) cs.add_condition(cond1) # This automatically checks the configuration! Configuration(cs, dict(parent=0, child=5)) # and now for something more complicated cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("input1", [0, 1]) hp2 = CategoricalHyperparameter("input2", [0, 1]) hp3 = CategoricalHyperparameter("input3", [0, 1]) hp4 = CategoricalHyperparameter("input4", [0, 1]) hp5 = CategoricalHyperparameter("input5", [0, 1]) hp6 = Constant("AND", "True") # More top-level hyperparameters hp7 = CategoricalHyperparameter("input7", [0, 1]) hps = [hp1, hp2, hp3, hp4, hp5, hp6, hp7] random.shuffle(hps) for hp in hps: cs.add_hyperparameter(hp) cond1 = EqualsCondition(hp6, hp1, 1) cond2 = NotEqualsCondition(hp6, hp2, 1) cond3 = InCondition(hp6, hp3, [1]) cond4 = EqualsCondition(hp5, hp3, 1) cond5 = EqualsCondition(hp4, hp5, 1) cond6 = EqualsCondition(hp6, hp4, 1) cond7 = EqualsCondition(hp6, hp5, 1) conj1 = AndConjunction(cond1, cond2) conj2 = OrConjunction(conj1, cond3) conj3 = AndConjunction(conj2, cond6, cond7) cs.add_condition(cond4) cs.add_condition(cond5) cs.add_condition(conj3) hps = cs.get_hyperparameters() self.assertEqual(hps.index(hp1), 0) self.assertEqual(hps.index(hp2), 1) self.assertEqual(hps.index(hp3), 2) self.assertEqual(hps.index(hp7), 3) self.assertEqual(hps.index(hp5), 4) self.assertEqual(hps.index(hp4), 5) self.assertEqual(hps.index(hp6), 6)
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() loss = cs.add_hyperparameter(CategoricalHyperparameter("loss", ["squared_loss", "huber", "epsilon_insensitive", "squared_epsilon_insensitive"], default="squared_loss")) penalty = cs.add_hyperparameter(CategoricalHyperparameter( "penalty", ["l1", "l2", "elasticnet"], default="l2")) alpha = cs.add_hyperparameter(UniformFloatHyperparameter( "alpha", 10e-7, 1e-1, log=True, default=0.01)) l1_ratio = cs.add_hyperparameter(UniformFloatHyperparameter( "l1_ratio", 1e-9, 1., log=True, default=0.15)) fit_intercept = cs.add_hyperparameter(UnParametrizedHyperparameter( "fit_intercept", "True")) n_iter = cs.add_hyperparameter(UniformIntegerHyperparameter( "n_iter", 5, 1000, log=True, default=20)) epsilon = cs.add_hyperparameter(UniformFloatHyperparameter( "epsilon", 1e-5, 1e-1, default=1e-4, log=True)) learning_rate = cs.add_hyperparameter(CategoricalHyperparameter( "learning_rate", ["optimal", "invscaling", "constant"], default="optimal")) eta0 = cs.add_hyperparameter(UniformFloatHyperparameter( "eta0", 10 ** -7, 0.1, default=0.01)) power_t = cs.add_hyperparameter(UniformFloatHyperparameter( "power_t", 1e-5, 1, default=0.5)) average = cs.add_hyperparameter(CategoricalHyperparameter( "average", ["False", "True"], default="False")) # TODO add passive/aggressive here, although not properly documented? elasticnet = EqualsCondition(l1_ratio, penalty, "elasticnet") epsilon_condition = InCondition(epsilon, loss, ["huber", "epsilon_insensitive", "squared_epsilon_insensitive"]) # eta0 seems to be always active according to the source code; when # learning_rate is set to optimial, eta0 is the starting value: # https://github.com/scikit-learn/scikit-learn/blob/0.15.X/sklearn/linear_model/sgd_fast.pyx # eta0_and_inv = EqualsCondition(eta0, learning_rate, "invscaling") #eta0_and_constant = EqualsCondition(eta0, learning_rate, "constant") #eta0_condition = OrConjunction(eta0_and_inv, eta0_and_constant) power_t_condition = EqualsCondition(power_t, learning_rate, "invscaling") cs.add_condition(elasticnet) cs.add_condition(epsilon_condition) cs.add_condition(power_t_condition) return cs
def get_hyperparameter_search_space(dataset_properties=None): C = UniformFloatHyperparameter("C", 0.03125, 32768, log=True, default=1.0) # No linear kernel here, because we have liblinear kernel = CategoricalHyperparameter(name="kernel", choices=["rbf", "poly", "sigmoid"], default="rbf") degree = UniformIntegerHyperparameter("degree", 1, 5, default=3) gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8, log=True, default=0.1) # TODO this is totally ad-hoc coef0 = UniformFloatHyperparameter("coef0", -1, 1, default=0) # probability is no hyperparameter, but an argument to the SVM algo shrinking = CategoricalHyperparameter("shrinking", ["True", "False"], default="True") tol = UniformFloatHyperparameter("tol", 1e-5, 1e-1, default=1e-4, log=True) # cache size is not a hyperparameter, but an argument to the program! max_iter = UnParametrizedHyperparameter("max_iter", -1) cs = ConfigurationSpace() cs.add_hyperparameter(C) cs.add_hyperparameter(kernel) cs.add_hyperparameter(degree) cs.add_hyperparameter(gamma) cs.add_hyperparameter(coef0) cs.add_hyperparameter(shrinking) cs.add_hyperparameter(tol) cs.add_hyperparameter(max_iter) degree_depends_on_poly = EqualsCondition(degree, kernel, "poly") coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"]) cs.add_condition(degree_depends_on_poly) cs.add_condition(coef0_condition) return cs
def get_hyperparameter_search_space(dataset_properties=None): max_num_layers = 7 # Maximum number of layers coded # Hacky way to condition layers params based on the number of layers # 'c'=1, 'd'=2, 'e'=3 ,'f'=4', g ='5', h='6' + output_layer layer_choices = [ chr(i) for i in range(ord('c'), ord('b') + max_num_layers) ] batch_size = UniformIntegerHyperparameter("batch_size", 32, 4096, log=True, default=32) number_epochs = UniformIntegerHyperparameter("number_epochs", 2, 80, default=5) num_layers = CategoricalHyperparameter("num_layers", choices=layer_choices, default='c') lr = UniformFloatHyperparameter("learning_rate", 1e-6, 1.0, log=True, default=0.01) l2 = UniformFloatHyperparameter("lambda2", 1e-7, 1e-2, log=True, default=1e-4) dropout_output = UniformFloatHyperparameter("dropout_output", 0.0, 0.99, default=0.5) # Define basic hyperparameters and define the config space # basic means that are independent from the number of layers cs = ConfigurationSpace() cs.add_hyperparameter(number_epochs) cs.add_hyperparameter(batch_size) cs.add_hyperparameter(num_layers) cs.add_hyperparameter(lr) cs.add_hyperparameter(l2) cs.add_hyperparameter(dropout_output) # Define parameters with different child parameters and conditions solver_choices = [ "adam", "adadelta", "adagrad", "sgd", "momentum", "nesterov", "smorm3s" ] solver = CategoricalHyperparameter(name="solver", choices=solver_choices, default="smorm3s") beta1 = UniformFloatHyperparameter("beta1", 1e-4, 0.1, log=True, default=0.1) beta2 = UniformFloatHyperparameter("beta2", 1e-4, 0.1, log=True, default=0.01) rho = UniformFloatHyperparameter("rho", 0.05, 0.99, log=True, default=0.95) momentum = UniformFloatHyperparameter("momentum", 0.3, 0.999, default=0.9) # TODO: Add policy based on this sklearn sgd policy_choices = ['fixed', 'inv', 'exp', 'step'] lr_policy = CategoricalHyperparameter(name="lr_policy", choices=policy_choices, default='fixed') gamma = UniformFloatHyperparameter(name="gamma", lower=1e-3, upper=1e-1, default=1e-2) power = UniformFloatHyperparameter("power", 0.0, 1.0, default=0.5) epoch_step = UniformIntegerHyperparameter("epoch_step", 2, 20, default=5) cs.add_hyperparameter(solver) cs.add_hyperparameter(beta1) cs.add_hyperparameter(beta2) cs.add_hyperparameter(momentum) cs.add_hyperparameter(rho) cs.add_hyperparameter(lr_policy) cs.add_hyperparameter(gamma) cs.add_hyperparameter(power) cs.add_hyperparameter(epoch_step) # Define parameters that are needed it for each layer output_activation_choices = ['softmax', 'sigmoid', 'softplus', 'tanh'] activations_choices = [ 'sigmoid', 'tanh', 'scaledTanh', 'elu', 'relu', 'leaky', 'linear' ] weight_choices = [ 'constant', 'normal', 'uniform', 'glorot_normal', 'glorot_uniform', 'he_normal', 'he_uniform', 'ortogonal', 'sparse' ] # Iterate over parameters that are used in each layer for i in range(1, max_num_layers): layer_units = UniformIntegerHyperparameter("num_units_layer_" + str(i), 64, 4096, log=True, default=128) cs.add_hyperparameter(layer_units) layer_dropout = UniformFloatHyperparameter("dropout_layer_" + str(i), 0.0, 0.99, default=0.5) cs.add_hyperparameter(layer_dropout) weight_initialization = CategoricalHyperparameter( 'weight_init_' + str(i), choices=weight_choices, default='he_normal') cs.add_hyperparameter(weight_initialization) layer_std = UniformFloatHyperparameter("std_layer_" + str(i), 1e-6, 0.1, log=True, default=0.005) cs.add_hyperparameter(layer_std) layer_activation = CategoricalHyperparameter( "activation_layer_" + str(i), choices=activations_choices, default="relu") cs.add_hyperparameter(layer_activation) layer_leakiness = UniformFloatHyperparameter('leakiness_layer_' + str(i), 0.01, 0.99, default=0.3) cs.add_hyperparameter(layer_leakiness) layer_tanh_alpha = UniformFloatHyperparameter('tanh_alpha_layer_' + str(i), 0.5, 1.0, default=2. / 3.) cs.add_hyperparameter(layer_tanh_alpha) layer_tanh_beta = UniformFloatHyperparameter('tanh_beta_layer_' + str(i), 1.1, 3.0, log=True, default=1.7159) cs.add_hyperparameter(layer_tanh_beta) # TODO: Could be in a function in a new module for i in range(2, max_num_layers): # Condition layers parameter on layer choice layer_unit_param = cs.get_hyperparameter("num_units_layer_" + str(i)) layer_cond = InCondition(child=layer_unit_param, parent=num_layers, values=[l for l in layer_choices[i - 1:]]) cs.add_condition(layer_cond) # Condition dropout parameter on layer choice layer_dropout_param = cs.get_hyperparameter("dropout_layer_" + str(i)) layer_cond = InCondition(child=layer_dropout_param, parent=num_layers, values=[l for l in layer_choices[i - 1:]]) cs.add_condition(layer_cond) # Condition weight initialization on layer choice layer_weight_param = cs.get_hyperparameter("weight_init_" + str(i)) layer_cond = InCondition(child=layer_weight_param, parent=num_layers, values=[l for l in layer_choices[i - 1:]]) cs.add_condition(layer_cond) # Condition std parameter on weight layer initialization choice layer_std_param = cs.get_hyperparameter("std_layer_" + str(i)) weight_cond = EqualsCondition(child=layer_std_param, parent=layer_weight_param, value='normal') cs.add_condition(weight_cond) # Condition activation parameter on layer choice layer_activation_param = cs.get_hyperparameter( "activation_layer_" + str(i)) layer_cond = InCondition(child=layer_activation_param, parent=num_layers, values=[l for l in layer_choices[i - 1:]]) cs.add_condition(layer_cond) # Condition leakiness on activation choice layer_leakiness_param = cs.get_hyperparameter("leakiness_layer_" + str(i)) activation_cond = EqualsCondition(child=layer_leakiness_param, parent=layer_activation_param, value='leaky') cs.add_condition(activation_cond) # Condition tanh on activation choice layer_tanh_alpha_param = cs.get_hyperparameter( "tanh_alpha_layer_" + str(i)) activation_cond = EqualsCondition(child=layer_tanh_alpha_param, parent=layer_activation_param, value='scaledTanh') cs.add_condition(activation_cond) layer_tanh_beta_param = cs.get_hyperparameter("tanh_beta_layer_" + str(i)) activation_cond = EqualsCondition(child=layer_tanh_beta_param, parent=layer_activation_param, value='scaledTanh') cs.add_condition(activation_cond) # Conditioning on solver momentum_depends_on_solver = InCondition( momentum, solver, values=["momentum", "nesterov"]) beta1_depends_on_solver = EqualsCondition(beta1, solver, "adam") beta2_depends_on_solver = EqualsCondition(beta2, solver, "adam") rho_depends_on_solver = EqualsCondition(rho, solver, "adadelta") cs.add_condition(momentum_depends_on_solver) cs.add_condition(beta1_depends_on_solver) cs.add_condition(beta2_depends_on_solver) cs.add_condition(rho_depends_on_solver) # Conditioning on learning rate policy lr_policy_depends_on_solver = InCondition( lr_policy, solver, ["adadelta", "adagrad", "sgd", "momentum", "nesterov"]) gamma_depends_on_policy = InCondition(child=gamma, parent=lr_policy, values=["inv", "exp", "step"]) power_depends_on_policy = EqualsCondition(power, lr_policy, "inv") epoch_step_depends_on_policy = EqualsCondition(epoch_step, lr_policy, "step") cs.add_condition(lr_policy_depends_on_solver) cs.add_condition(gamma_depends_on_policy) cs.add_condition(power_depends_on_policy) cs.add_condition(epoch_step_depends_on_policy) return cs
def get_hyperparameter_search_space(dataset_properties=None): # Copied from libsvm_c C = UniformFloatHyperparameter(name="C", lower=0.03125, upper=32768, log=True, default=1.0) kernel = CategoricalHyperparameter( name="kernel", choices=['linear', 'poly', 'rbf', 'sigmoid'], default="rbf") degree = UniformIntegerHyperparameter(name="degree", lower=1, upper=5, default=3) # Changed the gamma value to 0.0 (is 0.1 for classification) gamma = UniformFloatHyperparameter(name="gamma", lower=3.0517578125e-05, upper=8, log=True, default=0.1) # TODO this is totally ad-hoc coef0 = UniformFloatHyperparameter(name="coef0", lower=-1, upper=1, default=0) # probability is no hyperparameter, but an argument to the SVM algo shrinking = CategoricalHyperparameter(name="shrinking", choices=["True", "False"], default="True") tol = UniformFloatHyperparameter(name="tol", lower=1e-5, upper=1e-1, default=1e-3, log=True) max_iter = UnParametrizedHyperparameter("max_iter", -1) # Random Guess epsilon = UniformFloatHyperparameter(name="epsilon", lower=0.001, upper=1, default=0.1, log=True) cs = ConfigurationSpace() cs.add_hyperparameter(C) cs.add_hyperparameter(kernel) cs.add_hyperparameter(degree) cs.add_hyperparameter(gamma) cs.add_hyperparameter(coef0) cs.add_hyperparameter(shrinking) cs.add_hyperparameter(tol) cs.add_hyperparameter(max_iter) cs.add_hyperparameter(epsilon) degree_depends_on_kernel = InCondition(child=degree, parent=kernel, values=('poly', 'rbf', 'sigmoid')) gamma_depends_on_kernel = InCondition(child=gamma, parent=kernel, values=('poly', 'rbf')) coef0_depends_on_kernel = InCondition(child=coef0, parent=kernel, values=('poly', 'sigmoid')) cs.add_condition(degree_depends_on_kernel) cs.add_condition(gamma_depends_on_kernel) cs.add_condition(coef0_depends_on_kernel) return cs
def get_hyperparameter_search_space(dataset_properties=None): policy_choices = ['fixed', 'inv', 'exp', 'step'] batch_size = UniformIntegerHyperparameter("batch_size", 32, 2048, log=True, default=100) number_updates = UniformIntegerHyperparameter("number_updates", 500, 10500, log=True, default=1050) dropout_output = UniformFloatHyperparameter("dropout_output", 0.0, 0.99, default=0.5) lr = UniformFloatHyperparameter("learning_rate", 1e-6, 0.1, log=True, default=1e-6) l2 = UniformFloatHyperparameter("lambda2", 1e-6, 1e-2, log=True, default=1e-6) solver = CategoricalHyperparameter(name="solver", choices=["sgd", "adam"], default="adam") beta1 = UniformFloatHyperparameter("beta1", 1e-4, 0.1, log=True, default=0.1) beta2 = UniformFloatHyperparameter("beta2", 1e-4, 0.1, log=True, default=0.01) lr_policy = CategoricalHyperparameter(name="lr_policy", choices=policy_choices, default='fixed') gamma = UniformFloatHyperparameter(name="gamma", lower=1e-3, upper=1e-1, default=1e-2) power = UniformFloatHyperparameter("power", 0.0, 1.0, default=0.5) epoch_step = UniformIntegerHyperparameter("epoch_step", 2, 20, default=5) cs = ConfigurationSpace() cs.add_hyperparameter(number_updates) cs.add_hyperparameter(batch_size) cs.add_hyperparameter(dropout_output) cs.add_hyperparameter(lr) cs.add_hyperparameter(l2) cs.add_hyperparameter(solver) cs.add_hyperparameter(beta1) cs.add_hyperparameter(beta2) cs.add_hyperparameter(lr_policy) cs.add_hyperparameter(gamma) cs.add_hyperparameter(power) cs.add_hyperparameter(epoch_step) beta1_depends_on_solver = EqualsCondition(beta1, solver, "adam") beta2_depends_on_solver = EqualsCondition(beta2, solver, "adam") gamma_depends_on_policy = InCondition(child=gamma, parent=lr_policy, values=['inv', 'exp', 'step']) power_depends_on_policy = EqualsCondition(power, lr_policy, 'inv') epoch_step_depends_on_policy = EqualsCondition(epoch_step, lr_policy, 'step') cs.add_condition(beta1_depends_on_solver) cs.add_condition(beta2_depends_on_solver) cs.add_condition(gamma_depends_on_policy) cs.add_condition(power_depends_on_policy) cs.add_condition(epoch_step_depends_on_policy) return cs
def read(pcs_string, debug=False): configuration_space = ConfigurationSpace() conditions = [] forbidden = [] # some statistics ct = 0 cont_ct = 0 cat_ct = 0 line_ct = 0 for line in pcs_string: line_ct += 1 if "#" in line: # It contains a comment pos = line.find("#") line = line[:pos] # Remove quotes and whitespaces at beginning and end line = line.replace('"', "").replace("'", "") line = line.strip() if "|" in line: # It's a condition try: c = pp_condition.parseString(line) conditions.append(c) except pyparsing.ParseException: raise NotImplementedError("Could not parse condition: %s" % line) continue if "}" not in line and "]" not in line: print("Skipping: %s" % line) continue if line.startswith("{") and line.endswith("}"): forbidden.append(line) continue if len(line.strip()) == 0: continue ct += 1 param = None # print "Parsing: " + line create = { "int": UniformIntegerHyperparameter, "float": UniformFloatHyperparameter, "categorical": CategoricalHyperparameter } try: param_list = pp_cont_param.parseString(line) il = param_list[9:] if len(il) > 0: il = il[0] param_list = param_list[:9] name = param_list[0] lower = float(param_list[2]) upper = float(param_list[4]) paramtype = "int" if "i" in il else "float" log = True if "l" in il else False default = float(param_list[7]) param = create[paramtype](name=name, lower=lower, upper=upper, q=None, log=log, default=default) cont_ct += 1 except pyparsing.ParseException: pass try: param_list = pp_cat_param.parseString(line) name = param_list[0] choices = [c for c in param_list[2:-4:2]] default = param_list[-2] param = create["categorical"](name=name, choices=choices, default=default) cat_ct += 1 except pyparsing.ParseException: pass if param is None: raise NotImplementedError("Could not parse: %s" % line) configuration_space.add_hyperparameter(param) for clause in forbidden: # TODO test this properly! # TODO Add a try/catch here! # noinspection PyUnusedLocal param_list = pp_forbidden_clause.parseString(clause) tmp_list = [] clause_list = [] for value in param_list[1:]: if len(tmp_list) < 3: tmp_list.append(value) else: # So far, only equals is supported by SMAC if tmp_list[1] == '=': # TODO maybe add a check if the hyperparameter is # actually in the configuration space clause_list.append( ForbiddenEqualsClause( configuration_space.get_hyperparameter( tmp_list[0]), tmp_list[2])) else: raise NotImplementedError() tmp_list = [] configuration_space.add_forbidden_clause( ForbiddenAndConjunction(*clause_list)) #Now handle conditions # If there are two conditions for one child, these two conditions are an # AND-conjunction of conditions, thus we have to connect them conditions_per_child = defaultdict(list) for condition in conditions: child_name = condition[0] conditions_per_child[child_name].append(condition) for child_name in conditions_per_child: condition_objects = [] for condition in conditions_per_child[child_name]: child = configuration_space.get_hyperparameter(child_name) parent_name = condition[2] parent = configuration_space.get_hyperparameter(parent_name) restrictions = condition[5:-1:2] # TODO: cast the type of the restriction! if len(restrictions) == 1: condition = EqualsCondition(child, parent, restrictions[0]) else: condition = InCondition(child, parent, values=restrictions) condition_objects.append(condition) # Now we have all condition objects for this child, so we can build a # giant AND-conjunction of them (if number of conditions >= 2)! if len(condition_objects) > 1: and_conjunction = AndConjunction(*condition_objects) configuration_space.add_condition(and_conjunction) else: configuration_space.add_condition(condition_objects[0]) return configuration_space
def test_check_configuration(self): # TODO this is only a smoke test # TODO actually, this rather tests the evaluate methods in the # conditions module! cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("parent", [0, 1]) cs.add_hyperparameter(hp1) hp2 = UniformIntegerHyperparameter("child", 0, 10) cs.add_hyperparameter(hp2) cond1 = EqualsCondition(hp2, hp1, 0) cs.add_condition(cond1) # This automatically checks the configuration! Configuration(cs, dict(parent=0, child=5)) # and now for something more complicated cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("input1", [0, 1]) cs.add_hyperparameter(hp1) hp2 = CategoricalHyperparameter("input2", [0, 1]) cs.add_hyperparameter(hp2) hp3 = CategoricalHyperparameter("input3", [0, 1]) cs.add_hyperparameter(hp3) hp4 = CategoricalHyperparameter("input4", [0, 1]) cs.add_hyperparameter(hp4) hp5 = CategoricalHyperparameter("input5", [0, 1]) cs.add_hyperparameter(hp5) hp6 = Constant("AND", "True") cs.add_hyperparameter(hp6) cond1 = EqualsCondition(hp6, hp1, 1) cond2 = NotEqualsCondition(hp6, hp2, 1) cond3 = InCondition(hp6, hp3, [1]) cond4 = EqualsCondition(hp6, hp4, 1) cond5 = EqualsCondition(hp6, hp5, 1) conj1 = AndConjunction(cond1, cond2) conj2 = OrConjunction(conj1, cond3) conj3 = AndConjunction(conj2, cond4, cond5) cs.add_condition(conj3) expected_outcomes = [ False, False, False, False, False, False, False, True, False, False, False, False, False, False, False, True, False, False, False, True, False, False, False, True, False, False, False, False, False, False, False, True ] for idx, values in enumerate(product([0, 1], repeat=5)): # The hyperparameters aren't sorted, but the test assumes them to # be sorted. hyperparameters = sorted(cs.get_hyperparameters(), key=lambda t: t.name) instantiations = { hyperparameters[jdx + 1].name: values[jdx] for jdx in range(len(values)) } evaluation = conj3.evaluate(instantiations) self.assertEqual(expected_outcomes[idx], evaluation) if evaluation == False: self.assertRaisesRegexp(ValueError, "Inactive hyperparameter 'AND' must " "not be specified, but has the value: " "'True'.", Configuration, cs, values={ "input1": values[0], "input2": values[1], "input3": values[2], "input4": values[3], "input5": values[4], "AND": "True" }) else: Configuration(cs, values={ "input1": values[0], "input2": values[1], "input3": values[2], "input4": values[3], "input5": values[4], "AND": "True" })
def read(pcs_string, debug=False): configuration_space = ConfigurationSpace() conditions = [] forbidden = [] # some statistics ct = 0 cont_ct = 0 cat_ct = 0 line_ct = 0 for line in pcs_string: line_ct += 1 if "#" in line: # It contains a comment pos = line.find("#") line = line[:pos] # Remove quotes and whitespaces at beginning and end line = line.replace('"', "").replace("'", "") line = line.strip() if "|" in line: # It's a condition try: c = pp_condition.parseString(line) conditions.append(c) except pyparsing.ParseException: raise NotImplementedError("Could not parse condition: %s" % line) continue if "}" not in line and "]" not in line: print "Skipping: %s" % line continue if line.startswith("{") and line.endswith("}"): forbidden.append(line) continue if len(line.strip()) == 0: continue ct += 1 param = None # print "Parsing: " + line create = { "int": UniformIntegerHyperparameter, "float": UniformFloatHyperparameter, "categorical": CategoricalHyperparameter } try: param_list = pp_cont_param.parseString(line) il = param_list[9:] if len(il) > 0: il = il[0] param_list = param_list[:9] name = param_list[0] lower = float(param_list[2]) upper = float(param_list[4]) paramtype = "int" if "i" in il else "float" log = True if "l" in il else False default = float(param_list[7]) param = create[paramtype](name=name, lower=lower, upper=upper, q=None, log=log, default=default) cont_ct += 1 except pyparsing.ParseException: pass try: param_list = pp_cat_param.parseString(line) name = param_list[0] choices = [c for c in param_list[2:-4:2]] default = param_list[-2] param = create["categorical"](name=name, choices=choices, default=default) cat_ct += 1 except pyparsing.ParseException: pass if param is None: raise NotImplementedError("Could not parse: %s" % line) configuration_space.add_hyperparameter(param) for clause in forbidden: # TODO test this properly! # TODO Add a try/catch here! # noinspection PyUnusedLocal param_list = pp_forbidden_clause.parseString(clause) tmp_list = [] clause_list = [] for value in param_list[1:]: if len(tmp_list) < 3: tmp_list.append(value) else: # So far, only equals is supported by SMAC if tmp_list[1] == '=': # TODO maybe add a check if the hyperparameter is # actually in the configuration space clause_list.append( ForbiddenEqualsClause( configuration_space.get_hyperparameter( tmp_list[0]), tmp_list[2])) else: raise NotImplementedError() tmp_list = [] configuration_space.add_forbidden_clause( ForbiddenAndConjunction(*clause_list)) #Now handle conditions for condition in conditions: child_name = condition[0] child = configuration_space.get_hyperparameter(child_name) parent_name = condition[2] parent = configuration_space.get_hyperparameter(parent_name) restrictions = condition[5:-1:2] # TODO: cast the type of the restriction! if len(restrictions) == 1: condition = EqualsCondition(child, parent, restrictions[0]) else: condition = InCondition(child, parent, values=restrictions) configuration_space.add_condition(condition) if debug: print print "============== Reading Results" print "First 10 lines:" sp_list = ["%s: %s" % (j, str(searchspace[j])) for j in searchspace] print "\n".join(sp_list[:10]) print print "#Invalid lines: %d ( of %d )" % (line_ct - len(conditions) - ct, line_ct) print "#Parameter: %d" % len(searchspace) print "#Conditions: %d" % len(conditions) print "#Conditioned params: %d" % sum([ 1 if len(searchspace[j].conditions[0]) > 0 else 0 for j in searchspace ]) print "#Categorical: %d" % cat_ct print "#Continuous: %d" % cont_ct return configuration_space