def test_uniformfloat_to_integer(self): f1 = UniformFloatHyperparameter("param", 1, 10, q=0.1, log=True) with warnings.catch_warnings(): f2 = f1.to_integer() warnings.simplefilter("ignore") # TODO is this a useful rounding? # TODO should there be any rounding, if e.g. lower=0.1 self.assertEqual("param, Type: UniformInteger, Range: [1, 10], " "Default: 3, on log-scale", str(f2))
def get_hyperparameter_search_space(dataset_properties=None): C = UniformFloatHyperparameter("C", 0.03125, 32768, log=True, default=1.0) # No linear kernel here, because we have liblinear kernel = CategoricalHyperparameter(name="kernel", choices=["rbf", "poly", "sigmoid"], default="rbf") degree = UniformIntegerHyperparameter("degree", 1, 5, default=3) gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8, log=True, default=0.1) # TODO this is totally ad-hoc coef0 = UniformFloatHyperparameter("coef0", -1, 1, default=0) # probability is no hyperparameter, but an argument to the SVM algo shrinking = CategoricalHyperparameter("shrinking", ["True", "False"], default="True") tol = UniformFloatHyperparameter("tol", 1e-5, 1e-1, default=1e-4, log=True) # cache size is not a hyperparameter, but an argument to the program! max_iter = UnParametrizedHyperparameter("max_iter", -1) cs = ConfigurationSpace() cs.add_hyperparameter(C) cs.add_hyperparameter(kernel) cs.add_hyperparameter(degree) cs.add_hyperparameter(gamma) cs.add_hyperparameter(coef0) cs.add_hyperparameter(shrinking) cs.add_hyperparameter(tol) cs.add_hyperparameter(max_iter) degree_depends_on_poly = EqualsCondition(degree, kernel, "poly") coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"]) cs.add_condition(degree_depends_on_poly) cs.add_condition(coef0_condition) return cs
def get_hyperparameter_search_space(dataset_properties=None): gamma = UniformFloatHyperparameter( "gamma", 0.3, 2., default=1.0) n_components = UniformIntegerHyperparameter( "n_components", 50, 10000, default=100, log=True) cs = ConfigurationSpace() cs.add_hyperparameter(gamma) cs.add_hyperparameter(n_components) return cs
def get_hyperparameter_search_space(dataset_properties=None): keep_variance = UniformFloatHyperparameter( "keep_variance", 0.5, 0.9999, default=0.9999) whiten = CategoricalHyperparameter( "whiten", ["False", "True"], default="False") cs = ConfigurationSpace() cs.add_hyperparameter(keep_variance) cs.add_hyperparameter(whiten) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() loss = cs.add_hyperparameter(CategoricalHyperparameter("loss", ["hinge", "log", "modified_huber", "squared_hinge", "perceptron"], default="hinge")) penalty = cs.add_hyperparameter(CategoricalHyperparameter( "penalty", ["l1", "l2", "elasticnet"], default="l2")) alpha = cs.add_hyperparameter(UniformFloatHyperparameter( "alpha", 10e-7, 1e-1, log=True, default=0.0001)) l1_ratio = cs.add_hyperparameter(UniformFloatHyperparameter( "l1_ratio", 0, 1, default=0.15)) fit_intercept = cs.add_hyperparameter(UnParametrizedHyperparameter( "fit_intercept", "True")) n_iter = cs.add_hyperparameter(UniformIntegerHyperparameter( "n_iter", 5, 1000, default=20)) epsilon = cs.add_hyperparameter(UniformFloatHyperparameter( "epsilon", 1e-5, 1e-1, default=1e-4, log=True)) learning_rate = cs.add_hyperparameter(CategoricalHyperparameter( "learning_rate", ["optimal", "invscaling", "constant"], default="optimal")) eta0 = cs.add_hyperparameter(UniformFloatHyperparameter( "eta0", 10**-7, 0.1, default=0.01)) power_t = cs.add_hyperparameter(UniformFloatHyperparameter( "power_t", 1e-5, 1, default=0.25)) average = cs.add_hyperparameter(CategoricalHyperparameter( "average", ["False", "True"], default="False")) # TODO add passive/aggressive here, although not properly documented? elasticnet = EqualsCondition(l1_ratio, penalty, "elasticnet") epsilon_condition = EqualsCondition(epsilon, loss, "modified_huber") # eta0 seems to be always active according to the source code; when # learning_rate is set to optimial, eta0 is the starting value: # https://github.com/scikit-learn/scikit-learn/blob/0.15.X/sklearn/linear_model/sgd_fast.pyx #eta0_and_inv = EqualsCondition(eta0, learning_rate, "invscaling") #eta0_and_constant = EqualsCondition(eta0, learning_rate, "constant") #eta0_condition = OrConjunction(eta0_and_inv, eta0_and_constant) power_t_condition = EqualsCondition(power_t, learning_rate, "invscaling") cs.add_condition(elasticnet) cs.add_condition(epsilon_condition) cs.add_condition(power_t_condition) return cs
def get_hyperparameter_search_space(dataset_properties=None): alpha = UniformFloatHyperparameter(name="alpha", lower=0.0001, upper=10, default=1.0, log=True) cs = ConfigurationSpace() cs.add_hyperparameter(alpha) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() penalty = cs.add_hyperparameter( CategoricalHyperparameter("penalty", ["l1", "l2"], default="l2")) loss = cs.add_hyperparameter( CategoricalHyperparameter("loss", ["hinge", "squared_hinge"], default="squared_hinge")) dual = cs.add_hyperparameter(Constant("dual", "False")) # This is set ad-hoc tol = cs.add_hyperparameter( UniformFloatHyperparameter("tol", 1e-5, 1e-1, default=1e-4, log=True)) C = cs.add_hyperparameter( UniformFloatHyperparameter("C", 0.03125, 32768, log=True, default=1.0)) multi_class = cs.add_hyperparameter(Constant("multi_class", "ovr")) # These are set ad-hoc fit_intercept = cs.add_hyperparameter(Constant("fit_intercept", "True")) intercept_scaling = cs.add_hyperparameter( Constant("intercept_scaling", 1)) penalty_and_loss = ForbiddenAndConjunction( ForbiddenEqualsClause(penalty, "l1"), ForbiddenEqualsClause(loss, "hinge")) constant_penalty_and_loss = ForbiddenAndConjunction( ForbiddenEqualsClause(dual, "False"), ForbiddenEqualsClause(penalty, "l2"), ForbiddenEqualsClause(loss, "hinge")) penalty_and_dual = ForbiddenAndConjunction( ForbiddenEqualsClause(dual, "False"), ForbiddenEqualsClause(penalty, "l1")) cs.add_forbidden_clause(penalty_and_loss) cs.add_forbidden_clause(constant_penalty_and_loss) cs.add_forbidden_clause(penalty_and_dual) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() shrinkage = cs.add_hyperparameter( CategoricalHyperparameter("shrinkage", ["None", "auto", "manual"], default="None")) shrinkage_factor = cs.add_hyperparameter( UniformFloatHyperparameter("shrinkage_factor", 0., 1., 0.5)) n_components = cs.add_hyperparameter( UniformIntegerHyperparameter('n_components', 1, 250, default=10)) tol = cs.add_hyperparameter( UniformFloatHyperparameter("tol", 1e-5, 1e-1, default=1e-4, log=True)) cs.add_condition(EqualsCondition(shrinkage_factor, shrinkage, "manual")) return cs
def get_hyperparameter_search_space(dataset_properties=None): if dataset_properties is not None and \ (dataset_properties.get("sparse") is True or dataset_properties.get("signed") is False): allow_chi2 = False else: allow_chi2 = True possible_kernels = ['poly', 'rbf', 'sigmoid', 'cosine'] if allow_chi2: possible_kernels.append("chi2") kernel = CategoricalHyperparameter('kernel', possible_kernels, 'rbf') degree = UniformIntegerHyperparameter('degree', 2, 5, 3) gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8, log=True, default=0.1) coef0 = UniformFloatHyperparameter("coef0", -1, 1, default=0) n_components = UniformIntegerHyperparameter("n_components", 50, 10000, default=100, log=True) cs = ConfigurationSpace() cs.add_hyperparameter(kernel) cs.add_hyperparameter(degree) cs.add_hyperparameter(gamma) cs.add_hyperparameter(coef0) cs.add_hyperparameter(n_components) degree_depends_on_poly = EqualsCondition(degree, kernel, "poly") coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"]) gamma_kernels = ["poly", "rbf", "sigmoid"] if allow_chi2: gamma_kernels.append("chi2") gamma_condition = InCondition(gamma, kernel, gamma_kernels) cs.add_condition(degree_depends_on_poly) cs.add_condition(coef0_condition) cs.add_condition(gamma_condition) return cs
def get_hyperparameter_search_space(dataset_properties=None): loss = CategoricalHyperparameter( name="loss", choices=["ls", "lad"], default='ls') #, "huber", "quantile"], default='ls') learning_rate = UniformFloatHyperparameter(name="learning_rate", lower=0.0001, upper=1, default=0.1, log=True) subsample = UniformFloatHyperparameter(name="subsample", lower=0.01, upper=1.0, default=1.0, log=False) n_estimators = Constant("n_estimators", 100) max_features = UniformFloatHyperparameter("max_features", 0.5, 5, default=1) max_depth = UniformIntegerHyperparameter(name="max_depth", lower=1, upper=10, default=3) min_samples_split = UniformIntegerHyperparameter( name="min_samples_split", lower=2, upper=20, default=2, log=False) min_samples_leaf = UniformIntegerHyperparameter( name="min_samples_leaf", lower=1, upper=20, default=1, log=False) cs = ConfigurationSpace() cs.add_hyperparameter(n_estimators) cs.add_hyperparameter(loss) cs.add_hyperparameter(learning_rate) cs.add_hyperparameter(max_features) cs.add_hyperparameter(max_depth) cs.add_hyperparameter(min_samples_split) cs.add_hyperparameter(min_samples_leaf) cs.add_hyperparameter(subsample) return cs
def get_hyperparameter_search_space(dataset_properties=None): percentile = UniformFloatHyperparameter( "percentile", lower=1, upper=99, default=50) score_func = UnParametrizedHyperparameter( name="score_func", value="f_regression") cs = ConfigurationSpace() cs.add_hyperparameter(percentile) cs.add_hyperparameter(score_func) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() loss = cs.add_hyperparameter(Constant("loss", "deviance")) learning_rate = cs.add_hyperparameter( UniformFloatHyperparameter(name="learning_rate", lower=0.0001, upper=1, default=0.1, log=True)) n_estimators = cs.add_hyperparameter(Constant("n_estimators", 100)) max_depth = cs.add_hyperparameter( UniformIntegerHyperparameter(name="max_depth", lower=1, upper=10, default=3)) min_samples_split = cs.add_hyperparameter( UniformIntegerHyperparameter(name="min_samples_split", lower=2, upper=20, default=2, log=False)) min_samples_leaf = cs.add_hyperparameter( UniformIntegerHyperparameter(name="min_samples_leaf", lower=1, upper=20, default=1, log=False)) min_weight_fraction_leaf = cs.add_hyperparameter( UnParametrizedHyperparameter("min_weight_fraction_leaf", 0.)) subsample = cs.add_hyperparameter( UniformFloatHyperparameter(name="subsample", lower=0.01, upper=1.0, default=1.0, log=False)) max_features = cs.add_hyperparameter( UniformFloatHyperparameter("max_features", 0.5, 5, default=1)) max_leaf_nodes = cs.add_hyperparameter( UnParametrizedHyperparameter(name="max_leaf_nodes", value="None")) return cs
def _lr_policy_configuration_space(cs, policy=None): if policy == 'inv': lr_policy = Constant(name='lr_policy', value='inv') gamma = UniformFloatHyperparameter(name="gamma", lower=1e-2, upper=1.0, log=True, default=1e-2) power = UniformFloatHyperparameter("power", 0.0, 1.0, default=0.5) cs.add_hyperparameter(lr_policy) cs.add_hyperparameter(gamma) cs.add_hyperparameter(power) elif policy == 'exp': lr_policy = Constant(name='lr_policy', value='exp') gamma = UniformFloatHyperparameter(name="gamma", lower=0.7, upper=1.0, default=0.79) cs.add_hyperparameter(lr_policy) cs.add_hyperparameter(gamma) elif policy == 'step': lr_policy = Constant(name='lr_policy', value='step') gamma = UniformFloatHyperparameter(name="gamma", lower=1e-2, upper=1.0, log=True, default=1e-2) epoch_step = CategoricalHyperparameter("epoch_step", [6, 8, 12], default=8) cs.add_hyperparameter(lr_policy) cs.add_hyperparameter(gamma) cs.add_hyperparameter(epoch_step) else: lr_policy = Constant(name='lr_policy', value='fixed') cs.add_hyperparameter(lr_policy) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConstrainedFeedNet.get_hyperparameter_search_space() solver = Constant(name='solver', value='momentum') cs.add_hyperparameter(solver) momentum = UniformFloatHyperparameter("momentum", 0.3, 0.999, default=0.9) cs.add_hyperparameter(momentum) return cs
def get_hyperparameter_search_space(dataset_properties=None): loss = CategoricalHyperparameter("loss", ["hinge", "squared_hinge"], default="hinge") fit_intercept = UnParametrizedHyperparameter("fit_intercept", "True") n_iter = UniformIntegerHyperparameter("n_iter", 5, 1000, default=20) C = UniformFloatHyperparameter("C", 1e-5, 10, 1, log=True) cs = ConfigurationSpace() cs.add_hyperparameter(loss) cs.add_hyperparameter(fit_intercept) cs.add_hyperparameter(n_iter) cs.add_hyperparameter(C) return cs
def get_hyperparameter_search_space(dataset_properties=None): nugget = UniformFloatHyperparameter(name="nugget", lower=0.0001, upper=10, default=0.1, log=True) thetaL = UniformFloatHyperparameter(name="thetaL", lower=1e-6, upper=1e-3, default=1e-4, log=True) thetaU = UniformFloatHyperparameter(name="thetaU", lower=0.2, upper=10, default=1.0, log=True) cs = ConfigurationSpace() cs.add_hyperparameter(nugget) cs.add_hyperparameter(thetaL) cs.add_hyperparameter(thetaU) return cs
def test_uniformfloat_is_legal(self): f1 = UniformFloatHyperparameter("param", 0.1, 10, q=0.1, log=True) self.assertTrue(f1.is_legal(3.0)) self.assertTrue(f1.is_legal(3)) self.assertFalse(f1.is_legal(-0.1)) self.assertFalse(f1.is_legal(10.1)) self.assertFalse(f1.is_legal("AAA")) self.assertFalse(f1.is_legal(dict()))
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() C = cs.add_hyperparameter( UniformFloatHyperparameter("C", 0.03125, 32768, log=True, default=1.0)) loss = cs.add_hyperparameter( CategoricalHyperparameter( "loss", ["epsilon_insensitive", "squared_epsilon_insensitive"], default="squared_epsilon_insensitive")) # Random Guess epsilon = cs.add_hyperparameter( UniformFloatHyperparameter(name="epsilon", lower=0.001, upper=1, default=0.1, log=True)) dual = cs.add_hyperparameter(Constant("dual", "False")) # These are set ad-hoc tol = cs.add_hyperparameter( UniformFloatHyperparameter("tol", 1e-5, 1e-1, default=1e-4, log=True)) fit_intercept = cs.add_hyperparameter(Constant("fit_intercept", "True")) intercept_scaling = cs.add_hyperparameter( Constant("intercept_scaling", 1)) dual_and_loss = ForbiddenAndConjunction( ForbiddenEqualsClause(dual, "False"), ForbiddenEqualsClause(loss, "epsilon_insensitive")) cs.add_forbidden_clause(dual_and_loss) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() # base_estimator = Constant(name="base_estimator", value="None") n_estimators = cs.add_hyperparameter(UniformIntegerHyperparameter( name="n_estimators", lower=50, upper=500, default=50, log=False)) learning_rate = cs.add_hyperparameter(UniformFloatHyperparameter( name="learning_rate", lower=0.0001, upper=2, default=0.1, log=True)) loss = cs.add_hyperparameter(CategoricalHyperparameter( name="loss", choices=["linear", "square", "exponential"], default="linear")) max_depth = cs.add_hyperparameter(UniformIntegerHyperparameter( name="max_depth", lower=1, upper=10, default=1, log=False)) return cs
def get_hyperparameter_search_space(dataset_properties=None): # Constrain the search space of the DeepFeedNet # Fixed Architecture for MNIST batch_size = Constant(name='batch_size', value=963) # TODO: Decimal library to work around floating point issue dropout_layer_1 = Constant(name='dropout_layer_1', value=0.39426633933) dropout_output = Constant(name='dropout_output', value=0.085813712701) num_layers = Constant(name='num_layers', value='c') num_units_layer_1 = Constant(name='num_units_layer_1', value=1861) number_updates = Constant(name='number_updates', value=1105) std_layer_1 = Constant(name='std_layer_1', value=0.00351015701) # To Optimize for all cases l2 = UniformFloatHyperparameter("lambda2", 1e-6, 1e-2, log=True, default=1e-3) lr = UniformFloatHyperparameter("learning_rate", 1e-4, 1e-1, log=True, default=1e-2) cs = ConfigurationSpace() cs.add_hyperparameter(batch_size) cs.add_hyperparameter(number_updates) cs.add_hyperparameter(num_layers) cs.add_hyperparameter(num_units_layer_1) cs.add_hyperparameter(dropout_layer_1) cs.add_hyperparameter(dropout_output) cs.add_hyperparameter(std_layer_1) cs.add_hyperparameter(lr) cs.add_hyperparameter(l2) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() n_estimators = cs.add_hyperparameter(Constant("n_estimators", 100)) criterion = cs.add_hyperparameter(Constant("criterion", "mse")) max_features = cs.add_hyperparameter( UniformFloatHyperparameter("max_features", 0.5, 5, default=1)) max_depth = cs.add_hyperparameter( UnParametrizedHyperparameter(name="max_depth", value="None")) min_samples_split = cs.add_hyperparameter( UniformIntegerHyperparameter("min_samples_split", 2, 20, default=2)) min_samples_leaf = cs.add_hyperparameter( UniformIntegerHyperparameter("min_samples_leaf", 1, 20, default=1)) # Unparametrized, we use min_samples as regularization # max_leaf_nodes_or_max_depth = UnParametrizedHyperparameter( # name="max_leaf_nodes_or_max_depth", value="max_depth") # CategoricalHyperparameter("max_leaf_nodes_or_max_depth", # choices=["max_leaf_nodes", "max_depth"], default="max_depth") # min_weight_fraction_leaf = UniformFloatHyperparameter( # "min_weight_fraction_leaf", 0.0, 0.1) # max_leaf_nodes = UnParametrizedHyperparameter(name="max_leaf_nodes", # value="None") bootstrap = cs.add_hyperparameter( CategoricalHyperparameter("bootstrap", ["True", "False"], default="False")) # Conditions # Not applicable because max_leaf_nodes is no legal value of the parent #cond_max_leaf_nodes_or_max_depth = \ # EqualsCondition(child=max_leaf_nodes, # parent=max_leaf_nodes_or_max_depth, # value="max_leaf_nodes") #cond2_max_leaf_nodes_or_max_depth = \ # EqualsCondition(child=use_max_depth, # parent=max_leaf_nodes_or_max_depth, # value="max_depth") #cond_max_depth = EqualsCondition(child=max_depth, parent=use_max_depth, #value="True") #cs.add_condition(cond_max_leaf_nodes_or_max_depth) #cs.add_condition(cond2_max_leaf_nodes_or_max_depth) #cs.add_condition(cond_max_depth) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() use_minimum_fraction = cs.add_hyperparameter( CategoricalHyperparameter("use_minimum_fraction", ["True", "False"], default="True")) minimum_fraction = cs.add_hyperparameter( UniformFloatHyperparameter("minimum_fraction", lower=.0001, upper=0.5, default=0.01, log=True)) cs.add_condition( EqualsCondition(minimum_fraction, use_minimum_fraction, 'True')) return cs
def test_not_equals_condition_illegal_value(self): epsilon = UniformFloatHyperparameter("epsilon", 1e-5, 1e-1, default=1e-4, log=True) loss = CategoricalHyperparameter( "loss", ["hinge", "log", "modified_huber", "squared_hinge", "perceptron"], default="hinge") self.assertRaisesRegexp( ValueError, "Hyperparameter 'epsilon' is " "conditional on the illegal value 'huber' of " "its parent hyperparameter 'loss'", NotEqualsCondition, epsilon, loss, "huber")
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() cs.add_hyperparameter(Constant("n_estimators", 100)) cs.add_hyperparameter(Constant("criterion", "mse")) cs.add_hyperparameter(UniformFloatHyperparameter( "max_features", 0.5, 5, default=1)) cs.add_hyperparameter(UnParametrizedHyperparameter("max_depth", "None")) cs.add_hyperparameter(UniformIntegerHyperparameter( "min_samples_split", 2, 20, default=2)) cs.add_hyperparameter(UniformIntegerHyperparameter( "min_samples_leaf", 1, 20, default=1)) cs.add_hyperparameter( UnParametrizedHyperparameter("min_weight_fraction_leaf", 0.)) cs.add_hyperparameter(UnParametrizedHyperparameter("max_leaf_nodes", "None")) cs.add_hyperparameter(CategoricalHyperparameter( "bootstrap", ["True", "False"], default="True")) return cs
def get_hyperparameter_search_space(dataset_properties=None): percentile = UniformFloatHyperparameter( name="percentile", lower=10, upper=90, default=50) score_func = CategoricalHyperparameter( name="score_func", choices=["chi2", "f_classif"], default="chi2") if dataset_properties is not None: # Chi2 can handle sparse data, so we respect this if 'sparse' in dataset_properties and dataset_properties['sparse']: score_func = Constant( name="score_func", value="chi2") cs = ConfigurationSpace() cs.add_hyperparameter(percentile) cs.add_hyperparameter(score_func) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() criterion = cs.add_hyperparameter(Constant('criterion', 'mse')) splitter = cs.add_hyperparameter(Constant("splitter", "best")) max_features = cs.add_hyperparameter(Constant('max_features', 1.0)) max_depth = cs.add_hyperparameter(UniformFloatHyperparameter( 'max_depth', 0., 2., default=0.5)) min_samples_split = cs.add_hyperparameter(UniformIntegerHyperparameter( "min_samples_split", 2, 20, default=2)) min_samples_leaf = cs.add_hyperparameter(UniformIntegerHyperparameter( "min_samples_leaf", 1, 20, default=1)) min_weight_fraction_leaf = cs.add_hyperparameter( Constant("min_weight_fraction_leaf", 0.0)) max_leaf_nodes = cs.add_hyperparameter( UnParametrizedHyperparameter("max_leaf_nodes", "None")) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() # the smoothing parameter is a non-negative float # I will limit it to 1000 and put it on a logarithmic scale. (SF) # Please adjust that, if you know a proper range, this is just a guess. alpha = UniformFloatHyperparameter(name="alpha", lower=1e-2, upper=100, default=1, log=True) fit_prior = CategoricalHyperparameter(name="fit_prior", choices=["True", "False"], default="True") cs.add_hyperparameter(alpha) cs.add_hyperparameter(fit_prior) return cs
def read(pb_string): searchspace = dict() if type(pb_string) == file: pb_string = pb_string.read() if not isinstance(pb_string, str): raise ValueError("Input is not a string or a file") pb_space = Experiment() text_format.Merge(pb_string, pb_space) for para in pb_space.variable: name = str(para.name) size = para.size if size != 1: raise NotImplementedError( "%s has size %s, we only convert with size 1" % (name, size)) if para.type == Experiment.ParameterSpec.ENUM: choices = list() for ch in para.options: choices.append(str(ch)) searchspace[name] = CategoricalHyperparameter(name=name, choices=choices, conditions=None) elif para.type == Experiment.ParameterSpec.FLOAT: searchspace[name] = UniformFloatHyperparameter(name=name, lower=para.min, upper=para.max, base=None, q=None, conditions=None) elif para.type == Experiment.ParameterSpec.INT: searchspace[name] = UniformIntegerHyperparameter(name=name, lower=para.min, upper=para.max, base=None, q=None, conditions=None) else: raise NotImplementedError("Don't know that type: %s (%s)" % (type(para), para.name)) return searchspace
def get_hyperparameter_search_space(dataset_properties=None): alpha = UniformFloatHyperparameter(name="alpha", lower=0.01, upper=0.5, default=0.1) score_func = CategoricalHyperparameter(name="score_func", choices=["chi2", "f_classif"], default="chi2") if dataset_properties is not None: # Chi2 can handle sparse data, so we respect this if 'sparse' in dataset_properties and dataset_properties['sparse']: score_func = Constant(name="score_func", value="chi2") mode = CategoricalHyperparameter('mode', ['fpr', 'fdr', 'fwe'], 'fpr') cs = ConfigurationSpace() cs.add_hyperparameter(alpha) cs.add_hyperparameter(score_func) cs.add_hyperparameter(mode) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() n_iter = cs.add_hyperparameter( UnParametrizedHyperparameter("n_iter", value=300)) tol = cs.add_hyperparameter( UniformFloatHyperparameter("tol", 10**-5, 10**-1, default=10**-4, log=True)) alpha_1 = cs.add_hyperparameter( UniformFloatHyperparameter(name="alpha_1", lower=10**-10, upper=10**-3, default=10**-6)) alpha_2 = cs.add_hyperparameter( UniformFloatHyperparameter(name="alpha_2", log=True, lower=10**-10, upper=10**-3, default=10**-6)) lambda_1 = cs.add_hyperparameter( UniformFloatHyperparameter(name="lambda_1", log=True, lower=10**-10, upper=10**-3, default=10**-6)) lambda_2 = cs.add_hyperparameter( UniformFloatHyperparameter(name="lambda_2", log=True, lower=10**-10, upper=10**-3, default=10**-6)) threshold_lambda = cs.add_hyperparameter( UniformFloatHyperparameter(name="threshold_lambda", log=True, lower=10**3, upper=10**5, default=10**4)) fit_intercept = cs.add_hyperparameter( UnParametrizedHyperparameter("fit_intercept", "True")) return cs
def get_hyperparameter_search_space(dataset_properties=None): max_num_layers = 7 # Maximum number of layers coded # Hacky way to condition layers params based on the number of layers # 'c'=1, 'd'=2, 'e'=3 ,'f'=4', g ='5', h='6' + output_layer layer_choices = [ chr(i) for i in range(ord('c'), ord('b') + max_num_layers) ] batch_size = UniformIntegerHyperparameter("batch_size", 32, 4096, log=True, default=32) number_epochs = UniformIntegerHyperparameter("number_epochs", 2, 80, default=5) num_layers = CategoricalHyperparameter("num_layers", choices=layer_choices, default='c') lr = UniformFloatHyperparameter("learning_rate", 1e-6, 1.0, log=True, default=0.01) l2 = UniformFloatHyperparameter("lambda2", 1e-7, 1e-2, log=True, default=1e-4) dropout_output = UniformFloatHyperparameter("dropout_output", 0.0, 0.99, default=0.5) # Define basic hyperparameters and define the config space # basic means that are independent from the number of layers cs = ConfigurationSpace() cs.add_hyperparameter(number_epochs) cs.add_hyperparameter(batch_size) cs.add_hyperparameter(num_layers) cs.add_hyperparameter(lr) cs.add_hyperparameter(l2) cs.add_hyperparameter(dropout_output) # Define parameters with different child parameters and conditions solver_choices = [ "adam", "adadelta", "adagrad", "sgd", "momentum", "nesterov", "smorm3s" ] solver = CategoricalHyperparameter(name="solver", choices=solver_choices, default="smorm3s") beta1 = UniformFloatHyperparameter("beta1", 1e-4, 0.1, log=True, default=0.1) beta2 = UniformFloatHyperparameter("beta2", 1e-4, 0.1, log=True, default=0.01) rho = UniformFloatHyperparameter("rho", 0.05, 0.99, log=True, default=0.95) momentum = UniformFloatHyperparameter("momentum", 0.3, 0.999, default=0.9) # TODO: Add policy based on this sklearn sgd policy_choices = ['fixed', 'inv', 'exp', 'step'] lr_policy = CategoricalHyperparameter(name="lr_policy", choices=policy_choices, default='fixed') gamma = UniformFloatHyperparameter(name="gamma", lower=1e-3, upper=1e-1, default=1e-2) power = UniformFloatHyperparameter("power", 0.0, 1.0, default=0.5) epoch_step = UniformIntegerHyperparameter("epoch_step", 2, 20, default=5) cs.add_hyperparameter(solver) cs.add_hyperparameter(beta1) cs.add_hyperparameter(beta2) cs.add_hyperparameter(momentum) cs.add_hyperparameter(rho) cs.add_hyperparameter(lr_policy) cs.add_hyperparameter(gamma) cs.add_hyperparameter(power) cs.add_hyperparameter(epoch_step) # Define parameters that are needed it for each layer output_activation_choices = ['softmax', 'sigmoid', 'softplus', 'tanh'] activations_choices = [ 'sigmoid', 'tanh', 'scaledTanh', 'elu', 'relu', 'leaky', 'linear' ] weight_choices = [ 'constant', 'normal', 'uniform', 'glorot_normal', 'glorot_uniform', 'he_normal', 'he_uniform', 'ortogonal', 'sparse' ] # Iterate over parameters that are used in each layer for i in range(1, max_num_layers): layer_units = UniformIntegerHyperparameter("num_units_layer_" + str(i), 64, 4096, log=True, default=128) cs.add_hyperparameter(layer_units) layer_dropout = UniformFloatHyperparameter("dropout_layer_" + str(i), 0.0, 0.99, default=0.5) cs.add_hyperparameter(layer_dropout) weight_initialization = CategoricalHyperparameter( 'weight_init_' + str(i), choices=weight_choices, default='he_normal') cs.add_hyperparameter(weight_initialization) layer_std = UniformFloatHyperparameter("std_layer_" + str(i), 1e-6, 0.1, log=True, default=0.005) cs.add_hyperparameter(layer_std) layer_activation = CategoricalHyperparameter( "activation_layer_" + str(i), choices=activations_choices, default="relu") cs.add_hyperparameter(layer_activation) layer_leakiness = UniformFloatHyperparameter('leakiness_layer_' + str(i), 0.01, 0.99, default=0.3) cs.add_hyperparameter(layer_leakiness) layer_tanh_alpha = UniformFloatHyperparameter('tanh_alpha_layer_' + str(i), 0.5, 1.0, default=2. / 3.) cs.add_hyperparameter(layer_tanh_alpha) layer_tanh_beta = UniformFloatHyperparameter('tanh_beta_layer_' + str(i), 1.1, 3.0, log=True, default=1.7159) cs.add_hyperparameter(layer_tanh_beta) # TODO: Could be in a function in a new module for i in range(2, max_num_layers): # Condition layers parameter on layer choice layer_unit_param = cs.get_hyperparameter("num_units_layer_" + str(i)) layer_cond = InCondition(child=layer_unit_param, parent=num_layers, values=[l for l in layer_choices[i - 1:]]) cs.add_condition(layer_cond) # Condition dropout parameter on layer choice layer_dropout_param = cs.get_hyperparameter("dropout_layer_" + str(i)) layer_cond = InCondition(child=layer_dropout_param, parent=num_layers, values=[l for l in layer_choices[i - 1:]]) cs.add_condition(layer_cond) # Condition weight initialization on layer choice layer_weight_param = cs.get_hyperparameter("weight_init_" + str(i)) layer_cond = InCondition(child=layer_weight_param, parent=num_layers, values=[l for l in layer_choices[i - 1:]]) cs.add_condition(layer_cond) # Condition std parameter on weight layer initialization choice layer_std_param = cs.get_hyperparameter("std_layer_" + str(i)) weight_cond = EqualsCondition(child=layer_std_param, parent=layer_weight_param, value='normal') cs.add_condition(weight_cond) # Condition activation parameter on layer choice layer_activation_param = cs.get_hyperparameter( "activation_layer_" + str(i)) layer_cond = InCondition(child=layer_activation_param, parent=num_layers, values=[l for l in layer_choices[i - 1:]]) cs.add_condition(layer_cond) # Condition leakiness on activation choice layer_leakiness_param = cs.get_hyperparameter("leakiness_layer_" + str(i)) activation_cond = EqualsCondition(child=layer_leakiness_param, parent=layer_activation_param, value='leaky') cs.add_condition(activation_cond) # Condition tanh on activation choice layer_tanh_alpha_param = cs.get_hyperparameter( "tanh_alpha_layer_" + str(i)) activation_cond = EqualsCondition(child=layer_tanh_alpha_param, parent=layer_activation_param, value='scaledTanh') cs.add_condition(activation_cond) layer_tanh_beta_param = cs.get_hyperparameter("tanh_beta_layer_" + str(i)) activation_cond = EqualsCondition(child=layer_tanh_beta_param, parent=layer_activation_param, value='scaledTanh') cs.add_condition(activation_cond) # Conditioning on solver momentum_depends_on_solver = InCondition( momentum, solver, values=["momentum", "nesterov"]) beta1_depends_on_solver = EqualsCondition(beta1, solver, "adam") beta2_depends_on_solver = EqualsCondition(beta2, solver, "adam") rho_depends_on_solver = EqualsCondition(rho, solver, "adadelta") cs.add_condition(momentum_depends_on_solver) cs.add_condition(beta1_depends_on_solver) cs.add_condition(beta2_depends_on_solver) cs.add_condition(rho_depends_on_solver) # Conditioning on learning rate policy lr_policy_depends_on_solver = InCondition( lr_policy, solver, ["adadelta", "adagrad", "sgd", "momentum", "nesterov"]) gamma_depends_on_policy = InCondition(child=gamma, parent=lr_policy, values=["inv", "exp", "step"]) power_depends_on_policy = EqualsCondition(power, lr_policy, "inv") epoch_step_depends_on_policy = EqualsCondition(epoch_step, lr_policy, "step") cs.add_condition(lr_policy_depends_on_solver) cs.add_condition(gamma_depends_on_policy) cs.add_condition(power_depends_on_policy) cs.add_condition(epoch_step_depends_on_policy) return cs
def get_hyperparameter_search_space(dataset_properties=None): # Copied from libsvm_c C = UniformFloatHyperparameter(name="C", lower=0.03125, upper=32768, log=True, default=1.0) kernel = CategoricalHyperparameter( name="kernel", choices=['linear', 'poly', 'rbf', 'sigmoid'], default="rbf") degree = UniformIntegerHyperparameter(name="degree", lower=1, upper=5, default=3) # Changed the gamma value to 0.0 (is 0.1 for classification) gamma = UniformFloatHyperparameter(name="gamma", lower=3.0517578125e-05, upper=8, log=True, default=0.1) # TODO this is totally ad-hoc coef0 = UniformFloatHyperparameter(name="coef0", lower=-1, upper=1, default=0) # probability is no hyperparameter, but an argument to the SVM algo shrinking = CategoricalHyperparameter(name="shrinking", choices=["True", "False"], default="True") tol = UniformFloatHyperparameter(name="tol", lower=1e-5, upper=1e-1, default=1e-3, log=True) max_iter = UnParametrizedHyperparameter("max_iter", -1) # Random Guess epsilon = UniformFloatHyperparameter(name="epsilon", lower=0.001, upper=1, default=0.1, log=True) cs = ConfigurationSpace() cs.add_hyperparameter(C) cs.add_hyperparameter(kernel) cs.add_hyperparameter(degree) cs.add_hyperparameter(gamma) cs.add_hyperparameter(coef0) cs.add_hyperparameter(shrinking) cs.add_hyperparameter(tol) cs.add_hyperparameter(max_iter) cs.add_hyperparameter(epsilon) degree_depends_on_kernel = InCondition(child=degree, parent=kernel, values=('poly', 'rbf', 'sigmoid')) gamma_depends_on_kernel = InCondition(child=gamma, parent=kernel, values=('poly', 'rbf')) coef0_depends_on_kernel = InCondition(child=coef0, parent=kernel, values=('poly', 'sigmoid')) cs.add_condition(degree_depends_on_kernel) cs.add_condition(gamma_depends_on_kernel) cs.add_condition(coef0_depends_on_kernel) return cs