def get_hyperparameter_search_space(dataset_properties=None): n_components = UniformIntegerHyperparameter("n_components", 10, 2000, default_value=100) kernel = CategoricalHyperparameter( 'kernel', ['poly', 'rbf', 'sigmoid', 'cosine'], 'rbf') gamma = UniformFloatHyperparameter( "gamma", 3.0517578125e-05, 8, log=True, default_value=0.01, ) degree = UniformIntegerHyperparameter('degree', 2, 5, 3) coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0) cs = ConfigurationSpace() cs.add_hyperparameters([n_components, kernel, degree, gamma, coef0]) degree_depends_on_poly = EqualsCondition(degree, kernel, "poly") coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"]) gamma_condition = InCondition(gamma, kernel, ["poly", "rbf"]) cs.add_conditions( [degree_depends_on_poly, coef0_condition, gamma_condition]) return cs
def create_hyperspace(): cs = ConfigurationSpace() # training hyperparameters. learning_rate = UniformFloatHyperparameter("learning_rate", 1e-5, 5e-2, default_value=1e-4, q=3e-5, log=True) batch_size = UniformIntegerHyperparameter("batch_size", 16, 128, q=16, default_value=32) momentum = UniformFloatHyperparameter("momentum", 0., .5, default_value=0., q=.1) lr_decay = UniformFloatHyperparameter("lr_decay", .7, .99, default_value=9e-1, q=3e-2) dropout_value = UniformFloatHyperparameter("dropout", .1, .7, default_value=.5, q=.1) cs.add_hyperparameters([learning_rate, batch_size, momentum, lr_decay, dropout_value]) # network architecture hyperparameters. num_pooling_layer = UniformIntegerHyperparameter("n_pooling_layer", 2, 3, default_value=2) num_conv_layer1 = UniformIntegerHyperparameter("n_conv_layer1", 16, 64, default_value=32, q=2) num_conv_layer2 = UniformIntegerHyperparameter("n_conv_layer2", 32, 96, default_value=64, q=2) num_conv_layer3 = UniformIntegerHyperparameter("n_conv_layer3", 32, 96, default_value=64, q=2) num_fully_layer = UniformIntegerHyperparameter("n_fully_unit", 128, 512, default_value=256, q=64) cs.add_hyperparameters([num_pooling_layer, num_conv_layer1, num_conv_layer2, num_conv_layer3, num_fully_layer]) for i in [1, 2, 3]: kernel_init_stddev = UniformFloatHyperparameter( "kernel_init_stddev%d" % i, 1e-3, 5e-2, default_value=1e-2, q=2e-3) kernel_regularizer = UniformFloatHyperparameter( "kernel_regularizer%d" % i, 1e-9, 1e-4, default_value=1e-6, q=5e-7, log=True) cs.add_hyperparameters([kernel_init_stddev, kernel_regularizer]) if i == 3: k_init_cond = InCondition(child=kernel_init_stddev, parent=num_pooling_layer, values=[3]) k_reg_cond = InCondition(child=kernel_regularizer, parent=num_pooling_layer, values=[3]) cs.add_conditions([k_init_cond, k_reg_cond]) # configuration = cs.get_default_configuration() # print(configuration.configuration_space) # print(configuration.get_dictionary()) return cs
def get_hyperparameter_search_space(dataset_properties=None, optimizer='smac'): if dataset_properties is not None and \ (dataset_properties.get("sparse") is True or dataset_properties.get("signed") is False): allow_chi2 = False else: allow_chi2 = True possible_kernels = ['poly', 'rbf', 'sigmoid', 'cosine'] if allow_chi2: possible_kernels.append("chi2") kernel = CategoricalHyperparameter('kernel', possible_kernels, 'rbf') n_components = UniformIntegerHyperparameter( "n_components", 50, 5000, default_value=100, log=True) gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8, log=True, default_value=0.1) degree = UniformIntegerHyperparameter('degree', 2, 5, 3) coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0) cs = ConfigurationSpace() cs.add_hyperparameters([kernel, degree, gamma, coef0, n_components]) degree_depends_on_poly = EqualsCondition(degree, kernel, "poly") coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"]) gamma_kernels = ["poly", "rbf", "sigmoid"] if allow_chi2: gamma_kernels.append("chi2") gamma_condition = InCondition(gamma, kernel, gamma_kernels) cs.add_conditions([degree_depends_on_poly, coef0_condition, gamma_condition]) return cs
def get_hyperparameter_search_space(**kwargs): n_components_factor = UniformFloatHyperparameter("n_components_factor", 0., 1., default_value=1.) kernel = CategoricalHyperparameter( 'kernel', ['poly', 'rbf', 'sigmoid', 'cosine'], 'rbf') gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8, log=True, default_value=1.0) degree = UniformIntegerHyperparameter('degree', 2, 5, 3) coef0 = UniformFloatHyperparameter("coef0", -1., 1., default_value=0.) cs = ConfigurationSpace() cs.add_hyperparameters( [n_components_factor, kernel, degree, gamma, coef0]) degree_depends_on_poly = EqualsCondition(degree, kernel, "poly") coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"]) gamma_condition = InCondition(gamma, kernel, ["poly", "rbf"]) cs.add_conditions( [degree_depends_on_poly, coef0_condition, gamma_condition]) return cs
def optimize(): # We load the iris-dataset (a widely used benchmark) iris = datasets.load_iris() #logger = logging.getLogger("SVMExample") logging.basicConfig(level=logging.INFO) # logging.DEBUG for debug output # Build Configuration Space which defines all parameters and their ranges cs = ConfigurationSpace() # We define a few possible types of SVM-kernels and add them as "kernel" to our cs kernel = CategoricalHyperparameter("kernel", ["linear", "rbf", "poly", "sigmoid"], default="poly") cs.add_hyperparameter(kernel) # There are some hyperparameters shared by all kernels C = UniformFloatHyperparameter("C", 0.001, 1000.0, default=1.0) shrinking = CategoricalHyperparameter("shrinking", ["true", "false"], default="true") cs.add_hyperparameters([C, shrinking]) # Others are kernel-specific, so we can add conditions to limit the searchspace degree = UniformIntegerHyperparameter("degree", 1, 5, default=3) # Only used by kernel poly coef0 = UniformFloatHyperparameter("coef0", 0.0, 10.0, default=0.0) # poly, sigmoid cs.add_hyperparameters([degree, coef0]) use_degree = InCondition(child=degree, parent=kernel, values=["poly"]) use_coef0 = InCondition(child=coef0, parent=kernel, values=["poly", "sigmoid"]) cs.add_conditions([use_degree, use_coef0]) # This also works for parameters that are a mix of categorical and values from a range of numbers # For example, gamma can be either "auto" or a fixed float gamma = CategoricalHyperparameter("gamma", ["auto", "value"], default="auto") # only rbf, poly, sigmoid gamma_value = UniformFloatHyperparameter("gamma_value", 0.0001, 8, default=1) cs.add_hyperparameters([gamma, gamma_value]) # We only activate gamma_value if gamma is set to "value" cs.add_condition(InCondition(child=gamma_value, parent=gamma, values=["value"])) # And again we can restrict the use of gamma in general to the choice of the kernel cs.add_condition(InCondition(child=gamma, parent=kernel, values=["rbf", "poly", "sigmoid"])) # Scenario object scenario = Scenario("test/test_files/svm_scenario.txt") # Example call of the function # It returns: Status, Cost, Runtime, Additional Infos def_value = svm_from_cfg(cs.get_default_configuration()) print("Default Value: %.2f" % (def_value)) # Optimize, using a SMAC-object print("Optimizing! Depending on your machine, this might take a few minutes.") smac = SMAC(scenario=scenario, rng=np.random.RandomState(42), tae_runner=svm_from_cfg) incumbent = smac.optimize() inc_value = svm_from_cfg(incumbent) print("Optimized Value: %.2f" % (inc_value))
def get_hyperparameter_search_space(**kwargs): cs = ConfigurationSpace() penalty = CategoricalHyperparameter("penalty", ["l1", "l2", "elasticnet", "none"], default_value='l2') solver = CategoricalHyperparameter("solver", ["newton-cg", "lbfgs", "liblinear", "sag", "saga"], default_value="lbfgs") dual = CategoricalHyperparameter("dual", choices=[True, False], default_value=False) tol = UniformFloatHyperparameter("tol", lower=1e-7, upper=100., default_value=1.0e-4, log=True) C = UniformFloatHyperparameter("C", lower=1e-7, upper=100., default_value=1.0, log=True) fit_intercept = CategoricalHyperparameter("fit_intercept", choices=[True, False], default_value=True) intercept_scaling = UniformFloatHyperparameter("intercept_scaling", lower=0.0001, upper=2.0, default_value=1.0, log=True) max_iter = UniformIntegerHyperparameter("max_iter", lower=50, upper=10000, default_value=100) multi_class = CategoricalHyperparameter("multi_class", ["ovr", "multinomial", "auto"], default_value="auto") l1_ratio = UniformFloatHyperparameter("l1_ratio", lower=0., upper=1., default_value=0.1) l1_ratio_condition = InCondition(l1_ratio, penalty, ["elasticnet"]) dual_condition = AndConjunction(InCondition(dual, penalty, ["l2"]), InCondition(dual, solver, ["liblinear"])) cs.add_hyperparameters([penalty, solver, dual, tol, C, fit_intercept, intercept_scaling, max_iter, multi_class, l1_ratio]) penaltyAndLbfgs = ForbiddenAndConjunction( ForbiddenEqualsClause(solver, "lbfgs"), ForbiddenInClause(penalty, ["l1", "elasticnet"]) ) penaltyAndNewton = ForbiddenAndConjunction( ForbiddenEqualsClause(solver, "newton-cg"), ForbiddenInClause(penalty, ["l1", "elasticnet"]) ) penaltyAndSag = ForbiddenAndConjunction( ForbiddenEqualsClause(solver, "sag"), ForbiddenInClause(penalty, ["l1", "elasticnet"]) ) penaltyAndSaga = ForbiddenAndConjunction( ForbiddenInClause(penalty, ["elasticnet"]), ForbiddenInClause(solver, ["newton-cg", "lbfgs", "sag"]) ) penaltyAndSagaa = ForbiddenAndConjunction( ForbiddenInClause(penalty, ["elasticnet", "none"]), ForbiddenInClause(solver, ["liblinear"]) ) penaltyAndSagaaa = ForbiddenAndConjunction( ForbiddenInClause(multi_class, ["multinomial"]), ForbiddenInClause(solver, ["liblinear"]) ) cs.add_forbidden_clause(penaltyAndLbfgs) cs.add_forbidden_clause(penaltyAndNewton) cs.add_forbidden_clause(penaltyAndSag) cs.add_forbidden_clause(penaltyAndSagaa) cs.add_forbidden_clause(penaltyAndSaga) cs.add_forbidden_clause(penaltyAndSagaaa) cs.add_condition(l1_ratio_condition) cs.add_condition(dual_condition) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() loss = CategoricalHyperparameter( "loss", ["squared_loss", "huber", "epsilon_insensitive", "squared_epsilon_insensitive"], default_value="squared_loss", ) penalty = CategoricalHyperparameter( "penalty", ["l1", "l2", "elasticnet"], default_value="l2") alpha = UniformFloatHyperparameter( "alpha", 1e-7, 1e-1, log=True, default_value=0.0001) l1_ratio = UniformFloatHyperparameter( "l1_ratio", 1e-9, 1., log=True, default_value=0.15) fit_intercept = UnParametrizedHyperparameter( "fit_intercept", "True") tol = UniformFloatHyperparameter( "tol", 1e-5, 1e-1, default_value=1e-4, log=True) epsilon = UniformFloatHyperparameter( "epsilon", 1e-5, 1e-1, default_value=0.1, log=True) learning_rate = CategoricalHyperparameter( "learning_rate", ["optimal", "invscaling", "constant"], default_value="invscaling") eta0 = UniformFloatHyperparameter( "eta0", 1e-7, 1e-1, default_value=0.01, log=True) power_t = UniformFloatHyperparameter( "power_t", 1e-5, 1, default_value=0.25) average = CategoricalHyperparameter( "average", ["False", "True"], default_value="False") cs.add_hyperparameters([loss, penalty, alpha, l1_ratio, fit_intercept, tol, epsilon, learning_rate, eta0, power_t, average]) # TODO add passive/aggressive here, although not properly documented? elasticnet = EqualsCondition(l1_ratio, penalty, "elasticnet") epsilon_condition = InCondition( epsilon, loss, ["huber", "epsilon_insensitive", "squared_epsilon_insensitive"], ) # eta0 is only relevant if learning_rate!='optimal' according to code # https://github.com/scikit-learn/scikit-learn/blob/0.19.X/sklearn/ # linear_model/sgd_fast.pyx#L603 eta0_in_inv_con = InCondition(eta0, learning_rate, ["invscaling", "constant"]) power_t_condition = EqualsCondition(power_t, learning_rate, "invscaling") cs.add_conditions([elasticnet, epsilon_condition, power_t_condition, eta0_in_inv_con]) return cs
def test_in_condition(self): hp1 = CategoricalHyperparameter("parent", range(0, 11)) hp2 = UniformIntegerHyperparameter("child", 0, 10) cond = InCondition(hp2, hp1, [0, 1, 2, 3, 4, 5]) cond_ = InCondition(hp2, hp1, [0, 1, 2, 3, 4, 5]) self.assertEqual(cond, cond_) cond_reverse = InCondition(hp1, hp2, [0, 1, 2, 3, 4, 5]) self.assertNotEqual(cond, cond_reverse) self.assertNotEqual(cond, dict()) self.assertEqual("child | parent in {0, 1, 2, 3, 4, 5}", str(cond))
def get_hyperparameter_search_space(dataset_properties=None): # Copied from libsvm_c C = UniformFloatHyperparameter( name="C", lower=0.03125, upper=32768, log=True, default=1.0) kernel = CategoricalHyperparameter( name="kernel", choices=['linear', 'poly', 'rbf', 'sigmoid'], default="rbf") degree = UniformIntegerHyperparameter( name="degree", lower=1, upper=5, default=3) # Changed the gamma value to 0.0 (is 0.1 for classification) gamma = UniformFloatHyperparameter( name="gamma", lower=3.0517578125e-05, upper=8, log=True, default=0.1) # TODO this is totally ad-hoc coef0 = UniformFloatHyperparameter( name="coef0", lower=-1, upper=1, default=0) # probability is no hyperparameter, but an argument to the SVM algo shrinking = CategoricalHyperparameter( name="shrinking", choices=["True", "False"], default="True") tol = UniformFloatHyperparameter( name="tol", lower=1e-5, upper=1e-1, default=1e-3, log=True) max_iter = UnParametrizedHyperparameter("max_iter", -1) # Random Guess epsilon = UniformFloatHyperparameter(name="epsilon", lower=0.001, upper=1, default=0.1, log=True) cs = ConfigurationSpace() cs.add_hyperparameter(C) cs.add_hyperparameter(kernel) cs.add_hyperparameter(degree) cs.add_hyperparameter(gamma) cs.add_hyperparameter(coef0) cs.add_hyperparameter(shrinking) cs.add_hyperparameter(tol) cs.add_hyperparameter(max_iter) cs.add_hyperparameter(epsilon) degree_depends_on_kernel = InCondition(child=degree, parent=kernel, values=('poly', 'rbf', 'sigmoid')) gamma_depends_on_kernel = InCondition(child=gamma, parent=kernel, values=('poly', 'rbf')) coef0_depends_on_kernel = InCondition(child=coef0, parent=kernel, values=('poly', 'sigmoid')) cs.add_condition(degree_depends_on_kernel) cs.add_condition(gamma_depends_on_kernel) cs.add_condition(coef0_depends_on_kernel) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() loss = Constant("loss", "auto") learning_rate = UniformFloatHyperparameter(name="learning_rate", lower=0.01, upper=1, default_value=0.1, log=True) max_iter = UniformIntegerHyperparameter("max_iter", 32, 512, default_value=100) min_samples_leaf = UniformIntegerHyperparameter( name="min_samples_leaf", lower=1, upper=200, default_value=20, log=True) max_depth = UnParametrizedHyperparameter(name="max_depth", value="None") max_leaf_nodes = UniformIntegerHyperparameter(name="max_leaf_nodes", lower=3, upper=2047, default_value=31, log=True) max_bins = Constant("max_bins", 256) l2_regularization = UniformFloatHyperparameter( name="l2_regularization", lower=1E-10, upper=1, default_value=1E-10, log=True) early_stop = CategoricalHyperparameter( name="early_stop", choices=["off", "train", "valid"], default_value="off") tol = UnParametrizedHyperparameter(name="tol", value=1e-7) scoring = UnParametrizedHyperparameter(name="scoring", value="loss") n_iter_no_change = UniformIntegerHyperparameter( name="n_iter_no_change", lower=1, upper=20, default_value=10) validation_fraction = UniformFloatHyperparameter( name="validation_fraction", lower=0.01, upper=0.4, default_value=0.1) cs.add_hyperparameters([ loss, learning_rate, max_iter, min_samples_leaf, max_depth, max_leaf_nodes, max_bins, l2_regularization, early_stop, tol, scoring, n_iter_no_change, validation_fraction ]) n_iter_no_change_cond = InCondition(n_iter_no_change, early_stop, ["valid", "train"]) validation_fraction_cond = EqualsCondition(validation_fraction, early_stop, "valid") cs.add_conditions([n_iter_no_change_cond, validation_fraction_cond]) return cs
def get_hyperparameter_search_space(dataset_properties=None): C = UniformFloatHyperparameter("C", 0.03125, 32768, log=True, default_value=1.0) # No linear kernel here, because we have liblinear kernel = CategoricalHyperparameter(name="kernel", choices=["rbf", "poly", "sigmoid"], default_value="rbf") degree = UniformIntegerHyperparameter("degree", 2, 5, default_value=3) gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8, log=True, default_value=0.1) # TODO this is totally ad-hoc coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0) # probability is no hyperparameter, but an argument to the SVM algo shrinking = CategoricalHyperparameter("shrinking", ["True", "False"], default_value="True") tol = UniformFloatHyperparameter("tol", 1e-5, 1e-1, default_value=1e-3, log=True) # cache size is not a hyperparameter, but an argument to the program! max_iter = UnParametrizedHyperparameter("max_iter", -1) cs = ConfigurationSpace() cs.add_hyperparameters([C, kernel, degree, gamma, coef0, shrinking, tol, max_iter]) degree_depends_on_poly = EqualsCondition(degree, kernel, "poly") coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"]) cs.add_condition(degree_depends_on_poly) cs.add_condition(coef0_condition) return cs
def add_params(cs: ConfigurationSpace, cutoff: int): ''' adds parameters to ConfigurationSpace Arguments --------- cs: ConfigurationSpace configuration space to add new parameters and conditions cutoff: int maximal possible time for aspeed ''' pre_solving = CategoricalHyperparameter("presolving", choices=[True, False], default_value=False) cs.add_hyperparameter(pre_solving) pre_cutoff = UniformIntegerHyperparameter("pre:cutoff", lower=1, upper=cutoff, default_value=math.ceil( cutoff * 0.1), log=True) cs.add_hyperparameter(pre_cutoff) cond = InCondition(child=pre_cutoff, parent=pre_solving, values=[True]) cs.add_condition(cond)
def get_hyperparameter_search_space(seed): imputation = CategoricalHyperparameter('imputation__strategy', ['mean', 'median', 'most_frequent']) C = UniformFloatHyperparameter("classifier__C", 0.03125, 32768, log=True, default_value=1.0) # No linear kernel here, because we have liblinear kernel = CategoricalHyperparameter(name="classifier__kernel", choices=["rbf", "poly", "sigmoid"], default_value="rbf") degree = UniformIntegerHyperparameter("classifier__degree", 2, 5, default_value=3) gamma = UniformFloatHyperparameter("classifier__gamma", 3.0517578125e-05, 8, log=True, default_value=0.1) # TODO this is totally ad-hoc coef0 = UniformFloatHyperparameter("classifier__coef0", -1, 1, default_value=0) # probability is no hyperparameter, but an argument to the SVM algo shrinking = CategoricalHyperparameter("classifier__shrinking", [True, False], default_value=True) tol = UniformFloatHyperparameter("classifier__tol", 1e-5, 1e-1, default_value=1e-3, log=True) # cache size is not a hyperparameter, but an argument to the program! max_iter = UnParametrizedHyperparameter("classifier__max_iter", -1) cs = ConfigurationSpace('sklearn.svm.SVC', seed) cs.add_hyperparameters([imputation, C, kernel, degree, gamma, coef0, shrinking, tol, max_iter]) degree_depends_on_poly = EqualsCondition(degree, kernel, "poly") coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"]) cs.add_condition(degree_depends_on_poly) cs.add_condition(coef0_condition) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() loss = CategoricalHyperparameter( "loss", ["ls", "lad", "huber", "quantile"], default_value="ls") learning_rate = UniformFloatHyperparameter( name="learning_rate", lower=0.01, upper=1, default_value=0.1, log=True) n_estimators = UniformIntegerHyperparameter( "n_estimators", 50, 500, default_value=100) max_depth = UniformIntegerHyperparameter( name="max_depth", lower=1, upper=10, default_value=3) min_samples_split = UniformIntegerHyperparameter( name="min_samples_split", lower=2, upper=20, default_value=2, log=False) min_samples_leaf = UniformIntegerHyperparameter( name="min_samples_leaf", lower=1, upper=20, default_value=1, log=False) min_weight_fraction_leaf = UnParametrizedHyperparameter( "min_weight_fraction_leaf", 0.) subsample = UniformFloatHyperparameter( name="subsample", lower=0.01, upper=1.0, default_value=1.0, log=False) max_features = UniformFloatHyperparameter( "max_features", 0.1, 1.0, default_value=1) max_leaf_nodes = UnParametrizedHyperparameter( name="max_leaf_nodes", value="None") min_impurity_decrease = UnParametrizedHyperparameter( name='min_impurity_decrease', value=0.0) alpha = UniformFloatHyperparameter( "alpha", lower=0.75, upper=0.99, default_value=0.9) cs.add_hyperparameters([loss, learning_rate, n_estimators, max_depth, min_samples_split, min_samples_leaf, min_weight_fraction_leaf, subsample, max_features, max_leaf_nodes, min_impurity_decrease, alpha]) cs.add_condition(InCondition(alpha, loss, ['huber', 'quantile'])) return cs
def _union_to_config_space(self, name, hp_value): union_child = [] union_config = [] for union_name, union_hp_value in hp_value.configuration.items(): unique_union_name = "{}_{}".format(name, union_name) if isinstance(union_hp_value, (hyperparams.Bounded, hyperparams.Uniform, hyperparams.UniformInt)): child = self._bounded_to_config_space(unique_union_name, union_hp_value) elif isinstance( union_hp_value, (hyperparams.Enumeration, hyperparams.UniformBool)): child = self._enumeration_to_config_space( unique_union_name, union_hp_value) elif isinstance(union_hp_value, (hyperparams.Constant)): child = self._constant_to_config_space(unique_union_name, union_hp_value) union_child.append(unique_union_name) union_config.append(child) # params_config = CategoricalHyperparameter(name=name, choices=union_child, default_value=hp_value.get_default()) params_config = CategoricalHyperparameter(name=name, choices=union_child) self.cs.add_hyperparameter(params_config) [ self.cs.add_condition( InCondition(child=item, parent=params_config, values=[item.name])) for item in union_config ] return params_config
def get_hyperparameter_search_space(**kwargs): cs = ConfigurationSpace() n_components = UniformIntegerHyperparameter('n_components_factor', 1, 250, default_value=10) max_iter = UniformIntegerHyperparameter("max_iter", 10, 2000, default_value=1000) tol = UniformFloatHyperparameter("tol", 1e-5, 1e-1, default_value=1e-2, log=True) svd_method = CategoricalHyperparameter("svd_method", ["lapack", "randomized"], default_value="randomized") iterated_power = UniformIntegerHyperparameter("iterated_power", 1, 10, default_value=3) cs.add_hyperparameters( [n_components, max_iter, tol, svd_method, iterated_power]) iterated_power_condition = InCondition(iterated_power, svd_method, ["randomized"]) cs.add_condition(iterated_power_condition) return cs
def make_cs(): cs = ConfigurationSpace() cs.add_hyperparameter( UniformIntegerHyperparameter("n_estimators", 1, 30, default=10)) max_features = CategoricalHyperparameter('max_features', ['auto', 'value'], default='auto') max_features_value = UniformFloatHyperparameter('max_features_value', .1, 1) cs.add_hyperparameters([max_features, max_features_value]) cs.add_condition( InCondition(child=max_features_value, parent=max_features, values=['value'])) max_depth = CategoricalHyperparameter('max_depth', [None, 'value']) max_depth_value = UniformIntegerHyperparameter("max_depth_value", 1, 10) cs.add_hyperparameters([max_depth, max_depth_value]) cs.add_condition( InCondition(child=max_depth_value, parent=max_depth, values=['value'])) min_samples_split = UniformFloatHyperparameter("min_samples_split", .1, 1) cs.add_hyperparameter(min_samples_split) min_samples_leaf = UniformFloatHyperparameter("min_samples_leaf", .1, .5) cs.add_hyperparameter(min_samples_leaf) min_weight_fraction_leaf = UniformFloatHyperparameter( "min_weight_fraction_leaf", 0, .5) cs.add_hyperparameter(min_weight_fraction_leaf) max_leaf_nodes = CategoricalHyperparameter('max_leaf_nodes', [None, 'value']) max_leaf_nodes_value = UniformIntegerHyperparameter( 'max_leaf_nodes_value', 2, 100) cs.add_hyperparameters([max_leaf_nodes, max_leaf_nodes_value]) cs.add_condition( InCondition(child=max_leaf_nodes_value, parent=max_leaf_nodes, values=['value'])) min_impurity_split = UniformFloatHyperparameter('min_impurity_split', 0, 1) cs.add_hyperparameter(min_impurity_split) bootstrap = CategoricalHyperparameter('bootstrap', [True, False], default=True) cs.add_hyperparameter(bootstrap)
def getPCS(self): ''' maxIter: [1,100]最大迭代次数,默认50 regParam :[0,0.2] 正则化参数,默认0 tol:[1e-6,1e-1] 迭代算法收敛性,默认 1e-6 family ,link, variancePower 对应关系 • “gaussian” -> “identity”, “log”, “inverse” • “binomial” -> “logit”, “probit”, “cloglog” • “poisson” -> “log”, “identity”, “sqrt” • “gamma” -> “inverse”, “identity”, “log” • “tweedie” -> power link function specified through “linkPower”. The default link power in the tweedie family is 1 - variancePower. ''' # Build Configuration Space which defines all parameters and their # ranges cs = ConfigurationSpace() maxIter = UniformIntegerHyperparameter("maxIter", 1, 100, default_value=50) regParam = UniformFloatHyperparameter("regParam", 0, 0.4, default_value=1e-04) tol = UniformFloatHyperparameter("tol", 1e-06, 1e-01, default_value=1e-06) family = CategoricalHyperparameter("family", ["gaussian", "poisson"], default_value="gaussian") gaussianLink = CategoricalHyperparameter( "gaussianLink", ["identity", "log", "inverse"], default_value="identity") poissonLink = CategoricalHyperparameter("poissonLink", ["log", "identity", "sqrt"], default_value="log") cs.add_hyperparameters( [maxIter, regParam, tol, family, gaussianLink, poissonLink]) cs.add_condition( InCondition(child=gaussianLink, parent=family, values=["gaussian"])) cs.add_condition( InCondition(child=poissonLink, parent=family, values=["poisson"])) return cs
def get_complete_configspace(): """Creates a configspace that includes all kinds of parameters with complicated values. The idea is to provide a configspace that can be used to check modules using ConfigSpace as a dependency to check compatibility with e.g. Constants, log-scale, etc. Returns ------- cs: ConfigurationSpace cs containing all kinds of parameters """ cs = ConfigurationSpace() hp = {} # Add Constants for all allowed types ('int', 'float', 'string') hp['alpha'] = Constant("alpha", 0.0001) hp['tol'] = Constant("tol", '0.0001') hp['verbose'] = Constant("verbose", 1) # Add numericals # Add Floats hp['beta1'] = UniformFloatHyperparameter("beta1", 0.85, 0.95, log=False) hp['power_t'] = NormalFloatHyperparameter("power_t", mu=0.5, sigma=0.1, log=False) # Add Ints hp['momentum'] = UniformIntegerHyperparameter("momentum", 0, 100, False) hp['beta2'] = NormalIntegerHyperparameter("beta2", mu=1, sigma=0.001, log=False) # Add Floats (log) hp['learning_rate_init'] = UniformFloatHyperparameter("learning_rate_init", 0.0001, 0.1, log=True) hp['random1'] = NormalFloatHyperparameter("NormalFloat", mu=0, sigma=1, default_value=1, log=True) # Add Ints (log) hp['random2'] = UniformIntegerHyperparameter("UniformInt", 2, 100, log=True) hp['random3'] = NormalIntegerHyperparameter("NormalInt", mu=0, sigma=1, default_value=1, log=True) # Add Categorical for allowed types hp['activation'] = CategoricalHyperparameter('activation', choices=['identity', 'logistic', 'tanh', 'relu']) hp['solver'] = CategoricalHyperparameter('solver', choices=[-2, 0, 2]) # corrresponds to: ‘lbfgs’, ‘sgd’, ‘adam’ hp['batch_size_auto'] = CategoricalHyperparameter('batch_size_auto', choices=[True, False]) hp['learning_rate'] = CategoricalHyperparameter('learning_rate', choices=[-0.5, 0.0, 0.5]) # corresponds to {‘constant’, ‘invscaling’, ‘adaptive’} # Add Ordinal hp['batch_size'] = OrdinalHyperparameter('batch_size', sequence=[32, 64.0, '128']) for k, v in hp.items(): cs.add_hyperparameter(v) # learning_rate only with sgd c = InCondition(hp['learning_rate'], hp['solver'], [0]) c = EqualsCondition(hp['momentum'], hp['solver'], 0) # learning_rate_init only with sgd or adam cs.add_condition(OrConjunction(EqualsCondition(hp['learning_rate'], hp['solver'], 0), # sgd EqualsCondition(hp['learning_rate'], hp['solver'], 2))) # adam # batch_size only with not batch_size_auto cs.add_condition(NotEqualsCondition(hp['batch_size'], hp['batch_size_auto'], True)) # complicated way for solver == sgd #cs.add_condition(AndConjunction(LessThanCondition(hp['power_t'], hp['solver'], 1), # GreaterThanCondition(hp['power_t'], hp['solver'], -1))) # betas with adam cs.add_condition(EqualsCondition(hp['beta1'], hp['solver'], 2)) cs.add_condition(EqualsCondition(hp['beta2'], hp['solver'], 2)) return cs
def test_write_AndConjunction_condition(self): expected = "lp '--lp ' c {mi,bo}\nls '--ls ' c {sa,ca,ny}\ntemp '--temp ' r (0.500000, 1.000000)| ls %in% c(sa,ca) && lp %in% c(bo)\n" temp = UniformFloatHyperparameter("temp", 0.5, 1) ls = CategoricalHyperparameter("ls", ["sa", "ca", "ny"], "sa") lp = CategoricalHyperparameter("lp", ["mi", "bo"], "bo") cs = ConfigurationSpace() cs.add_hyperparameter(temp) cs.add_hyperparameter(lp) cs.add_hyperparameter(ls) c1 = InCondition(temp, ls, ['sa','ca']) c2 = InCondition(temp, lp, ['bo']) c3 = AndConjunction(c1, c2) cs.add_condition(c3) value = irace.write(cs) self.assertEqual(expected, value)
def _construct_in_condition( condition: Dict, cs: ConfigurationSpace, ) -> InCondition: return InCondition( child=cs.get_hyperparameter(condition['child']), parent=cs.get_hyperparameter(condition['parent']), values=condition['values'], )
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() hidden_layer_depth = UniformIntegerHyperparameter(name="hidden_layer_depth", lower=1, upper=3, default_value=1) num_nodes_per_layer = UniformIntegerHyperparameter(name="num_nodes_per_layer", lower=16, upper=264, default_value=32, log=True) activation = CategoricalHyperparameter(name="activation", choices=['tanh', 'relu'], default_value='tanh') alpha = UniformFloatHyperparameter(name="alpha", lower=1e-7, upper=1e-1, default_value=1e-4, log=True) learning_rate_init = UniformFloatHyperparameter(name="learning_rate_init", lower=1e-4, upper=0.5, default_value=1e-3, log=True) # Not allowing to turn off early stopping early_stopping = CategoricalHyperparameter(name="early_stopping", choices=["valid", "train"], # , "off"], default_value="valid") # Constants n_iter_no_change = Constant(name="n_iter_no_change", value=32) # default=10 is too low validation_fraction = Constant(name="validation_fraction", value=0.1) tol = UnParametrizedHyperparameter(name="tol", value=1e-4) solver = Constant(name="solver", value='adam') # Relying on sklearn defaults for now batch_size = UnParametrizedHyperparameter(name="batch_size", value="auto") shuffle = UnParametrizedHyperparameter(name="shuffle", value="True") beta_1 = UnParametrizedHyperparameter(name="beta_1", value=0.9) beta_2 = UnParametrizedHyperparameter(name="beta_2", value=0.999) epsilon = UnParametrizedHyperparameter(name="epsilon", value=1e-8) # Not used # solver=["sgd", "lbfgs"] --> not used to keep searchspace simpler # learning_rate --> only used when using solver=sgd # power_t --> only used when using solver=sgd & learning_rate=invscaling # momentum --> only used when solver=sgd # nesterovs_momentum --> only used when solver=sgd # max_fun --> only used when solver=lbfgs # activation=["identity", "logistic"] --> not useful for classification cs.add_hyperparameters([hidden_layer_depth, num_nodes_per_layer, activation, alpha, learning_rate_init, early_stopping, n_iter_no_change, validation_fraction, tol, solver, batch_size, shuffle, beta_1, beta_2, epsilon]) validation_fraction_cond = InCondition(validation_fraction, early_stopping, ["valid"]) cs.add_conditions([validation_fraction_cond]) # We always use early stopping # n_iter_no_change_cond = InCondition(n_iter_no_change, early_stopping, ["valid", "train"]) # tol_cond = InCondition(n_iter_no_change, early_stopping, ["valid", "train"]) # cs.add_conditions([n_iter_no_change_cond, tol_cond]) return cs
def test_in_condition(self): hp1 = CategoricalHyperparameter("parent", range(0, 11)) hp2 = UniformIntegerHyperparameter("child", 0, 10) cond = InCondition(hp2, hp1, [0, 1, 2, 3, 4, 5]) cond_ = InCondition(hp2, hp1, [0, 1, 2, 3, 4, 5]) self.assertEqual(cond, cond_) # Test vector value: self.assertEqual( cond.vector_values, [hp1._inverse_transform(i) for i in [0, 1, 2, 3, 4, 5]]) self.assertEqual(cond.vector_values, cond_.vector_values) cond_reverse = InCondition(hp1, hp2, [0, 1, 2, 3, 4, 5]) self.assertNotEqual(cond, cond_reverse) self.assertNotEqual(cond, dict()) self.assertEqual("child | parent in {0, 1, 2, 3, 4, 5}", str(cond))
def add_params(cs: ConfigurationSpace): ''' adds parameters to ConfigurationSpace ''' selector = cs.get_hyperparameter("selector") regressor = cs.get_hyperparameter("regressor") if "PairwiseRegressor" in selector.choices: cond = InCondition(child=regressor, parent=selector, values=["PairwiseRegressor"]) cs.add_condition(cond)
def get_libsvm_svc_default_search_space(): classif_prefix = "classifier:libsvm_svc:" model_type = CategoricalHyperparameter('classifier:__choice__', ['libsvm_svc']) imputation = CategoricalHyperparameter('imputation:strategy', ['mean', 'median', 'most_frequent']) C = UniformFloatHyperparameter(classif_prefix + "C", 0.03125, 32768, log=True, default_value=1.0) # No linear kernel here, because we have liblinear kernel = CategoricalHyperparameter(name=classif_prefix + "kernel", choices=["rbf", "poly", "sigmoid"], default_value="rbf") degree = UniformIntegerHyperparameter(classif_prefix + "degree", 2, 5, default_value=3) gamma = UniformFloatHyperparameter(classif_prefix + "gamma", 3.0517578125e-05, 8, log=True, default_value=0.1) # TODO this is totally ad-hoc coef0 = UniformFloatHyperparameter(classif_prefix + "coef0", -1, 1, default_value=0) # probability is no hyperparameter, but an argument to the SVM algo shrinking = CategoricalHyperparameter(classif_prefix + "shrinking", ["True", "False"], default_value="True") tol = UniformFloatHyperparameter(classif_prefix + "tol", 1e-5, 1e-1, default_value=1e-3, log=True) # cache size is not a hyperparameter, but an argument to the program! max_iter = UnParametrizedHyperparameter(classif_prefix + "max_iter", -1) cs = ConfigurationSpace() cs.add_hyperparameters([ model_type, imputation, C, kernel, degree, gamma, coef0, shrinking, tol, max_iter ]) degree_depends_on_poly = EqualsCondition(degree, kernel, "poly") coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"]) cs.add_condition(degree_depends_on_poly) cs.add_condition(coef0_condition) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() loss = cs.add_hyperparameter( CategoricalHyperparameter("loss", ["ls", "lad", "huber", "quantile"], default="ls")) learning_rate = cs.add_hyperparameter( UniformFloatHyperparameter(name="learning_rate", lower=0.01, upper=1, default=0.1, log=True)) n_estimators = cs.add_hyperparameter( UniformIntegerHyperparameter(name="n_estimators", lower=50, upper=500, default=100)) max_depth = cs.add_hyperparameter( UniformIntegerHyperparameter(name="max_depth", lower=1, upper=10, default=3)) min_samples_split = cs.add_hyperparameter( UniformIntegerHyperparameter(name="min_samples_split", lower=2, upper=20, default=2, log=False)) min_samples_leaf = cs.add_hyperparameter( UniformIntegerHyperparameter(name="min_samples_leaf", lower=1, upper=20, default=1, log=False)) min_weight_fraction_leaf = cs.add_hyperparameter( UnParametrizedHyperparameter("min_weight_fraction_leaf", 0.)) subsample = cs.add_hyperparameter( UniformFloatHyperparameter(name="subsample", lower=0.01, upper=1.0, default=1.0, log=False)) max_features = cs.add_hyperparameter( UniformFloatHyperparameter("max_features", 0.5, 5, default=1)) max_leaf_nodes = cs.add_hyperparameter( UnParametrizedHyperparameter(name="max_leaf_nodes", value="None")) alpha = cs.add_hyperparameter( UniformFloatHyperparameter("alpha", lower=0.75, upper=0.99, default=0.9)) cs.add_condition(InCondition(alpha, loss, ['huber', 'quantile'])) return cs
def get_hyperparameter_search_space(dataset_properties=None): C = UniformFloatHyperparameter( name="C", lower=0.03125, upper=32768, log=True, default_value=1.0) # Random Guess epsilon = UniformFloatHyperparameter(name="epsilon", lower=0.001, upper=1, default_value=0.1, log=True) kernel = CategoricalHyperparameter( name="kernel", choices=['linear', 'poly', 'rbf', 'sigmoid'], default_value="rbf") degree = UniformIntegerHyperparameter( name="degree", lower=2, upper=5, default_value=3) gamma = UniformFloatHyperparameter( name="gamma", lower=3.0517578125e-05, upper=8, log=True, default_value=0.1) # TODO this is totally ad-hoc coef0 = UniformFloatHyperparameter( name="coef0", lower=-1, upper=1, default_value=0) # probability is no hyperparameter, but an argument to the SVM algo shrinking = CategoricalHyperparameter( name="shrinking", choices=["True", "False"], default_value="True") tol = UniformFloatHyperparameter( name="tol", lower=1e-5, upper=1e-1, default_value=1e-3, log=True) max_iter = UnParametrizedHyperparameter("max_iter", -1) cs = ConfigurationSpace() cs.add_hyperparameters([C, kernel, degree, gamma, coef0, shrinking, tol, max_iter, epsilon]) degree_depends_on_kernel = InCondition(child=degree, parent=kernel, values=('poly', 'rbf', 'sigmoid')) gamma_depends_on_kernel = InCondition(child=gamma, parent=kernel, values=('poly', 'rbf')) coef0_depends_on_kernel = InCondition(child=coef0, parent=kernel, values=('poly', 'sigmoid')) cs.add_conditions([degree_depends_on_kernel, gamma_depends_on_kernel, coef0_depends_on_kernel]) return cs
def add_params(cs: ConfigurationSpace): ''' adds parameters to ConfigurationSpace ''' selector = cs.get_hyperparameter("selector") classifier = cs.get_hyperparameter("classifier") if "MultiClassifier" in selector.choices: cond = InCondition(child=classifier, parent=selector, values=["MultiClassifier"]) cs.add_condition(cond)
def main_loop(problem): logging.basicConfig(level=logging.INFO) # logging.DEBUG for debug output cs = ConfigurationSpace() n_neighbors = UniformIntegerHyperparameter("n_neighbors", 2,10, default_value=5) cs.add_hyperparameter(n_neighbors) weights = CategoricalHyperparameter("weights", ["uniform","distance"], default_value="uniform") algorithm = CategoricalHyperparameter("algorithm", ["ball_tree", "kd_tree","brute","auto"], default_value="auto") cs.add_hyperparameters([weights, algorithm]) leaf_size = UniformIntegerHyperparameter("leaf_size", 1, 100, default_value=50) cs.add_hyperparameter(leaf_size) use_leaf_size= InCondition(child=leaf_size, parent=algorithm, values=["ball_tree","kd_tree"]) cs.add_condition(use_leaf_size) p = UniformIntegerHyperparameter("p", 1,3, default_value=2) cs.add_hyperparameter(p) # Scenario object max_eval=100000 scenario = Scenario({"run_obj": "quality", # we optimize quality (alternatively runtime) "runcount-limit": max_eval, # maximum function evaluations "cs": cs, # configuration space "shared_model": True, "output_dir": "/home/naamah/Documents/CatES/result_All/smac/KNN/run_{}_{}_{}".format(max_eval, datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d_%H:%M:%S'), problem) # "output_dir": "/home/naamah/Documents/CatES/result_All/smac/KNN/{}/run_{}_{}".format(problem,max_eval, datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d_%H:%M:%S_%f')), # "input_psmac_dirs":"/home/naamah/Documents/CatES/result_All/", # "deterministic": "true" }) # Example call of the function # It returns: Status, Cost, Runtime, Additional Infos def_value = svm_from_cfg(cs.get_default_configuration()) print("Default Value: %.2f" % (def_value)) # Optimize, using a SMAC-object print("Optimizing! Depending on your machine, this might take a few minutes.") smac = SMAC(scenario=scenario,tae_runner=svm_from_cfg) incumbent = smac.optimize() inc_value = svm_from_cfg(incumbent) print("Optimized Value: %.2f" % (inc_value)) return (incumbent) # main_loop()
def config_space(self): """SVC hyperparameter space.""" C_param = UniformFloatHyperparameter( 'C', lower=1e-8, upper=100.0, default_value=1.0 ) shrinking = CategoricalHyperparameter( 'shrinking', [True, False], default_value=True ) kernel = CategoricalHyperparameter( 'kernel', ['linear', 'rbf', 'poly', 'sigmoid'], ) degree = UniformIntegerHyperparameter( 'degree', lower=1, upper=5, default_value=2 ) coef0 = UniformFloatHyperparameter( 'coef0', lower=0.0, upper=10.0, default_value=0.0 ) # Add hyperparameters to config space. config = ConfigurationSpace() config.seed(self.random_state) config.add_hyperparameters( ( C_param, shrinking, kernel, degree, coef0, ) ) # Conditionals on hyperparameters specific to kernels. config.add_conditions( ( InCondition(child=degree, parent=kernel, values=['poly']), InCondition( child=coef0, parent=kernel, values=['poly', 'sigmoid'] ) ) ) return config