def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() n_iter = UnParametrizedHyperparameter("n_iter", value=300) tol = UniformFloatHyperparameter("tol", 10 ** -5, 10 ** -1, default_value=10 ** -3, log=True) alpha_1 = UniformFloatHyperparameter(name="alpha_1", lower=10 ** -10, upper=10 ** -3, default_value=10 ** -6) alpha_2 = UniformFloatHyperparameter(name="alpha_2", log=True, lower=10 ** -10, upper=10 ** -3, default_value=10 ** -6) lambda_1 = UniformFloatHyperparameter(name="lambda_1", log=True, lower=10 ** -10, upper=10 ** -3, default_value=10 ** -6) lambda_2 = UniformFloatHyperparameter(name="lambda_2", log=True, lower=10 ** -10, upper=10 ** -3, default_value=10 ** -6) threshold_lambda = UniformFloatHyperparameter(name="threshold_lambda", log=True, lower=10 ** 3, upper=10 ** 5, default_value=10 ** 4) fit_intercept = UnParametrizedHyperparameter("fit_intercept", "True") cs.add_hyperparameters([n_iter, tol, alpha_1, alpha_2, lambda_1, lambda_2, threshold_lambda, fit_intercept]) return cs
def test_write_forbidden(self): cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("parent", [0, 1]) hp2 = UniformIntegerHyperparameter("child", 0, 2) hp3 = UniformIntegerHyperparameter("child2", 0, 2) hp4 = UniformIntegerHyperparameter("child3", 0, 2) hp5 = CategoricalHyperparameter("child4", [4, 5, 6, 7]) cs.add_hyperparameters([hp1, hp2, hp3, hp4, hp5]) forb2 = ForbiddenEqualsClause(hp1, 1) forb3 = ForbiddenInClause(hp2, range(2, 3)) forb4 = ForbiddenInClause(hp3, range(2, 3)) forb5 = ForbiddenInClause(hp4, range(2, 3)) forb6 = ForbiddenInClause(hp5, [6, 7]) and1 = ForbiddenAndConjunction(forb2, forb3) and2 = ForbiddenAndConjunction(forb2, forb4) and3 = ForbiddenAndConjunction(forb2, forb5) cs.add_forbidden_clauses( [forb2, forb3, forb4, forb5, forb6, and1, and2, and3]) irace.write(cs) # generates file called forbidden.txt
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() C = UniformFloatHyperparameter( "C", 0.03125, 32768, log=True, default_value=1.0) loss = CategoricalHyperparameter( "loss", ["epsilon_insensitive", "squared_epsilon_insensitive"], default_value="squared_epsilon_insensitive") # Random Guess epsilon = UniformFloatHyperparameter( name="epsilon", lower=0.001, upper=1, default_value=0.1, log=True) dual = Constant("dual", "False") # These are set ad-hoc tol = UniformFloatHyperparameter( "tol", 1e-5, 1e-1, default_value=1e-4, log=True) fit_intercept =Constant("fit_intercept", "True") intercept_scaling = Constant("intercept_scaling", 1) cs.add_hyperparameters([C, loss, epsilon, dual, tol, fit_intercept, intercept_scaling]) dual_and_loss = ForbiddenAndConjunction( ForbiddenEqualsClause(dual, "False"), ForbiddenEqualsClause(loss, "epsilon_insensitive") ) cs.add_forbidden_clause(dual_and_loss) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() loss = Constant("loss", "deviance") learning_rate = UniformFloatHyperparameter( name="learning_rate", lower=0.01, upper=1, default_value=0.1, log=True) n_estimators = UniformIntegerHyperparameter( "n_estimators", 50, 500, default_value=100) max_depth = UniformIntegerHyperparameter( name="max_depth", lower=1, upper=10, default_value=3) criterion = CategoricalHyperparameter( 'criterion', ['friedman_mse', 'mse', 'mae'], default_value='mse') min_samples_split = UniformIntegerHyperparameter( name="min_samples_split", lower=2, upper=20, default_value=2) min_samples_leaf = UniformIntegerHyperparameter( name="min_samples_leaf", lower=1, upper=20, default_value=1) min_weight_fraction_leaf = UnParametrizedHyperparameter("min_weight_fraction_leaf", 0.) subsample = UniformFloatHyperparameter( name="subsample", lower=0.01, upper=1.0, default_value=1.0) max_features = UniformFloatHyperparameter( "max_features", 0.1, 1.0 , default_value=1) max_leaf_nodes = UnParametrizedHyperparameter( name="max_leaf_nodes", value="None") min_impurity_decrease = UnParametrizedHyperparameter( name='min_impurity_decrease', value=0.0) cs.add_hyperparameters([loss, learning_rate, n_estimators, max_depth, criterion, min_samples_split, min_samples_leaf, min_weight_fraction_leaf, subsample, max_features, max_leaf_nodes, min_impurity_decrease]) return cs
def get_hyperparameter_search_space(dataset_properties=None): C = UniformFloatHyperparameter("C", 0.03125, 32768, log=True, default_value=1.0) # No linear kernel here, because we have liblinear kernel = CategoricalHyperparameter(name="kernel", choices=["rbf", "poly", "sigmoid"], default_value="rbf") degree = UniformIntegerHyperparameter("degree", 2, 5, default_value=3) gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8, log=True, default_value=0.1) # TODO this is totally ad-hoc coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0) # probability is no hyperparameter, but an argument to the SVM algo shrinking = CategoricalHyperparameter("shrinking", ["True", "False"], default_value="True") tol = UniformFloatHyperparameter("tol", 1e-5, 1e-1, default_value=1e-3, log=True) # cache size is not a hyperparameter, but an argument to the program! max_iter = UnParametrizedHyperparameter("max_iter", -1) cs = ConfigurationSpace() cs.add_hyperparameters([C, kernel, degree, gamma, coef0, shrinking, tol, max_iter]) degree_depends_on_poly = EqualsCondition(degree, kernel, "poly") coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"]) cs.add_condition(degree_depends_on_poly) cs.add_condition(coef0_condition) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() n_estimators = Constant("n_estimators", 100) criterion = CategoricalHyperparameter( "criterion", ["gini", "entropy"], default_value="gini") # The maximum number of features used in the forest is calculated as m^max_features, where # m is the total number of features, and max_features is the hyperparameter specified below. # The default is 0.5, which yields sqrt(m) features as max_features in the estimator. This # corresponds with Geurts' heuristic. max_features = UniformFloatHyperparameter( "max_features", 0., 1., default_value=0.5) max_depth = UnParametrizedHyperparameter("max_depth", "None") min_samples_split = UniformIntegerHyperparameter( "min_samples_split", 2, 20, default_value=2) min_samples_leaf = UniformIntegerHyperparameter( "min_samples_leaf", 1, 20, default_value=1) min_weight_fraction_leaf = UnParametrizedHyperparameter("min_weight_fraction_leaf", 0.) max_leaf_nodes = UnParametrizedHyperparameter("max_leaf_nodes", "None") min_impurity_decrease = UnParametrizedHyperparameter('min_impurity_decrease', 0.0) bootstrap = CategoricalHyperparameter( "bootstrap", ["True", "False"], default_value="True") cs.add_hyperparameters([n_estimators, criterion, max_features, max_depth, min_samples_split, min_samples_leaf, min_weight_fraction_leaf, max_leaf_nodes, bootstrap, min_impurity_decrease]) return cs
def get_hyperparameter_search_space(dataset_properties=None): if dataset_properties is not None and \ (dataset_properties.get("sparse") is True or dataset_properties.get("signed") is False): allow_chi2 = False else: allow_chi2 = True possible_kernels = ['poly', 'rbf', 'sigmoid', 'cosine'] if allow_chi2: possible_kernels.append("chi2") kernel = CategoricalHyperparameter('kernel', possible_kernels, 'rbf') n_components = UniformIntegerHyperparameter( "n_components", 50, 10000, default_value=100, log=True) gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8, log=True, default_value=0.1) degree = UniformIntegerHyperparameter('degree', 2, 5, 3) coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0) cs = ConfigurationSpace() cs.add_hyperparameters([kernel, degree, gamma, coef0, n_components]) degree_depends_on_poly = EqualsCondition(degree, kernel, "poly") coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"]) gamma_kernels = ["poly", "rbf", "sigmoid"] if allow_chi2: gamma_kernels.append("chi2") gamma_condition = InCondition(gamma, kernel, gamma_kernels) cs.add_conditions([degree_depends_on_poly, coef0_condition, gamma_condition]) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() n_estimators = Constant("n_estimators", 100) criterion = CategoricalHyperparameter( "criterion", ["gini", "entropy"], default_value="gini") max_features = UniformFloatHyperparameter("max_features", 0, 1, default_value=0.5) max_depth = UnParametrizedHyperparameter(name="max_depth", value="None") max_leaf_nodes = UnParametrizedHyperparameter("max_leaf_nodes", "None") min_samples_split = UniformIntegerHyperparameter( "min_samples_split", 2, 20, default_value=2) min_samples_leaf = UniformIntegerHyperparameter( "min_samples_leaf", 1, 20, default_value=1) min_weight_fraction_leaf = UnParametrizedHyperparameter( 'min_weight_fraction_leaf', 0.) min_impurity_decrease = UnParametrizedHyperparameter( 'min_impurity_decrease', 0.) bootstrap = CategoricalHyperparameter( "bootstrap", ["True", "False"], default_value="False") cs.add_hyperparameters([n_estimators, criterion, max_features, max_depth, max_leaf_nodes, min_samples_split, min_samples_leaf, min_weight_fraction_leaf, min_impurity_decrease, bootstrap]) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() loss = CategoricalHyperparameter( "loss", ["ls", "lad", "huber", "quantile"], default_value="ls") learning_rate = UniformFloatHyperparameter( name="learning_rate", lower=0.01, upper=1, default_value=0.1, log=True) n_estimators = UniformIntegerHyperparameter( "n_estimators", 50, 500, default_value=100) max_depth = UniformIntegerHyperparameter( name="max_depth", lower=1, upper=10, default_value=3) min_samples_split = UniformIntegerHyperparameter( name="min_samples_split", lower=2, upper=20, default_value=2, log=False) min_samples_leaf = UniformIntegerHyperparameter( name="min_samples_leaf", lower=1, upper=20, default_value=1, log=False) min_weight_fraction_leaf = UnParametrizedHyperparameter( "min_weight_fraction_leaf", 0.) subsample = UniformFloatHyperparameter( name="subsample", lower=0.01, upper=1.0, default_value=1.0, log=False) max_features = UniformFloatHyperparameter( "max_features", 0.1, 1.0, default_value=1) max_leaf_nodes = UnParametrizedHyperparameter( name="max_leaf_nodes", value="None") min_impurity_decrease = UnParametrizedHyperparameter( name='min_impurity_decrease', value=0.0) alpha = UniformFloatHyperparameter( "alpha", lower=0.75, upper=0.99, default_value=0.9) cs.add_hyperparameters([loss, learning_rate, n_estimators, max_depth, min_samples_split, min_samples_leaf, min_weight_fraction_leaf, subsample, max_features, max_leaf_nodes, min_impurity_decrease, alpha]) cs.add_condition(InCondition(alpha, loss, ['huber', 'quantile'])) return cs
def get_hyperparameter_search_space(dataset_properties=None): gamma = UniformFloatHyperparameter( "gamma", 3.0517578125e-05, 8, default_value=1.0, log=True) n_components = UniformIntegerHyperparameter( "n_components", 50, 10000, default_value=100, log=True) cs = ConfigurationSpace() cs.add_hyperparameters([gamma, n_components]) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() alpha = UniformFloatHyperparameter( "alpha", 10 ** -5, 10., log=True, default_value=1.) fit_intercept = UnParametrizedHyperparameter("fit_intercept", "True") tol = UniformFloatHyperparameter("tol", 1e-5, 1e-1, default_value=1e-3, log=True) cs.add_hyperparameters([alpha, fit_intercept, tol]) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() use_minimum_fraction = CategoricalHyperparameter( "use_minimum_fraction", ["True", "False"], default_value="True") minimum_fraction = UniformFloatHyperparameter( "minimum_fraction", lower=.0001, upper=0.5, default_value=0.01, log=True) cs.add_hyperparameters([use_minimum_fraction, minimum_fraction]) cs.add_condition(EqualsCondition(minimum_fraction, use_minimum_fraction, 'True')) return cs
def get_hyperparameter_search_space(dataset_properties=None): percentile = UniformFloatHyperparameter( "percentile", lower=1, upper=99, default_value=50) score_func = CategoricalHyperparameter( name="score_func", choices=["f_regression", "mutual_info"]) cs = ConfigurationSpace() cs.add_hyperparameters([percentile, score_func]) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() q_min = UniformFloatHyperparameter( 'q_min', 0.001, 0.3, default_value=0.25 ) q_max = UniformFloatHyperparameter( 'q_max', 0.7, 0.999, default_value=0.75 ) cs.add_hyperparameters((q_min, q_max)) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() n_neighbors = UniformIntegerHyperparameter( name="n_neighbors", lower=1, upper=100, log=True, default_value=1) weights = CategoricalHyperparameter( name="weights", choices=["uniform", "distance"], default_value="uniform") p = CategoricalHyperparameter(name="p", choices=[1, 2], default_value=2) cs.add_hyperparameters([n_neighbors, weights, p]) return cs
def get_hyperparameter_search_space(dataset_properties=None): alpha = UniformFloatHyperparameter( name="alpha", lower=1e-14, upper=1.0, default_value=1e-8, log=True) thetaL = UniformFloatHyperparameter( name="thetaL", lower=1e-10, upper=1e-3, default_value=1e-6, log=True) thetaU = UniformFloatHyperparameter( name="thetaU", lower=1.0, upper=100000, default_value=100000.0, log=True) cs = ConfigurationSpace() cs.add_hyperparameters([alpha, thetaL, thetaU]) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() # TODO parametrize like the Random Forest as n_quantiles = n_features^param n_quantiles = UniformIntegerHyperparameter( 'n_quantiles', lower=10, upper=2000, default_value=1000 ) output_distribution = CategoricalHyperparameter( 'output_distribution', ['uniform', 'normal'] ) cs.add_hyperparameters((n_quantiles, output_distribution)) return cs
def get_hyperparameter_search_space(dataset_properties=None): # More than degree 3 is too expensive! degree = UniformIntegerHyperparameter("degree", 2, 3, 2) interaction_only = CategoricalHyperparameter("interaction_only", ["False", "True"], "False") include_bias = CategoricalHyperparameter("include_bias", ["True", "False"], "True") cs = ConfigurationSpace() cs.add_hyperparameters([degree, interaction_only, include_bias]) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() loss = CategoricalHyperparameter("loss", ["squared_loss", "huber", "epsilon_insensitive", "squared_epsilon_insensitive"], default_value="squared_loss") penalty = CategoricalHyperparameter( "penalty", ["l1", "l2", "elasticnet"], default_value="l2") alpha = UniformFloatHyperparameter( "alpha", 1e-7, 1e-1, log=True, default_value=0.0001) l1_ratio = UniformFloatHyperparameter( "l1_ratio", 1e-9, 1., log=True, default_value=0.15) fit_intercept = UnParametrizedHyperparameter( "fit_intercept", "True") tol = UniformFloatHyperparameter( "tol", 1e-5, 1e-1, default_value=1e-4, log=True) epsilon = UniformFloatHyperparameter( "epsilon", 1e-5, 1e-1, default_value=0.1, log=True) learning_rate = CategoricalHyperparameter( "learning_rate", ["optimal", "invscaling", "constant"], default_value="invscaling") eta0 = UniformFloatHyperparameter( "eta0", 1e-7, 1e-1, default_value=0.01, log=True) power_t = UniformFloatHyperparameter( "power_t", 1e-5, 1, default_value=0.25) average = CategoricalHyperparameter( "average", ["False", "True"], default_value="False") cs.add_hyperparameters([loss, penalty, alpha, l1_ratio, fit_intercept, tol, epsilon, learning_rate, eta0, power_t, average]) # TODO add passive/aggressive here, although not properly documented? elasticnet = EqualsCondition(l1_ratio, penalty, "elasticnet") epsilon_condition = InCondition(epsilon, loss, ["huber", "epsilon_insensitive", "squared_epsilon_insensitive"]) # eta0 is only relevant if learning_rate!='optimal' according to code # https://github.com/scikit-learn/scikit-learn/blob/0.19.X/sklearn/ # linear_model/sgd_fast.pyx#L603 eta0_in_inv_con = InCondition(eta0, learning_rate, ["invscaling", "constant"]) power_t_condition = EqualsCondition(power_t, learning_rate, "invscaling") cs.add_conditions([elasticnet, epsilon_condition, power_t_condition, eta0_in_inv_con]) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() n_estimators = UniformIntegerHyperparameter( name="n_estimators", lower=50, upper=500, default_value=50, log=False) learning_rate = UniformFloatHyperparameter( name="learning_rate", lower=0.01, upper=2, default_value=0.1, log=True) algorithm = CategoricalHyperparameter( name="algorithm", choices=["SAMME.R", "SAMME"], default_value="SAMME.R") max_depth = UniformIntegerHyperparameter( name="max_depth", lower=1, upper=10, default_value=1, log=False) cs.add_hyperparameters([n_estimators, learning_rate, algorithm, max_depth]) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() # the smoothing parameter is a non-negative float # I will limit it to 1000 and put it on a logarithmic scale. (SF) # Please adjust that, if you know a proper range, this is just a guess. alpha = UniformFloatHyperparameter(name="alpha", lower=1e-2, upper=100, default_value=1, log=True) fit_prior = CategoricalHyperparameter(name="fit_prior", choices=["True", "False"], default_value="True") cs.add_hyperparameters([alpha, fit_prior]) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() n_components = UniformIntegerHyperparameter( "n_components", 10, 2000, default_value=100) algorithm = CategoricalHyperparameter('algorithm', ['parallel', 'deflation'], 'parallel') whiten = CategoricalHyperparameter('whiten', ['False', 'True'], 'False') fun = CategoricalHyperparameter( 'fun', ['logcosh', 'exp', 'cube'], 'logcosh') cs.add_hyperparameters([n_components, algorithm, whiten, fun]) cs.add_condition(EqualsCondition(n_components, whiten, "True")) return cs
def get_hyperparameter_search_space(dataset_properties=None): C = UniformFloatHyperparameter("C", 1e-5, 10, 1.0, log=True) fit_intercept = UnParametrizedHyperparameter("fit_intercept", "True") loss = CategoricalHyperparameter( "loss", ["hinge", "squared_hinge"], default_value="hinge" ) tol = UniformFloatHyperparameter("tol", 1e-5, 1e-1, default_value=1e-4, log=True) # Note: Average could also be an Integer if > 1 average = CategoricalHyperparameter('average', ['False', 'True'], default_value='False') cs = ConfigurationSpace() cs.add_hyperparameters([loss, fit_intercept, tol, C, average]) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() n_clusters = UniformIntegerHyperparameter("n_clusters", 2, 400, 25) affinity = CategoricalHyperparameter( "affinity", ["euclidean", "manhattan", "cosine"], "euclidean") linkage = CategoricalHyperparameter( "linkage", ["ward", "complete", "average"], "ward") pooling_func = CategoricalHyperparameter( "pooling_func", ["mean", "median", "max"]) cs.add_hyperparameters([n_clusters, affinity, linkage, pooling_func]) affinity_and_linkage = ForbiddenAndConjunction( ForbiddenInClause(affinity, ["manhattan", "cosine"]), ForbiddenEqualsClause(linkage, "ward")) cs.add_forbidden_clause(affinity_and_linkage) return cs
def get_hyperparameter_search_space(dataset_properties=None): n_components = UniformIntegerHyperparameter( "n_components", 10, 2000, default_value=100) kernel = CategoricalHyperparameter('kernel', ['poly', 'rbf', 'sigmoid', 'cosine'], 'rbf') gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8, log=True, default_value=1.0) degree = UniformIntegerHyperparameter('degree', 2, 5, 3) coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0) cs = ConfigurationSpace() cs.add_hyperparameters([n_components, kernel, degree, gamma, coef0]) degree_depends_on_poly = EqualsCondition(degree, kernel, "poly") coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"]) gamma_condition = InCondition(gamma, kernel, ["poly", "rbf"]) cs.add_conditions([degree_depends_on_poly, coef0_condition, gamma_condition]) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() # base_estimator = Constant(name="base_estimator", value="None") n_estimators = UniformIntegerHyperparameter( name="n_estimators", lower=50, upper=500, default_value=50, log=False) learning_rate = UniformFloatHyperparameter( name="learning_rate", lower=0.01, upper=2, default_value=0.1, log=True) loss = CategoricalHyperparameter( name="loss", choices=["linear", "square", "exponential"], default_value="linear") max_depth = UniformIntegerHyperparameter( name="max_depth", lower=1, upper=10, default_value=1, log=False) cs.add_hyperparameters([n_estimators, learning_rate, loss, max_depth]) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() loss = Constant("loss", "deviance") learning_rate = UniformFloatHyperparameter(name="learning_rate", lower=0.01, upper=1, default_value=0.1, log=True) n_estimators = UniformIntegerHyperparameter("n_estimators", 50, 500, default_value=100) max_depth = UniformIntegerHyperparameter(name="max_depth", lower=1, upper=10, default_value=3) criterion = CategoricalHyperparameter('criterion', ['friedman_mse', 'mse', 'mae'], default_value='mse') min_samples_split = UniformIntegerHyperparameter( name="min_samples_split", lower=2, upper=20, default_value=2) min_samples_leaf = UniformIntegerHyperparameter( name="min_samples_leaf", lower=1, upper=20, default_value=1) min_weight_fraction_leaf = UnParametrizedHyperparameter( "min_weight_fraction_leaf", 0.) subsample = UniformFloatHyperparameter(name="subsample", lower=0.01, upper=1.0, default_value=1.0) max_features = UniformFloatHyperparameter("max_features", 0.1, 1.0, default_value=1) max_leaf_nodes = UnParametrizedHyperparameter(name="max_leaf_nodes", value="None") min_impurity_decrease = UnParametrizedHyperparameter( name='min_impurity_decrease', value=0.0) cs.add_hyperparameters([ loss, learning_rate, n_estimators, max_depth, criterion, min_samples_split, min_samples_leaf, min_weight_fraction_leaf, subsample, max_features, max_leaf_nodes, min_impurity_decrease ]) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() loss = CategoricalHyperparameter("loss", ["hinge", "log", "modified_huber", "squared_hinge", "perceptron"], default_value="log") penalty = CategoricalHyperparameter( "penalty", ["l1", "l2", "elasticnet"], default_value="l2") alpha = UniformFloatHyperparameter( "alpha", 1e-7, 1e-1, log=True, default_value=0.0001) l1_ratio = UniformFloatHyperparameter( "l1_ratio", 1e-9, 1, log=True, default_value=0.15) fit_intercept = UnParametrizedHyperparameter("fit_intercept", "True") epsilon = UniformFloatHyperparameter( "epsilon", 1e-5, 1e-1, default_value=1e-4, log=True) learning_rate = CategoricalHyperparameter( "learning_rate", ["optimal", "invscaling", "constant"], default_value="invscaling") eta0 = UniformFloatHyperparameter( "eta0", 1e-7, 1e-1, default_value=0.01, log=True) power_t = UniformFloatHyperparameter("power_t", 1e-5, 1, default_value=0.5) average = CategoricalHyperparameter( "average", ["False", "True"], default_value="False") cs.add_hyperparameters([loss, penalty, alpha, l1_ratio, fit_intercept, epsilon, learning_rate, eta0, power_t, average]) # TODO add passive/aggressive here, although not properly documented? elasticnet = EqualsCondition(l1_ratio, penalty, "elasticnet") epsilon_condition = EqualsCondition(epsilon, loss, "modified_huber") power_t_condition = EqualsCondition(power_t, learning_rate, "invscaling") # eta0 is only relevant if learning_rate!='optimal' according to code # https://github.com/scikit-learn/scikit-learn/blob/0.19.X/sklearn/ # linear_model/sgd_fast.pyx#L603 eta0_in_inv_con = InCondition(eta0, learning_rate, ["invscaling", "constant"]) cs.add_conditions([elasticnet, epsilon_condition, power_t_condition, eta0_in_inv_con]) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() Window_size = UniformIntegerHyperparameter( name="Window_size", lower=5, upper=50, default_value=20) Difference = CategoricalHyperparameter( name="Difference", choices=["True", "False"], default_value="True") tsfresh_feature = CategoricalHyperparameter( name="tsfresh_feature", choices=["True", "False"], default_value="True") n_estimators = UniformIntegerHyperparameter("n_estimators", 10, 200, default_value=100) criterion = CategoricalHyperparameter("criterion", ['mse', 'friedman_mse', 'mae'], default_value='mse') max_features = UniformFloatHyperparameter( "max_features", 0.1, 1.0, default_value=1.0) max_depth = UnParametrizedHyperparameter("max_depth", "None") min_samples_split = UniformIntegerHyperparameter( "min_samples_split", 2, 20, default_value=2) min_samples_leaf = UniformIntegerHyperparameter( "min_samples_leaf", 1, 20, default_value=1) min_weight_fraction_leaf = UnParametrizedHyperparameter("min_weight_fraction_leaf", 0.) max_leaf_nodes = UnParametrizedHyperparameter("max_leaf_nodes", "None") min_impurity_decrease = UnParametrizedHyperparameter( 'min_impurity_decrease', 0.0) bootstrap = CategoricalHyperparameter( "bootstrap", ["True", "False"], default_value="True") cs.add_hyperparameters([n_estimators, criterion, max_features, Difference, max_depth, min_samples_split, min_samples_leaf, min_weight_fraction_leaf, max_leaf_nodes, min_impurity_decrease, bootstrap, tsfresh_feature, Window_size]) return cs
def get_hyperparameter_search_space(dataset_properties=None, optimizer='smac'): if optimizer == 'smac': alpha = UniformFloatHyperparameter("alpha", 0.01, 32, log=True, default_value=1.0) tol = UniformFloatHyperparameter("tol", 1e-6, 1e-2, default_value=1e-4, log=True) max_iter = UniformFloatHyperparameter("max_iter", 100, 1000, q=100, default_value=100) solver = CategoricalHyperparameter("solver", choices=["auto", "saga"], default_value="auto") cs = ConfigurationSpace() cs.add_hyperparameters([alpha, tol, max_iter, solver]) return cs elif optimizer == 'tpe': from hyperopt import hp space = { 'alpha': hp.loguniform('ridge_alpha', np.log(0.01), np.log(32)), 'tol': hp.loguniform('ridge_tol', np.log(1e-6), np.log(1e-2)), 'max_iter': hp.uniform('ridge_max_iter', 100, 1000), 'solver': hp.choice('ridge_solver', ["auto", "saga"]) } init_trial = { 'alpha': 1, 'tol': 1e-4, 'max_iter': 100, 'solver': "auto" } return space
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() solver = CategoricalHyperparameter(name="solver", choices=['svd', 'lsqr', 'eigen'], default_value='svd') shrinkage = UniformFloatHyperparameter(name="shrinkage", lower=0.0, upper=1.0, default_value=0.5) n_components = UniformIntegerHyperparameter(name="n_components", lower=1, upper=29, default_value=10) tol = UniformFloatHyperparameter(name="tol", lower=0.0001, upper=1, default_value=0.0001) cs.add_hyperparameters([solver, shrinkage, n_components, tol]) return cs
def get_hyperparameter_search_space(**kwargs): cs = ConfigurationSpace() n_estimators = UniformIntegerHyperparameter("n_estimators", 50, 500, default_value=50, log=False) learning_rate = UniformFloatHyperparameter(name="learning_rate", lower=0.01, upper=2., default_value=1.0, log=True) algorithm = CategoricalHyperparameter("algorithm", ["SAMME", "SAMME.R"], default_value="SAMME.R") cs.add_hyperparameters([learning_rate, n_estimators, algorithm]) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() # the smoothing parameter is a non-negative float # I will limit it to 100 and put it on a logarithmic scale. (SF) # Please adjust that, if you know a proper range, this is just a guess. alpha = UniformFloatHyperparameter(name="alpha", lower=1e-2, upper=100, default_value=1, log=True) fit_prior = CategoricalHyperparameter(name="fit_prior", choices=["True", "False"], default_value="True") cs.add_hyperparameters([alpha, fit_prior]) return cs
def get_hyperparameter_search_space( dataset_properties: Optional[Dict[str, BaseDatasetPropertiesType]] = None, use_augmenter: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_augmenter", value_range=(True, False), default_value=True, ), sigma_offset: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="sigma_offset", value_range=(0.0, 3.0), default_value=0.3, ), ) -> ConfigurationSpace: cs = ConfigurationSpace() use_augmenter = get_hyperparameter(use_augmenter, CategoricalHyperparameter) sigma_offset = get_hyperparameter(sigma_offset, UniformFloatHyperparameter) cs.add_hyperparameters([use_augmenter, sigma_offset]) # only add hyperparameters to configuration space if we are using the augmenter cs.add_condition(CS.EqualsCondition(sigma_offset, use_augmenter, True)) return cs
def get_hyperparameter_search_space(dataset_properties=None): percentile = UniformFloatHyperparameter(name="percentile", lower=1, upper=99, default_value=50) score_func = CategoricalHyperparameter( name="score_func", choices=["chi2", "f_classif", "mutual_info"], default_value="chi2") if dataset_properties is not None: # Chi2 can handle sparse data, so we respect this if 'sparse' in dataset_properties and dataset_properties['sparse']: score_func = Constant(name="score_func", value="chi2") cs = ConfigurationSpace() cs.add_hyperparameters([percentile, score_func]) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() n_components = UniformIntegerHyperparameter("n_components", 10, 2000, default_value=100) algorithm = CategoricalHyperparameter('algorithm', ['parallel', 'deflation'], 'parallel') whiten = CategoricalHyperparameter('whiten', ['False', 'True'], 'False') fun = CategoricalHyperparameter('fun', ['logcosh', 'exp', 'cube'], 'logcosh') cs.add_hyperparameters([n_components, algorithm, whiten, fun]) cs.add_condition(EqualsCondition(n_components, whiten, "True")) return cs
def get_hyperparameter_search_space(dataset_properties=None, optimizer='smac'): if optimizer == 'smac': cs = ConfigurationSpace() epsilon = CategoricalHyperparameter("epsilon", [1e-4, 1e-3, 1e-2, 1e-1, 1], default_value=1e-4) loss = CategoricalHyperparameter( "loss", ["epsilon_insensitive", "squared_epsilon_insensitive"], default_value="epsilon_insensitive") dual = CategoricalHyperparameter("dual", ['True', 'False'], default_value='True') tol = UniformFloatHyperparameter( "tol", 1e-5, 1e-1, default_value=1e-4, log=True) C = UniformFloatHyperparameter( "C", 0.03125, 32768, log=True, default_value=1.0) fit_intercept = Constant("fit_intercept", "True") intercept_scaling = Constant("intercept_scaling", 1) cs.add_hyperparameters([epsilon, loss, dual, tol, C, fit_intercept, intercept_scaling]) dual_and_loss = ForbiddenAndConjunction( ForbiddenEqualsClause(dual, "False"), ForbiddenEqualsClause(loss, "epsilon_insensitive") ) cs.add_forbidden_clause(dual_and_loss) return cs elif optimizer == 'tpe': from hyperopt import hp space = {'loss': hp.choice('liblinear_combination', [{'loss': "epsilon_insensitive", 'dual': "True"}, {'loss': "squared_epsilon_insensitive", 'dual': "True"}, {'loss': "squared_epsilon_insensitive", 'dual': "False"}]), 'dual': None, 'tol': hp.loguniform('liblinear_tol', np.log(1e-5), np.log(1e-1)), 'C': hp.loguniform('liblinear_C', np.log(0.03125), np.log(32768)), 'fit_intercept': hp.choice('liblinear_fit_intercept', ["True"]), 'intercept_scaling': hp.choice('liblinear_intercept_scaling', [1])} init_trial = {'loss': {'loss': "epsilon_insensitive", 'dual': "True"}, 'tol': 1e-4, 'C': 1, 'fit_intercept': "True", 'intercept_scaling': 1} return space
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() # n_estimators = Constant("n_estimators", 100) n_estimators = UniformIntegerHyperparameter(name="n_estimators", lower=10, upper=600, default_value=100) criterion = CategoricalHyperparameter("criterion", ["gini", "entropy"], default_value="gini") # The maximum number of features used in the forest is calculated as m^max_features, where # m is the total number of features, and max_features is the hyperparameter specified below. # The default is 0.5, which yields sqrt(m) features as max_features in the estimator. This # corresponds with Geurts' heuristic. max_features = UniformFloatHyperparameter("max_features", 0., 1., default_value=0.5) max_depth = UnParametrizedHyperparameter("max_depth", "None") # max_depth = UniformIntegerHyperparameter("max_depth", 1, 20, default_value=10) min_samples_split = UniformIntegerHyperparameter("min_samples_split", 2, 20, default_value=2) min_samples_leaf = UniformIntegerHyperparameter("min_samples_leaf", 0, 20, default_value=1) min_weight_fraction_leaf = UnParametrizedHyperparameter( "min_weight_fraction_leaf", 0.) max_leaf_nodes = UnParametrizedHyperparameter("max_leaf_nodes", "None") min_impurity_decrease = UnParametrizedHyperparameter( 'min_impurity_decrease', 0.0) bootstrap = CategoricalHyperparameter("bootstrap", ["True", "False"], default_value="True") cs.add_hyperparameters([ n_estimators, criterion, max_features, max_depth, min_samples_split, min_samples_leaf, min_weight_fraction_leaf, max_leaf_nodes, bootstrap, min_impurity_decrease ]) return cs
def get_hyperparameter_search_space(dataset_properties=None, optimizer='smac'): if optimizer == 'smac': cs = ConfigurationSpace() shrinkage = CategoricalHyperparameter("shrinkage", ["None", "auto", "manual"], default_value="None") shrinkage_factor = UniformFloatHyperparameter( "shrinkage_factor", 0., 1., 0.5) n_components = UniformIntegerHyperparameter('n_components', 1, 250, default_value=1) tol = UniformFloatHyperparameter("tol", 1e-5, 1e-1, default_value=1e-4, log=True) cs.add_hyperparameters( [shrinkage, shrinkage_factor, n_components, tol]) cs.add_condition( EqualsCondition(shrinkage_factor, shrinkage, "manual")) return cs elif optimizer == 'tpe': from hyperopt import hp space = { 'n_components': hp.randint('lda_n_components', 250) + 1, 'tol': hp.loguniform('lda_tol', np.log(1e-5), np.log(1e-1)), 'shrinkage': hp.choice('lda_shrinkage', [ "None", "auto", ("manual", { 'shrinkage_factor': hp.uniform('lda_shrinkage_factor', 0, 1) }) ]) } init_trial = {'n_components': 10, 'tol': 1e-4, 'shrinkage': "None"} return space
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() penalty = CategoricalHyperparameter("penalty", ["l1", "l2"], default="l2") loss = CategoricalHyperparameter("loss", ["hinge", "squared_hinge"], default="squared_hinge") dual = Constant("dual", "False") # This is set ad-hoc tol = UniformFloatHyperparameter("tol", 1e-5, 1e-1, default=1e-4, log=True) C = UniformFloatHyperparameter("C", 0.03125, 32768, log=True, default=1.0) multi_class = Constant("multi_class", "ovr") # These are set ad-hoc fit_intercept = Constant("fit_intercept", "True") intercept_scaling = Constant("intercept_scaling", 1) cs.add_hyperparameters([ penalty, loss, dual, tol, C, multi_class, fit_intercept, intercept_scaling ]) penalty_and_loss = ForbiddenAndConjunction( ForbiddenEqualsClause(penalty, "l1"), ForbiddenEqualsClause(loss, "hinge")) constant_penalty_and_loss = ForbiddenAndConjunction( ForbiddenEqualsClause(dual, "False"), ForbiddenEqualsClause(penalty, "l2"), ForbiddenEqualsClause(loss, "hinge")) penalty_and_dual = ForbiddenAndConjunction( ForbiddenEqualsClause(dual, "False"), ForbiddenEqualsClause(penalty, "l1")) cs.add_forbidden_clause(penalty_and_loss) cs.add_forbidden_clause(constant_penalty_and_loss) cs.add_forbidden_clause(penalty_and_dual) return cs
def get_hyperparameter_search_space(dataset_properties=None): # Copied from libsvm_c C = UniformFloatHyperparameter( name="C", lower=0.03125, upper=32768, log=True, default=1.0) kernel = CategoricalHyperparameter( name="kernel", choices=['linear', 'poly', 'rbf', 'sigmoid'], default="rbf") degree = UniformIntegerHyperparameter( name="degree", lower=1, upper=5, default=3) # Changed the gamma value to 0.0 (is 0.1 for classification) gamma = UniformFloatHyperparameter( name="gamma", lower=3.0517578125e-05, upper=8, log=True, default=0.1) # TODO this is totally ad-hoc coef0 = UniformFloatHyperparameter( name="coef0", lower=-1, upper=1, default=0) # probability is no hyperparameter, but an argument to the SVM algo shrinking = CategoricalHyperparameter( name="shrinking", choices=["True", "False"], default="True") tol = UniformFloatHyperparameter( name="tol", lower=1e-5, upper=1e-1, default=1e-3, log=True) max_iter = UnParametrizedHyperparameter("max_iter", -1) # Random Guess epsilon = UniformFloatHyperparameter(name="epsilon", lower=0.001, upper=1, default=0.1, log=True) cs = ConfigurationSpace() cs.add_hyperparameters([C, kernel, degree, gamma, coef0, shrinking, tol, max_iter, epsilon]) degree_depends_on_kernel = InCondition(child=degree, parent=kernel, values=('poly', 'rbf', 'sigmoid')) gamma_depends_on_kernel = InCondition(child=gamma, parent=kernel, values=('poly', 'rbf')) coef0_depends_on_kernel = InCondition(child=coef0, parent=kernel, values=('poly', 'sigmoid')) cs.add_conditions([degree_depends_on_kernel, gamma_depends_on_kernel, coef0_depends_on_kernel]) return cs
def get_cs(): epsilon = CategoricalHyperparameter("epsilon", [1e-4, 1e-3, 1e-2, 1e-1, 1], default_value=1e-4) C = UniformFloatHyperparameter("C", 0.03125, 32768, log=True, default_value=1.0) # No linear kernel here, because we have liblinear kernel = CategoricalHyperparameter(name="kernel", choices=["rbf", "poly", "sigmoid"], default_value="rbf") degree = UniformIntegerHyperparameter("degree", 2, 5, default_value=3) gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8, log=True, default_value=0.1) coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0) # probability is no hyperparameter, but an argument to the SVM algo shrinking = CategoricalHyperparameter("shrinking", ["True", "False"], default_value="True") tol = UniformFloatHyperparameter("tol", 1e-5, 1e-1, default_value=1e-3, log=True) # cache size is not a hyperparameter, but an argument to the program! max_iter = UnParametrizedHyperparameter("max_iter", 2000) cs = ConfigurationSpace() cs.add_hyperparameters([ epsilon, C, kernel, degree, gamma, coef0, shrinking, tol, max_iter ]) degree_depends_on_poly = EqualsCondition(degree, kernel, "poly") coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"]) cs.add_condition(degree_depends_on_poly) cs.add_condition(coef0_condition) return cs
def get_cs(): cs = ConfigurationSpace() n_estimators = UniformIntegerHyperparameter("n_estimators", 100, 1000, default_value=500) num_leaves = UniformIntegerHyperparameter("num_leaves", 31, 1023, default_value=31) learning_rate = UniformFloatHyperparameter("learning_rate", 0.025, 0.3, default_value=0.1, log=True) min_child_weight = UniformIntegerHyperparameter("min_child_weight", 1, 10, default_value=1) subsample = UniformFloatHyperparameter("subsample", 0.5, 1, default_value=1) colsample_bytree = UniformFloatHyperparameter("colsample_bytree", 0.5, 1, default_value=1) reg_alpha = UniformFloatHyperparameter('reg_alpha', 1e-10, 10, log=True, default_value=1e-10) reg_lambda = UniformFloatHyperparameter("reg_lambda", 1e-10, 10, log=True, default_value=1e-10) cs.add_hyperparameters([ n_estimators, num_leaves, learning_rate, min_child_weight, subsample, colsample_bytree, reg_alpha, reg_lambda ]) return cs
def get_hyperparameter_search_space(dataset_properties=None, optimizer='smac'): if optimizer == 'smac': cs = ConfigurationSpace() n_estimators = Constant("n_estimators", 100) criterion = CategoricalHyperparameter( "criterion", ["gini", "entropy"], default_value="gini") max_features = UniformFloatHyperparameter("max_features", 0, 1, default_value=0.5, q=0.05) max_depth = UnParametrizedHyperparameter(name="max_depth", value="None") max_leaf_nodes = UnParametrizedHyperparameter("max_leaf_nodes", "None") min_samples_split = UniformIntegerHyperparameter( "min_samples_split", 2, 20, default_value=2) min_samples_leaf = UniformIntegerHyperparameter( "min_samples_leaf", 1, 20, default_value=1) min_weight_fraction_leaf = UnParametrizedHyperparameter( 'min_weight_fraction_leaf', 0.) min_impurity_decrease = UnParametrizedHyperparameter( 'min_impurity_decrease', 0.) bootstrap = CategoricalHyperparameter( "bootstrap", ["True", "False"], default_value="False") cs.add_hyperparameters([n_estimators, criterion, max_features, max_depth, max_leaf_nodes, min_samples_split, min_samples_leaf, min_weight_fraction_leaf, min_impurity_decrease, bootstrap]) return cs elif optimizer == 'tpe': from hyperopt import hp space = {'n_estimators': 100, 'criterion': hp.choice('ets_criterion', ['gini', 'entropy']), 'max_features': hp.uniform('ets_max_features', 0, 1), 'max_depth': "None", 'max_leaf_nodes': "None", 'min_samples_leaf': hp.randint('ets_samples_leaf', 20) + 1, 'min_samples_split': hp.randint('ets_samples_split', 19) + 2, 'min_weight_fraction_leaf': 0., 'min_impurity_decrease': 0., 'bootstrap': hp.choice('ets_bootstrap', ['True', 'False'])} return space
def get_hyperparameter_search_space(dataset_properties=None, optimizer='smac'): if optimizer == 'smac': cs = ConfigurationSpace() n_estimators = UniformIntegerHyperparameter( name="n_estimators", lower=50, upper=500, default_value=50, log=False) learning_rate = UniformFloatHyperparameter( name="learning_rate", lower=0.01, upper=2, default_value=0.1, log=True) max_depth = UniformIntegerHyperparameter( name="max_depth", lower=1, upper=10, default_value=1, log=False) cs.add_hyperparameters([n_estimators, learning_rate, max_depth]) return cs elif optimizer == 'tpe': space = {'n_estimators': hp.randint('ab_n_estimators', 451) + 50, 'learning_rate': hp.loguniform('ab_learning_rate', np.log(0.01), np.log(2)), 'max_depth': hp.randint('ab_max_depth', 10) + 1} init_trial = {'n_estimators': 50, 'learning_rate': 0.1, 'algorithm': "SAMME.R", 'max_depth': 1} return space
def get_hyperparameter_search_space(dataset_properties=None, optimizer='smac'): if optimizer == 'smac': alpha = UniformFloatHyperparameter("alpha", 0.01, 32, log=True, default_value=1.0) tol = UniformFloatHyperparameter("tol", 1e-6, 1e-2, default_value=1e-4, log=True) max_iter = UniformFloatHyperparameter("max_iter", 100, 1000, q=100, default_value=100) cs = ConfigurationSpace() cs.add_hyperparameters([alpha, tol, max_iter]) return cs elif optimizer == 'tpe': from hyperopt import hp space = {'alpha': hp.loguniform('lasso_alpha', np.log(0.01), np.log(32)), 'tol': hp.loguniform('lasso_tol', np.log(1e-6), np.log(1e-2)), 'max_iter': hp.uniform('lasso_max_iter', 100, 1000)} init_trial = {'alpha': 1, 'tol': 1e-4, 'max_iter': 100} return space
def get_hyperparameter_search_space(**kwargs): cs = ConfigurationSpace() learning_rate = UniformFloatHyperparameter(name="learning_rate", lower=0.01, upper=1, default_value=0.1, log=True) min_samples_leaf = UniformFloatHyperparameter("min_samples_leaf_factor", 0.0001, 0.25, default_value=0.0001, log=True) max_depth_factor = UniformFloatHyperparameter("max_depth_factor", 1e-5, 2.5, default_value=1.) max_leaf_nodes_factor = UniformFloatHyperparameter("max_leaf_nodes_factor", 1e-5, 1., default_value=1.) l2_regularization = UniformFloatHyperparameter(name="l2_regularization", lower=1e-10, upper=1., log=True, default_value=1e-10) n_iter_no_change = UniformIntegerHyperparameter(name="n_iter_no_change", lower=1, upper=20, default_value=10) validation_fraction = UniformFloatHyperparameter(name="validation_fraction", lower=0.01, upper=0.4, default_value=0.1) cs.add_hyperparameters( [learning_rate, min_samples_leaf, max_depth_factor, max_leaf_nodes_factor, l2_regularization, n_iter_no_change, validation_fraction]) return cs
def get_hyperparameter_search_space(dataset_properties=None): alpha = UniformFloatHyperparameter(name="alpha", lower=1e-14, upper=1.0, default_value=1e-8, log=True) thetaL = UniformFloatHyperparameter(name="thetaL", lower=1e-10, upper=1e-3, default_value=1e-6, log=True) thetaU = UniformFloatHyperparameter(name="thetaU", lower=1.0, upper=100000, default_value=100000.0, log=True) cs = ConfigurationSpace() cs.add_hyperparameters([alpha, thetaL, thetaU]) return cs
def get_hyperparameter_search_space(dataset_properties=None): n_components = UniformIntegerHyperparameter( "n_components", 10, 2000, default_value=100) kernel = CategoricalHyperparameter('kernel', ['poly', 'rbf', 'sigmoid', 'cosine'], 'rbf') gamma = UniformFloatHyperparameter( "gamma", 3.0517578125e-05, 8, log=True, default_value=1.0, ) degree = UniformIntegerHyperparameter('degree', 2, 5, 3) coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0) cs = ConfigurationSpace() cs.add_hyperparameters([n_components, kernel, degree, gamma, coef0]) degree_depends_on_poly = EqualsCondition(degree, kernel, "poly") coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"]) gamma_condition = InCondition(gamma, kernel, ["poly", "rbf"]) cs.add_conditions([degree_depends_on_poly, coef0_condition, gamma_condition]) return cs
def get_cs(): cs = ConfigurationSpace() shrinkage = CategoricalHyperparameter("shrinkage", ["None", "auto", "manual"], default_value="None") shrinkage_factor = UniformFloatHyperparameter("shrinkage_factor", 0., 1., 0.5) n_components = UniformIntegerHyperparameter('n_components', 1, 250, default_value=10) tol = UniformFloatHyperparameter("tol", 1e-5, 1e-1, default_value=1e-4, log=True) cs.add_hyperparameters([shrinkage, shrinkage_factor, n_components, tol]) cs.add_condition(EqualsCondition(shrinkage_factor, shrinkage, "manual")) return cs
def get_hyperparameter_search_space(dataset_properties=None): nugget = UniformFloatHyperparameter(name="nugget", lower=0.0001, upper=10, default=0.1, log=True) thetaL = UniformFloatHyperparameter(name="thetaL", lower=1e-6, upper=1e-3, default=1e-4, log=True) thetaU = UniformFloatHyperparameter(name="thetaU", lower=0.2, upper=10, default=1.0, log=True) cs = ConfigurationSpace() cs.add_hyperparameters([nugget, thetaL, thetaU]) return cs
def get_adaboost_default_search_space(dataset_properties=None): classif_prefix = 'classifier:adaboost:' cs = ConfigurationSpace() model_type = CategoricalHyperparameter('classifier:__choice__', ['adaboost']) imputation = CategoricalHyperparameter('imputation:strategy', ['mean', 'median', 'most_frequent']) n_estimators = UniformIntegerHyperparameter( name=classif_prefix + "n_estimators", lower=50, upper=500, default_value=50, log=False) learning_rate = UniformFloatHyperparameter( name=classif_prefix + "learning_rate", lower=0.01, upper=2, default_value=0.1, log=True) algorithm = CategoricalHyperparameter( name=classif_prefix + "algorithm", choices=["SAMME.R", "SAMME"], default_value="SAMME.R") max_depth = UniformIntegerHyperparameter( name=classif_prefix + "max_depth", lower=1, upper=10, default_value=1, log=False) cs.add_hyperparameters([model_type, imputation, n_estimators, learning_rate, algorithm, max_depth]) return cs
def make_config_space(dataset_shape: tuple) -> ConfigurationSpace: r"""Build a ConfigurationSpace object encompassing all different autoencoder types and parameters. The resulting ConfigurationSpace object depends on the given dataset shape. Adapting it to different datasets requires manual adjustments (see :func:`build_autoencoder_from_existing`). :param dataset_shape: Shape of the dataset this ConfigurationSpace should be valid for. :return: A ConfigurationSpace object tied to the given dataset size. .. note: Changes to this function invalidate previous optimization results. """ input_dim = dataset_shape[-1] # Build Configuration Space which defines all parameters and their ranges cs = ConfigurationSpace() ae_type = CategoricalHyperparameter('ae_type', ['deep', 'deep_ksparse'], default_value='deep_ksparse') act_type = CategoricalHyperparameter('act_type', ['relu', 'sigmoid', 'tanh'], default_value='relu') epochs = UniformIntegerHyperparameter('epochs', 1, 50, default_value=10) cs.add_hyperparameters([ae_type, act_type, epochs]) num_layers = UniformIntegerHyperparameter('num_hidden_layers', 1, 5, default_value=3) latent_dim = UniformIntegerHyperparameter('latent_dim', 2, input_dim // 2, default_value=2) cs.add_hyperparameters([num_layers, latent_dim]) ksparse_k = UniformIntegerHyperparameter('k', 50, 200, default_value=200) cs.add_hyperparameter(ksparse_k) cs.add_condition( InCondition(child=ksparse_k, parent=ae_type, values=['deep_ksparse'])) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() #Parametrized learning_rate = UniformFloatHyperparameter( name='learning_rate', lower=0.01, upper=1, default=0.1) feature_fraction = UniformFloatHyperparameter( name='feature_fraction', lower=0.5, upper=1.0, default=0.9) num_leaves = UniformIntegerHyperparameter( name='num_leaves', lower=5, upper=500, default=31) min_data_in_leaf = UniformIntegerHyperparameter( name='min_data_in_leaf', lower=1, upper=30, default=20) lambda_l1 = UniformFloatHyperparameter( name='lambda_l1', lower=0.0, upper=10.0, default=0.0) lambda_l2 = UniformFloatHyperparameter( name='lambda_l2', lower=0.0, upper=10.0, default=0.0) max_bin = UniformIntegerHyperparameter( name='max_bin', lower=50, upper=500, default=255) #UnParametrized bagging_fraction = UnParametrizedHyperparameter( name='bagging_fraction', value=1.0) bagging_freq = UnParametrizedHyperparameter( name='bagging_freq', value=5) max_depth = UnParametrizedHyperparameter( name='max_depth', value=-1) min_data_in_bin = UnParametrizedHyperparameter( name='min_data_in_bin', value=5) min_gain_to_split = UnParametrizedHyperparameter( name='min_gain_to_split', value=0) min_sum_hessian_in_leaf = UnParametrizedHyperparameter( name='min_sum_hessian_in_leaf', value=1e-3) cs.add_hyperparameters([learning_rate, feature_fraction, bagging_fraction, num_leaves, min_data_in_leaf, lambda_l1, lambda_l2, min_data_in_bin, max_depth, max_bin, min_sum_hessian_in_leaf, min_gain_to_split]) return cs
def get_hyperparameter_search_space(dataset_properties=None): C = UniformFloatHyperparameter( name="C", lower=0.03125, upper=32768, log=True, default_value=1.0) # Random Guess epsilon = UniformFloatHyperparameter(name="epsilon", lower=0.001, upper=1, default_value=0.1, log=True) kernel = CategoricalHyperparameter( name="kernel", choices=['linear', 'poly', 'rbf', 'sigmoid'], default_value="rbf") degree = UniformIntegerHyperparameter( name="degree", lower=2, upper=5, default_value=3) gamma = UniformFloatHyperparameter( name="gamma", lower=3.0517578125e-05, upper=8, log=True, default_value=0.1) # TODO this is totally ad-hoc coef0 = UniformFloatHyperparameter( name="coef0", lower=-1, upper=1, default_value=0) # probability is no hyperparameter, but an argument to the SVM algo shrinking = CategoricalHyperparameter( name="shrinking", choices=["True", "False"], default_value="True") tol = UniformFloatHyperparameter( name="tol", lower=1e-5, upper=1e-1, default_value=1e-3, log=True) max_iter = UnParametrizedHyperparameter("max_iter", -1) cs = ConfigurationSpace() cs.add_hyperparameters([C, kernel, degree, gamma, coef0, shrinking, tol, max_iter, epsilon]) degree_depends_on_kernel = InCondition(child=degree, parent=kernel, values=('poly', 'rbf', 'sigmoid')) gamma_depends_on_kernel = InCondition(child=gamma, parent=kernel, values=('poly', 'rbf')) coef0_depends_on_kernel = InCondition(child=coef0, parent=kernel, values=('poly', 'sigmoid')) cs.add_conditions([degree_depends_on_kernel, gamma_depends_on_kernel, coef0_depends_on_kernel]) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() Window_size = UniformIntegerHyperparameter(name="Window_size", lower=5, upper=50, default_value=20) Difference = CategoricalHyperparameter(name="Difference", choices=["True", "False"], default_value="True") tsfresh_feature = CategoricalHyperparameter(name="tsfresh_feature", choices=["True", "False"], default_value="True") n_estimators = UniformIntegerHyperparameter(name="n_estimators", lower=50, upper=500, default_value=50, log=False) learning_rate = UniformFloatHyperparameter(name="learning_rate", lower=0.01, upper=2, default_value=0.1, log=True) loss = CategoricalHyperparameter( name="loss", choices=["linear", "square", "exponential"], default_value="linear") max_depth = UniformIntegerHyperparameter(name="max_depth", lower=1, upper=10, default_value=3, log=False) cs.add_hyperparameters([ n_estimators, Difference, learning_rate, loss, max_depth, Window_size, tsfresh_feature ]) return cs
def get_hyperparameter_search_space(dataset_properties=None, optimizer='smac'): if optimizer == 'smac': cs = ConfigurationSpace() n_estimators = UniformIntegerHyperparameter( name="n_estimators", lower=50, upper=500, default_value=50, log=False) sampling_strategy = CategoricalHyperparameter( name="sampling_strategy", choices=["majority", "not minority", "not majority", "all"], default_value="not minority") replacement = CategoricalHyperparameter( "replacement", ["True", "False"], default_value="False") ab_n_estimators = UniformIntegerHyperparameter( name="ab_n_estimators", lower=50, upper=500, default_value=50, log=False) ab_learning_rate = UniformFloatHyperparameter( name="ab_learning_rate", lower=0.01, upper=2, default_value=0.1, log=True) ab_algorithm = CategoricalHyperparameter( name="ab_algorithm", choices=["SAMME.R", "SAMME"], default_value="SAMME.R") ab_max_depth = UniformIntegerHyperparameter( name="ab_max_depth", lower=1, upper=10, default_value=1, log=False) cs.add_hyperparameters([n_estimators, sampling_strategy, replacement, ab_n_estimators, ab_learning_rate, ab_algorithm, ab_max_depth]) return cs elif optimizer == 'tpe': from hyperopt import hp space = {'n_estimators': hp.randint('easy_ensemble_n_estimators', 451) + 50, 'sampling_strategy': hp.choice('easy_ensemble_sampling_strategy', ["majority", "not minority", "not majority", "all"]), 'replacement': hp.choice('easy_ensemble_replacement', ["True", "False"]), 'ab_n_estimators': hp.randint('ab_n_estimators', 451) + 50, 'ab_learning_rate': hp.loguniform('ab_learning_rate', np.log(0.01), np.log(2)), 'ab_algorithm': hp.choice('ab_algorithm', ["SAMME.R", "SAMME"]), 'ab_max_depth': hp.randint('ab_max_depth', 10) + 1} init_trial = {'n_estimators': 10, 'sampling_strategy': "not minority", 'replacement': "False", 'ab_n_estimators': 50, 'ab_learning_rate': 0.1, 'ab_algorithm': "SAMME.R", 'ab_max_depth': 1} return space
def get_hyperparameter_search_space(dataset_properties=None): n_estimators = UniformIntegerHyperparameter(name="n_estimators", lower=10, upper=100, default_value=10) max_depth = UniformIntegerHyperparameter(name="max_depth", lower=2, upper=10, default_value=5) min_samples_split = UniformIntegerHyperparameter(name="min_samples_split", lower=2, upper=20, default_value=2) min_samples_leaf = UniformIntegerHyperparameter(name="min_samples_leaf", lower=1, upper=20, default_value=1) min_weight_fraction_leaf = Constant('min_weight_fraction_leaf', 1.0) max_leaf_nodes = UnParametrizedHyperparameter(name="max_leaf_nodes", value="None") bootstrap = CategoricalHyperparameter('bootstrap', ['True', 'False']) cs = ConfigurationSpace() cs.add_hyperparameters([n_estimators, max_depth, min_samples_split, min_samples_leaf, min_weight_fraction_leaf, max_leaf_nodes, bootstrap]) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() criterion = CategoricalHyperparameter( "criterion", ["gini", "entropy"], default_value="gini") max_depth = UniformFloatHyperparameter( 'max_depth', 0., 2., default_value=0.5) min_samples_split = UniformIntegerHyperparameter( "min_samples_split", 2, 20, default_value=2) min_samples_leaf = UniformIntegerHyperparameter( "min_samples_leaf", 1, 20, default_value=1) min_weight_fraction_leaf = Constant("min_weight_fraction_leaf", 0.0) max_features = UnParametrizedHyperparameter('max_features', 1.0) max_leaf_nodes = UnParametrizedHyperparameter("max_leaf_nodes", "None") min_impurity_decrease = UnParametrizedHyperparameter('min_impurity_decrease', 0.0) cs.add_hyperparameters([criterion, max_features, max_depth, min_samples_split, min_samples_leaf, min_weight_fraction_leaf, max_leaf_nodes, min_impurity_decrease]) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() penalty = CategoricalHyperparameter( "penalty", ["l1", "l2"], default_value="l2") loss = CategoricalHyperparameter( "loss", ["hinge", "squared_hinge"], default_value="squared_hinge") dual = Constant("dual", "False") # This is set ad-hoc tol = UniformFloatHyperparameter( "tol", 1e-5, 1e-1, default_value=1e-4, log=True) C = UniformFloatHyperparameter( "C", 0.03125, 32768, log=True, default_value=1.0) multi_class = Constant("multi_class", "ovr") # These are set ad-hoc fit_intercept = Constant("fit_intercept", "True") intercept_scaling = Constant("intercept_scaling", 1) cs.add_hyperparameters([penalty, loss, dual, tol, C, multi_class, fit_intercept, intercept_scaling]) penalty_and_loss = ForbiddenAndConjunction( ForbiddenEqualsClause(penalty, "l1"), ForbiddenEqualsClause(loss, "hinge") ) constant_penalty_and_loss = ForbiddenAndConjunction( ForbiddenEqualsClause(dual, "False"), ForbiddenEqualsClause(penalty, "l2"), ForbiddenEqualsClause(loss, "hinge") ) penalty_and_dual = ForbiddenAndConjunction( ForbiddenEqualsClause(dual, "False"), ForbiddenEqualsClause(penalty, "l1") ) cs.add_forbidden_clause(penalty_and_loss) cs.add_forbidden_clause(constant_penalty_and_loss) cs.add_forbidden_clause(penalty_and_dual) return cs