def get_hyperparameter_search_space(dataset_properties=None): if dataset_properties is not None and \ (dataset_properties.get("sparse") is True or dataset_properties.get("signed") is False): allow_chi2 = False else: allow_chi2 = True possible_kernels = ['poly', 'rbf', 'sigmoid', 'cosine'] if allow_chi2: possible_kernels.append("chi2") kernel = CategoricalHyperparameter('kernel', possible_kernels, 'rbf') n_components = UniformIntegerHyperparameter( "n_components", 50, 10000, default_value=100, log=True) gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8, log=True, default_value=0.1) degree = UniformIntegerHyperparameter('degree', 2, 5, 3) coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0) cs = ConfigurationSpace() cs.add_hyperparameters([kernel, degree, gamma, coef0, n_components]) degree_depends_on_poly = EqualsCondition(degree, kernel, "poly") coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"]) gamma_kernels = ["poly", "rbf", "sigmoid"] if allow_chi2: gamma_kernels.append("chi2") gamma_condition = InCondition(gamma, kernel, gamma_kernels) cs.add_conditions([degree_depends_on_poly, coef0_condition, gamma_condition]) return cs
def get_hyperparameter_search_space(**kwargs): n_components_factor = UniformFloatHyperparameter("n_components_factor", 0., 1., default_value=1.) kernel = CategoricalHyperparameter( 'kernel', ['poly', 'rbf', 'sigmoid', 'cosine'], 'rbf') gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8, log=True, default_value=1.0) degree = UniformIntegerHyperparameter('degree', 2, 5, 3) coef0 = UniformFloatHyperparameter("coef0", -1., 1., default_value=0.) cs = ConfigurationSpace() cs.add_hyperparameters( [n_components_factor, kernel, degree, gamma, coef0]) degree_depends_on_poly = EqualsCondition(degree, kernel, "poly") coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"]) gamma_condition = InCondition(gamma, kernel, ["poly", "rbf"]) cs.add_conditions( [degree_depends_on_poly, coef0_condition, gamma_condition]) return cs
def get_hyperparameter_search_space(dataset_properties=None, optimizer='smac'): if dataset_properties is not None and \ (dataset_properties.get("sparse") is True or dataset_properties.get("signed") is False): allow_chi2 = False else: allow_chi2 = True possible_kernels = ['poly', 'rbf', 'sigmoid', 'cosine'] if allow_chi2: possible_kernels.append("chi2") kernel = CategoricalHyperparameter('kernel', possible_kernels, 'rbf') n_components = UniformIntegerHyperparameter( "n_components", 50, 5000, default_value=100, log=True) gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8, log=True, default_value=0.1) degree = UniformIntegerHyperparameter('degree', 2, 5, 3) coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0) cs = ConfigurationSpace() cs.add_hyperparameters([kernel, degree, gamma, coef0, n_components]) degree_depends_on_poly = EqualsCondition(degree, kernel, "poly") coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"]) gamma_kernels = ["poly", "rbf", "sigmoid"] if allow_chi2: gamma_kernels.append("chi2") gamma_condition = InCondition(gamma, kernel, gamma_kernels) cs.add_conditions([degree_depends_on_poly, coef0_condition, gamma_condition]) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() alpha = UniformFloatHyperparameter(name='alpha', lower=10**-5, upper=1, log=True, default_value=1.0) kernel = CategoricalHyperparameter( name='kernel', # We restrict ourselves to two possible kernels for this example choices=['polynomial', 'rbf'], default_value='polynomial') gamma = UniformFloatHyperparameter(name='gamma', lower=0.00001, upper=1, default_value=0.1, log=True) degree = UniformIntegerHyperparameter(name='degree', lower=2, upper=5, default_value=3) coef0 = UniformFloatHyperparameter( name='coef0', lower=1e-2, upper=1e2, log=True, default_value=1, ) cs.add_hyperparameters([alpha, kernel, gamma, degree, coef0]) degree_condition = EqualsCondition(degree, kernel, 'polynomial') coef0_condition = EqualsCondition(coef0, kernel, 'polynomial') cs.add_conditions([degree_condition, coef0_condition]) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() loss = Constant("loss", "auto") learning_rate = UniformFloatHyperparameter(name="learning_rate", lower=0.01, upper=1, default_value=0.1, log=True) max_iter = UniformIntegerHyperparameter("max_iter", 32, 512, default_value=100) min_samples_leaf = UniformIntegerHyperparameter( name="min_samples_leaf", lower=1, upper=200, default_value=20, log=True) max_depth = UnParametrizedHyperparameter(name="max_depth", value="None") max_leaf_nodes = UniformIntegerHyperparameter(name="max_leaf_nodes", lower=3, upper=2047, default_value=31, log=True) max_bins = Constant("max_bins", 256) l2_regularization = UniformFloatHyperparameter( name="l2_regularization", lower=1E-10, upper=1, default_value=1E-10, log=True) early_stop = CategoricalHyperparameter( name="early_stop", choices=["off", "train", "valid"], default_value="off") tol = UnParametrizedHyperparameter(name="tol", value=1e-7) scoring = UnParametrizedHyperparameter(name="scoring", value="loss") n_iter_no_change = UniformIntegerHyperparameter( name="n_iter_no_change", lower=1, upper=20, default_value=10) validation_fraction = UniformFloatHyperparameter( name="validation_fraction", lower=0.01, upper=0.4, default_value=0.1) cs.add_hyperparameters([ loss, learning_rate, max_iter, min_samples_leaf, max_depth, max_leaf_nodes, max_bins, l2_regularization, early_stop, tol, scoring, n_iter_no_change, validation_fraction ]) n_iter_no_change_cond = InCondition(n_iter_no_change, early_stop, ["valid", "train"]) validation_fraction_cond = EqualsCondition(validation_fraction, early_stop, "valid") cs.add_conditions([n_iter_no_change_cond, validation_fraction_cond]) return cs
def get_hyperparameter_search_space(dataset_properties=None): n_components = UniformIntegerHyperparameter("n_components", 10, 2000, default_value=100) kernel = CategoricalHyperparameter( 'kernel', ['poly', 'rbf', 'sigmoid', 'cosine'], 'rbf') gamma = UniformFloatHyperparameter( "gamma", 3.0517578125e-05, 8, log=True, default_value=0.01, ) degree = UniformIntegerHyperparameter('degree', 2, 5, 3) coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0) cs = ConfigurationSpace() cs.add_hyperparameters([n_components, kernel, degree, gamma, coef0]) degree_depends_on_poly = EqualsCondition(degree, kernel, "poly") coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"]) gamma_condition = InCondition(gamma, kernel, ["poly", "rbf"]) cs.add_conditions( [degree_depends_on_poly, coef0_condition, gamma_condition]) return cs
def load_configspace(self, folder): """Will try to load the configspace. If it's a pcs-file, backup_cs will be a list containing all possible combinations of interpretation for Categoricals. If this issue will be fixed, we can drop this procedure.""" cs_fn_json = os.path.join(folder, 'configspace.json') cs_fn_pcs = os.path.join(folder, 'configspace.pcs') if os.path.exists(cs_fn_json): with open(cs_fn_json, 'r') as fh: cs = pcs_json.read(fh.read()) backup_cs = [] self.logger.debug( "Detected and loaded \"%s\". No backup-cs necessary", cs_fn_json) elif os.path.exists(cs_fn_pcs): with open(cs_fn_pcs, 'r') as fh: cs = pcs_new.read(fh.readlines()) # Create alternative interpretations categoricals = [ hp for hp in cs.get_hyperparameters() if isinstance(hp, CategoricalHyperparameter) ] non_categoricals = [ hp for hp in cs.get_hyperparameters() if not isinstance(hp, CategoricalHyperparameter) ] def _get_interpretations(choices): result = [] if set(choices) == {"True", "False"}: result.append([True, False]) if all([c.isdigit() for c in choices]): result.append([int(c) for c in choices]) result.append(choices) return result choices_per_cat = [ _get_interpretations(hp.choices) for hp in categoricals ] combinations = itertools.product(*choices_per_cat) self.logger.debug(combinations) backup_cs = [] for combi in combinations: bcs = ConfigurationSpace() for hp in non_categoricals: bcs.add_hyperparameter(hp) for name, choices in zip([hp.name for hp in categoricals], combi): bcs.add_hyperparameter( CategoricalHyperparameter(name, choices)) bcs.add_conditions(cs.get_conditions()) backup_cs.append(bcs) self.logger.debug("Sampled %d interpretations of \"%s\"", len(backup_cs), cs_fn_pcs) self.logger.debug(choices_per_cat) else: raise ValueError("Missing pcs-file at '%s.[pcs|json]'!" % os.path.join(folder, 'configspace')) return cs, backup_cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() loss = CategoricalHyperparameter( "loss", ["hinge", "log", "modified_huber", "squared_hinge", "perceptron"], default="log") penalty = CategoricalHyperparameter("penalty", ["l1", "l2", "elasticnet"], default="l2") alpha = UniformFloatHyperparameter("alpha", 10e-7, 1e-1, log=True, default=0.0001) l1_ratio = UniformFloatHyperparameter("l1_ratio", 1e-9, 1, log=True, default=0.15) fit_intercept = UnParametrizedHyperparameter("fit_intercept", "True") n_iter = UniformIntegerHyperparameter("n_iter", 5, 1000, log=True, default=20) epsilon = UniformFloatHyperparameter("epsilon", 1e-5, 1e-1, default=1e-4, log=True) learning_rate = CategoricalHyperparameter( "learning_rate", ["optimal", "invscaling", "constant"], default="optimal") eta0 = UniformFloatHyperparameter("eta0", 10**-7, 0.1, default=0.01) power_t = UniformFloatHyperparameter("power_t", 1e-5, 1, default=0.25) average = CategoricalHyperparameter("average", ["False", "True"], default="False") cs.add_hyperparameters([ loss, penalty, alpha, l1_ratio, fit_intercept, n_iter, epsilon, learning_rate, eta0, power_t, average ]) # TODO add passive/aggressive here, although not properly documented? elasticnet = EqualsCondition(l1_ratio, penalty, "elasticnet") epsilon_condition = EqualsCondition(epsilon, loss, "modified_huber") # eta0 seems to be always active according to the source code; when # learning_rate is set to optimial, eta0 is the starting value: # https://github.com/scikit-learn/scikit-learn/blob/0.15.X/sklearn/linear_model/sgd_fast.pyx #eta0_and_inv = EqualsCondition(eta0, learning_rate, "invscaling") #eta0_and_constant = EqualsCondition(eta0, learning_rate, "constant") #eta0_condition = OrConjunction(eta0_and_inv, eta0_and_constant) power_t_condition = EqualsCondition(power_t, learning_rate, "invscaling") cs.add_conditions([elasticnet, epsilon_condition, power_t_condition]) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() hidden_layer_depth = UniformIntegerHyperparameter(name="hidden_layer_depth", lower=1, upper=3, default_value=1) num_nodes_per_layer = UniformIntegerHyperparameter(name="num_nodes_per_layer", lower=16, upper=264, default_value=32, log=True) activation = CategoricalHyperparameter(name="activation", choices=['tanh', 'relu'], default_value='tanh') alpha = UniformFloatHyperparameter(name="alpha", lower=1e-7, upper=1e-1, default_value=1e-4, log=True) learning_rate_init = UniformFloatHyperparameter(name="learning_rate_init", lower=1e-4, upper=0.5, default_value=1e-3, log=True) # Not allowing to turn off early stopping early_stopping = CategoricalHyperparameter(name="early_stopping", choices=["valid", "train"], # , "off"], default_value="valid") # Constants n_iter_no_change = Constant(name="n_iter_no_change", value=32) # default=10 is too low validation_fraction = Constant(name="validation_fraction", value=0.1) tol = UnParametrizedHyperparameter(name="tol", value=1e-4) solver = Constant(name="solver", value='adam') # Relying on sklearn defaults for now batch_size = UnParametrizedHyperparameter(name="batch_size", value="auto") shuffle = UnParametrizedHyperparameter(name="shuffle", value="True") beta_1 = UnParametrizedHyperparameter(name="beta_1", value=0.9) beta_2 = UnParametrizedHyperparameter(name="beta_2", value=0.999) epsilon = UnParametrizedHyperparameter(name="epsilon", value=1e-8) # Not used # solver=["sgd", "lbfgs"] --> not used to keep searchspace simpler # learning_rate --> only used when using solver=sgd # power_t --> only used when using solver=sgd & learning_rate=invscaling # momentum --> only used when solver=sgd # nesterovs_momentum --> only used when solver=sgd # max_fun --> only used when solver=lbfgs # activation=["identity", "logistic"] --> not useful for classification cs.add_hyperparameters([hidden_layer_depth, num_nodes_per_layer, activation, alpha, learning_rate_init, early_stopping, n_iter_no_change, validation_fraction, tol, solver, batch_size, shuffle, beta_1, beta_2, epsilon]) validation_fraction_cond = InCondition(validation_fraction, early_stopping, ["valid"]) cs.add_conditions([validation_fraction_cond]) # We always use early stopping # n_iter_no_change_cond = InCondition(n_iter_no_change, early_stopping, ["valid", "train"]) # tol_cond = InCondition(n_iter_no_change, early_stopping, ["valid", "train"]) # cs.add_conditions([n_iter_no_change_cond, tol_cond]) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() loss = CategoricalHyperparameter( "loss", ["squared_loss", "huber", "epsilon_insensitive", "squared_epsilon_insensitive"], default_value="squared_loss", ) penalty = CategoricalHyperparameter( "penalty", ["l1", "l2", "elasticnet"], default_value="l2") alpha = UniformFloatHyperparameter( "alpha", 1e-7, 1e-1, log=True, default_value=0.0001) l1_ratio = UniformFloatHyperparameter( "l1_ratio", 1e-9, 1., log=True, default_value=0.15) fit_intercept = UnParametrizedHyperparameter( "fit_intercept", "True") tol = UniformFloatHyperparameter( "tol", 1e-5, 1e-1, default_value=1e-4, log=True) epsilon = UniformFloatHyperparameter( "epsilon", 1e-5, 1e-1, default_value=0.1, log=True) learning_rate = CategoricalHyperparameter( "learning_rate", ["optimal", "invscaling", "constant"], default_value="invscaling") eta0 = UniformFloatHyperparameter( "eta0", 1e-7, 1e-1, default_value=0.01, log=True) power_t = UniformFloatHyperparameter( "power_t", 1e-5, 1, default_value=0.25) average = CategoricalHyperparameter( "average", ["False", "True"], default_value="False") cs.add_hyperparameters([loss, penalty, alpha, l1_ratio, fit_intercept, tol, epsilon, learning_rate, eta0, power_t, average]) # TODO add passive/aggressive here, although not properly documented? elasticnet = EqualsCondition(l1_ratio, penalty, "elasticnet") epsilon_condition = InCondition( epsilon, loss, ["huber", "epsilon_insensitive", "squared_epsilon_insensitive"], ) # eta0 is only relevant if learning_rate!='optimal' according to code # https://github.com/scikit-learn/scikit-learn/blob/0.19.X/sklearn/ # linear_model/sgd_fast.pyx#L603 eta0_in_inv_con = InCondition(eta0, learning_rate, ["invscaling", "constant"]) power_t_condition = EqualsCondition(power_t, learning_rate, "invscaling") cs.add_conditions([elasticnet, epsilon_condition, power_t_condition, eta0_in_inv_con]) return cs
def get_hyperparameter_search_space(dataset_properties=None, optimizer='tpe'): method = CategoricalHyperparameter("method", ['average', 'weighted'], default_value='weighted') alpha = UniformFloatHyperparameter("alpha", 1e-5, 1e-3, log=True, default_value=1e-4) cs = ConfigurationSpace() cs.add_hyperparameters([method, alpha]) alpha_cond = EqualsCondition(alpha, method, 'weighted') cs.add_conditions([alpha_cond]) return cs
def get_aug_hyperparameter_space(): cs = ConfigurationSpace() aug = CategoricalHyperparameter('aug', choices=['True', 'False'], default_value='True') auto_aug = CategoricalHyperparameter('auto_aug', choices=['True', 'False'], default_value='False') random_flip = CategoricalHyperparameter('random_flip', choices=['True', 'False'], default_value='True') affine = CategoricalHyperparameter('affine', choices=['True', 'False'], default_value='True') jitter = CategoricalHyperparameter('jitter', choices=['True', 'False'], default_value='True') brightness = CategoricalHyperparameter('brightness', choices=[0.2], default_value=0.2) saturation = CategoricalHyperparameter('saturation', choices=[0.2], default_value=0.2) hue = CategoricalHyperparameter('hue', choices=[0.15], default_value=0.15) degree = CategoricalHyperparameter('degree', choices=[10, 20, 30], default_value=10) shear = CategoricalHyperparameter('shear', choices=[0.05, 0.1, 0.2], default_value=0.1) cs.add_hyperparameters([ aug, random_flip, auto_aug, affine, jitter, brightness, saturation, hue, degree, shear ]) auto_aug_on_aug = EqualsCondition(auto_aug, aug, 'True') random_flip_on_auto_aug = EqualsCondition(random_flip, auto_aug, 'False') affine_on_auto_aug = EqualsCondition(affine, auto_aug, 'False') jitter_on_auto_aug = EqualsCondition(jitter, auto_aug, 'False') brightness_on_jitter = EqualsCondition(brightness, jitter, 'True') saturation_on_jitter = EqualsCondition(saturation, jitter, 'True') hue_on_jitter = EqualsCondition(hue, jitter, 'True') degree_on_affine = EqualsCondition(degree, affine, 'True') shear_on_affine = EqualsCondition(shear, affine, 'True') cs.add_conditions([ auto_aug_on_aug, random_flip_on_auto_aug, affine_on_auto_aug, jitter_on_auto_aug, brightness_on_jitter, saturation_on_jitter, hue_on_jitter, degree_on_affine, shear_on_affine ]) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() loss = CategoricalHyperparameter("loss", ["squared_loss", "huber", "epsilon_insensitive", "squared_epsilon_insensitive"], default_value="squared_loss") penalty = CategoricalHyperparameter( "penalty", ["l1", "l2", "elasticnet"], default_value="l2") alpha = UniformFloatHyperparameter( "alpha", 1e-7, 1e-1, log=True, default_value=0.0001) l1_ratio = UniformFloatHyperparameter( "l1_ratio", 1e-9, 1., log=True, default_value=0.15) fit_intercept = UnParametrizedHyperparameter( "fit_intercept", "True") tol = UniformFloatHyperparameter( "tol", 1e-5, 1e-1, default_value=1e-4, log=True) epsilon = UniformFloatHyperparameter( "epsilon", 1e-5, 1e-1, default_value=0.1, log=True) learning_rate = CategoricalHyperparameter( "learning_rate", ["optimal", "invscaling", "constant"], default_value="invscaling") eta0 = UniformFloatHyperparameter( "eta0", 1e-7, 1e-1, default_value=0.01, log=True) power_t = UniformFloatHyperparameter( "power_t", 1e-5, 1, default_value=0.25) average = CategoricalHyperparameter( "average", ["False", "True"], default_value="False") cs.add_hyperparameters([loss, penalty, alpha, l1_ratio, fit_intercept, tol, epsilon, learning_rate, eta0, power_t, average]) # TODO add passive/aggressive here, although not properly documented? elasticnet = EqualsCondition(l1_ratio, penalty, "elasticnet") epsilon_condition = InCondition(epsilon, loss, ["huber", "epsilon_insensitive", "squared_epsilon_insensitive"]) # eta0 is only relevant if learning_rate!='optimal' according to code # https://github.com/scikit-learn/scikit-learn/blob/0.19.X/sklearn/ # linear_model/sgd_fast.pyx#L603 eta0_in_inv_con = InCondition(eta0, learning_rate, ["invscaling", "constant"]) power_t_condition = EqualsCondition(power_t, learning_rate, "invscaling") cs.add_conditions([elasticnet, epsilon_condition, power_t_condition, eta0_in_inv_con]) return cs
def get_hyperparameter_search_space(dataset_properties=None): n_components = UniformIntegerHyperparameter( "n_components", 10, 2000, default_value=100) kernel = CategoricalHyperparameter('kernel', ['poly', 'rbf', 'sigmoid', 'cosine'], 'rbf') gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8, log=True, default_value=1.0) degree = UniformIntegerHyperparameter('degree', 2, 5, 3) coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0) cs = ConfigurationSpace() cs.add_hyperparameters([n_components, kernel, degree, gamma, coef0]) degree_depends_on_poly = EqualsCondition(degree, kernel, "poly") coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"]) gamma_condition = InCondition(gamma, kernel, ["poly", "rbf"]) cs.add_conditions([degree_depends_on_poly, coef0_condition, gamma_condition]) return cs
def get_hyperparameter_search_space(dataset_properties=None): # Copied from libsvm_c C = UniformFloatHyperparameter( name="C", lower=0.03125, upper=32768, log=True, default=1.0) kernel = CategoricalHyperparameter( name="kernel", choices=['linear', 'poly', 'rbf', 'sigmoid'], default="rbf") degree = UniformIntegerHyperparameter( name="degree", lower=1, upper=5, default=3) # Changed the gamma value to 0.0 (is 0.1 for classification) gamma = UniformFloatHyperparameter( name="gamma", lower=3.0517578125e-05, upper=8, log=True, default=0.1) # TODO this is totally ad-hoc coef0 = UniformFloatHyperparameter( name="coef0", lower=-1, upper=1, default=0) # probability is no hyperparameter, but an argument to the SVM algo shrinking = CategoricalHyperparameter( name="shrinking", choices=["True", "False"], default="True") tol = UniformFloatHyperparameter( name="tol", lower=1e-5, upper=1e-1, default=1e-3, log=True) max_iter = UnParametrizedHyperparameter("max_iter", -1) # Random Guess epsilon = UniformFloatHyperparameter(name="epsilon", lower=0.001, upper=1, default=0.1, log=True) cs = ConfigurationSpace() cs.add_hyperparameters([C, kernel, degree, gamma, coef0, shrinking, tol, max_iter, epsilon]) degree_depends_on_kernel = InCondition(child=degree, parent=kernel, values=('poly', 'rbf', 'sigmoid')) gamma_depends_on_kernel = InCondition(child=gamma, parent=kernel, values=('poly', 'rbf')) coef0_depends_on_kernel = InCondition(child=coef0, parent=kernel, values=('poly', 'sigmoid')) cs.add_conditions([degree_depends_on_kernel, gamma_depends_on_kernel, coef0_depends_on_kernel]) return cs
def get_hyperparameter_search_space(dataset_properties=None): C = UniformFloatHyperparameter( name="C", lower=0.03125, upper=32768, log=True, default_value=1.0) # Random Guess epsilon = UniformFloatHyperparameter(name="epsilon", lower=0.001, upper=1, default_value=0.1, log=True) kernel = CategoricalHyperparameter( name="kernel", choices=['linear', 'poly', 'rbf', 'sigmoid'], default_value="rbf") degree = UniformIntegerHyperparameter( name="degree", lower=2, upper=5, default_value=3) gamma = UniformFloatHyperparameter( name="gamma", lower=3.0517578125e-05, upper=8, log=True, default_value=0.1) # TODO this is totally ad-hoc coef0 = UniformFloatHyperparameter( name="coef0", lower=-1, upper=1, default_value=0) # probability is no hyperparameter, but an argument to the SVM algo shrinking = CategoricalHyperparameter( name="shrinking", choices=["True", "False"], default_value="True") tol = UniformFloatHyperparameter( name="tol", lower=1e-5, upper=1e-1, default_value=1e-3, log=True) max_iter = UnParametrizedHyperparameter("max_iter", -1) cs = ConfigurationSpace() cs.add_hyperparameters([C, kernel, degree, gamma, coef0, shrinking, tol, max_iter, epsilon]) degree_depends_on_kernel = InCondition(child=degree, parent=kernel, values=('poly', 'rbf', 'sigmoid')) gamma_depends_on_kernel = InCondition(child=gamma, parent=kernel, values=('poly', 'rbf')) coef0_depends_on_kernel = InCondition(child=coef0, parent=kernel, values=('poly', 'sigmoid')) cs.add_conditions([degree_depends_on_kernel, gamma_depends_on_kernel, coef0_depends_on_kernel]) return cs
def load_configspace(self, folder): """Will try to load the configspace. cs_options will be a list containing all possible combinations of interpretation for Categoricals. If this issue will be fixed, we can drop this procedure. Parameters ---------- folder: str path to folder in which to look for configspace Returns ------- cs_options: list[ConfigurationSpace] list with possible interpretations for config-space-file. Only contains multiple items if file-format is pcs. """ cs_options = [] cs_fn_json = os.path.join(folder, 'configspace.json') cs_fn_pcs = os.path.join(folder, 'configspace.pcs') if os.path.exists(cs_fn_json): with open(cs_fn_json, 'r') as fh: cs_options = [pcs_json.read(fh.read())] self.logger.debug( "Detected and loaded \"%s\". No alternative interpretations necessary", cs_fn_json) elif os.path.exists(cs_fn_pcs): with open(cs_fn_pcs, 'r') as fh: cs = pcs_new.read(fh.readlines()) # Create alternative interpretations categoricals = [ hp for hp in cs.get_hyperparameters() if isinstance(hp, CategoricalHyperparameter) ] non_categoricals = [ hp for hp in cs.get_hyperparameters() if not isinstance(hp, CategoricalHyperparameter) ] def _get_interpretations(choices): """ Generate different interpretations for critical categorical hyperparameters that are not seamlessly supported by pcs-format.""" result = [] if set(choices) == {"True", "False"}: result.append([True, False]) if all([c.isdigit() for c in choices]): result.append([int(c) for c in choices]) result.append(choices) return result choices_per_cat = [ _get_interpretations(hp.choices) for hp in categoricals ] combinations = itertools.product(*choices_per_cat) self.logger.debug(combinations) for combi in combinations: bcs = ConfigurationSpace() for hp in non_categoricals: bcs.add_hyperparameter(hp) for name, choices in zip([hp.name for hp in categoricals], combi): bcs.add_hyperparameter( CategoricalHyperparameter(name, choices)) bcs.add_conditions(cs.get_conditions()) cs_options.append(bcs) self.logger.debug("Sampled %d interpretations of \"%s\"", len(cs_options), cs_fn_pcs) else: raise ValueError("Missing pcs-file at '%s.[pcs|json]'!" % os.path.join(folder, 'configspace')) return cs_options
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() Window_size = UniformIntegerHyperparameter(name="Window_size", lower=5, upper=50, default_value=20) tsfresh_feature = CategoricalHyperparameter(name="tsfresh_feature", choices=["True", "False"], default_value="True") Difference = CategoricalHyperparameter(name="Difference", choices=["True", "False"], default_value="True") loss = CategoricalHyperparameter("loss", [ "squared_loss", "huber", "epsilon_insensitive", "squared_epsilon_insensitive" ], default_value="squared_loss") penalty = CategoricalHyperparameter("penalty", ["l1", "l2", "elasticnet"], default_value="l2") alpha = UniformFloatHyperparameter("alpha", 1e-7, 1e-1, log=True, default_value=0.0001) l1_ratio = UniformFloatHyperparameter("l1_ratio", 1e-9, 1., log=True, default_value=0.15) tol = UniformFloatHyperparameter("tol", 1e-4, 1e-1, default_value=1e-3, log=True) epsilon = UniformFloatHyperparameter("epsilon", 1e-5, 1e-1, default_value=0.1, log=True) learning_rate = CategoricalHyperparameter( "learning_rate", ["optimal", "invscaling", "constant"], default_value="invscaling") eta0 = UniformFloatHyperparameter("eta0", 1e-7, 1e-1, default_value=0.01) power_t = UniformFloatHyperparameter("power_t", 1e-5, 1, default_value=0.25) average = CategoricalHyperparameter("average", ["False", "True"], default_value="False") # un parametrized parameter fit_intercept = UnParametrizedHyperparameter("fit_intercept", "True") max_iter = UnParametrizedHyperparameter("max_iter", 1000) cs.add_hyperparameters([ loss, penalty, alpha, l1_ratio, fit_intercept, tol, epsilon, learning_rate, eta0, Difference, power_t, average, Window_size, tsfresh_feature, max_iter ]) elasticnet = EqualsCondition(l1_ratio, penalty, "elasticnet") epsilon_condition = InCondition( epsilon, loss, ["huber", "epsilon_insensitive", "squared_epsilon_insensitive"]) power_t_condition = EqualsCondition(power_t, learning_rate, "invscaling") cs.add_conditions([elasticnet, epsilon_condition, power_t_condition]) return cs
default_value=0) # probability is no hyperparameter, but an argument to the SVM algo shrinking = CategoricalHyperparameter(name="shrinking", choices=["True", "False"], default_value="True") tol = UniformFloatHyperparameter(name="tol", lower=1e-5, upper=1e-1, default_value=1e-3, log=True) max_iter = Constant("max_iter", -1) cs = ConfigurationSpace() cs.add_hyperparameters( [C, kernel, degree, gamma, coef0, shrinking, tol, max_iter, epsilon]) degree_depends_on_kernel = InCondition(child=degree, parent=kernel, values=('poly', 'rbf', 'sigmoid')) gamma_depends_on_kernel = InCondition(child=gamma, parent=kernel, values=('poly', 'rbf')) coef0_depends_on_kernel = InCondition(child=coef0, parent=kernel, values=('poly', 'sigmoid')) cs.add_conditions([ degree_depends_on_kernel, gamma_depends_on_kernel, coef0_depends_on_kernel ]) json_utils.write_cs_to_json_file(cs, "SVR")
"learning_rate", ["optimal", "invscaling", "constant"], default_value="invscaling") eta0 = UniformFloatHyperparameter("eta0", 1e-7, 1e-1, default_value=0.01, log=True) power_t = UniformFloatHyperparameter("power_t", 1e-5, 1, default_value=0.5) average = CategoricalHyperparameter("average", ["False", "True"], default_value="False") cs.add_hyperparameters([ loss, penalty, alpha, l1_ratio, fit_intercept, tol, epsilon, learning_rate, eta0, power_t, average ]) elasticnet = EqualsCondition(l1_ratio, penalty, "elasticnet") epsilon_condition = InCondition( epsilon, loss, ["huber", "epsilon_insensitive", "squared_epsilon_insensitive"]) # eta0 is only relevant if learning_rate!='optimal' according to code # https://github.com/scikit-learn/scikit-learn/blob/0.19.X/sklearn/ # linear_model/sgd_fast.pyx#L603 eta0_in_inv_con = InCondition(eta0, learning_rate, ["invscaling", "constant"]) power_t_condition = EqualsCondition(power_t, learning_rate, "invscaling") cs.add_conditions( [elasticnet, epsilon_condition, power_t_condition, eta0_in_inv_con]) json_utils.write_cs_to_json_file(cs, "SGDRegressor")
from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import UniformFloatHyperparameter, \ UniformIntegerHyperparameter, CategoricalHyperparameter from ConfigSpace.conditions import InCondition, EqualsCondition from automl.utl import json_utils cs = ConfigurationSpace() kernel = CategoricalHyperparameter('kernel', ['poly', 'rbf', 'sigmoid', 'cosine'], 'rbf') n_components = UniformIntegerHyperparameter( "n_components", 50, 10000, default_value=100, log=True) gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8, log=True, default_value=0.1) degree = UniformIntegerHyperparameter('degree', 2, 5, 3) coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0) cs.add_hyperparameters([kernel, degree, gamma, coef0, n_components]) degree_depends_on_poly = EqualsCondition(degree, kernel, "poly") coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"]) gamma_kernels = ["poly", "rbf", "sigmoid"] gamma_condition = InCondition(gamma, kernel, gamma_kernels) cs.add_conditions([degree_depends_on_poly, coef0_condition, gamma_condition]) json_utils.write_cs_to_json_file(cs, "Nystroem")
def get_hyperparameter_search_space(**kwargs): cs = ConfigurationSpace() loss = CategoricalHyperparameter( "loss", ["log", "modified_huber", "squared_hinge", "perceptron"], default_value="log") penalty = CategoricalHyperparameter("penalty", ["l2", "l1", "elasticnet"], default_value="l2") alpha = UniformFloatHyperparameter("alpha", 1e-7, 1e-1, default_value=0.0001, log=True) l1_ratio = UniformFloatHyperparameter("l1_ratio", 1e-9, 1., default_value=0.15, log=True) fit_intercept = CategoricalHyperparameter("fit_intercept", [True, False], default_value=True) tol = UniformFloatHyperparameter("tol", 1e-5, 1e-1, default_value=1e-4, log=True) epsilon = UniformFloatHyperparameter("epsilon", 1e-5, 1e-1, default_value=1e-4, log=True) learning_rate = CategoricalHyperparameter( "learning_rate", ["constant", "optimal", "invscaling"], default_value="invscaling") eta0 = UniformFloatHyperparameter("eta0", 1e-7, 1e-1, default_value=0.01, log=True) power_t = UniformFloatHyperparameter("power_t", 1e-5, 1, default_value=0.5) average = CategoricalHyperparameter("average", [True, False], default_value=False) cs.add_hyperparameters([ loss, penalty, alpha, l1_ratio, fit_intercept, tol, epsilon, learning_rate, eta0, power_t, average ]) elasticnet = EqualsCondition(l1_ratio, penalty, "elasticnet") epsilon_condition = EqualsCondition(epsilon, loss, "modified_huber") power_t_condition = EqualsCondition(power_t, learning_rate, "invscaling") # eta0 is only relevant if learning_rate!='optimal' according to code # https://github.com/scikit-learn/scikit-learn/blob/0.19.X/sklearn/ # linear_model/sgd_fast.pyx#L603 eta0_in_inv_con = InCondition(eta0, learning_rate, ["invscaling", "constant"]) cs.add_conditions([ elasticnet, epsilon_condition, power_t_condition, eta0_in_inv_con ]) return cs
def get_hyperparameter_search_space(dataset_properties=None, optimizer='smac'): if optimizer == 'smac': cs = ConfigurationSpace() loss = CategoricalHyperparameter("loss", ["hinge", "log", "modified_huber", "squared_hinge", "perceptron"], default_value="log") penalty = CategoricalHyperparameter( "penalty", ["l1", "l2", "elasticnet"], default_value="l2") alpha = UniformFloatHyperparameter( "alpha", 1e-7, 1e-1, log=True, default_value=0.0001) l1_ratio = UniformFloatHyperparameter( "l1_ratio", 1e-9, 1, log=True, default_value=0.15) fit_intercept = UnParametrizedHyperparameter("fit_intercept", "True") tol = UniformFloatHyperparameter("tol", 1e-5, 1e-1, log=True, default_value=1e-4) epsilon = UniformFloatHyperparameter( "epsilon", 1e-5, 1e-1, default_value=1e-4, log=True) learning_rate = CategoricalHyperparameter( "learning_rate", ["optimal", "invscaling", "constant"], default_value="invscaling") eta0 = UniformFloatHyperparameter( "eta0", 1e-7, 1e-1, default_value=0.01, log=True) power_t = UniformFloatHyperparameter("power_t", 1e-5, 1, log=True, default_value=0.5) average = CategoricalHyperparameter( "average", ["False", "True"], default_value="False") cs.add_hyperparameters([loss, penalty, alpha, l1_ratio, fit_intercept, tol, epsilon, learning_rate, eta0, power_t, average]) # TODO add passive/aggressive here, although not properly documented? elasticnet = EqualsCondition(l1_ratio, penalty, "elasticnet") epsilon_condition = EqualsCondition(epsilon, loss, "modified_huber") power_t_condition = EqualsCondition(power_t, learning_rate, "invscaling") # eta0 is only relevant if learning_rate!='optimal' according to code # https://github.com/scikit-learn/scikit-learn/blob/0.19.X/sklearn/ # linear_model/sgd_fast.pyx#L603 eta0_in_inv_con = InCondition(eta0, learning_rate, ["invscaling", "constant"]) cs.add_conditions([elasticnet, epsilon_condition, power_t_condition, eta0_in_inv_con]) return cs elif optimizer == 'tpe': eta0 = hp.loguniform('sgd_eta0', np.log(1e-7), np.log(1e-1)) space = { 'loss': hp.choice('sgd_loss', [ ("modified_huber", {'epsilon': hp.loguniform('sgd_epsilon', np.log(1e-5), np.log(1e-1))}), ("hinge", {}), ("log", {}), ("squared_hinge", {}), ("perceptron", {})]), 'penalty': hp.choice('sgd_penalty', [("elasticnet", {'l1_ratio': hp.loguniform('sgd_l1_ratio', np.log(1e-9), np.log(1))}), ("l1", None), ("l2", None)]), 'alpha': hp.loguniform('sgd_alpha', np.log(1e-7), np.log(1e-1)), 'fit_intercept': hp.choice('sgd_fit_intercept', ["True"]), 'tol': hp.loguniform('sgd_tol', np.log(1e-5), np.log(1e-1)), 'learning_rate': hp.choice('sgd_learning_rate', [("optimal", {}), ("invscaling", {'power_t': hp.loguniform('sgd_power_t', np.log(1e-5), np.log(1)), 'eta0': eta0}), ("constant", {'eta0': eta0})]), 'average': hp.choice('sgd_average', ["True", "False"])} init_trial = {'loss': ("log", {}), 'penalty': ("l2", {}), 'alpha': 1e-4, 'fit_intercept': "True", 'tol': 1e-4, 'learning_rate': ("invscaling", {'power_t': 0.5, 'eta0': 0.01}), 'average': "False"} return space
batch_size = Constant(name="batch_size", value="auto") shuffle = Constant(name="shuffle", value="True") beta_1 = Constant(name="beta_1", value=0.9) beta_2 = Constant(name="beta_2", value=0.999) epsilon = Constant(name="epsilon", value=1e-8) # Not used # solver=["sgd", "lbfgs"] --> not used to keep searchspace simpler # learning_rate --> only used when using solver=sgd # power_t --> only used when using solver=sgd & learning_rate=invscaling # momentum --> only used when solver=sgd # nesterovs_momentum --> only used when solver=sgd # max_fun --> only used when solver=lbfgs # activation=["identity", "logistic"] --> not useful for classification cs.add_hyperparameters([ hidden_layer_sizes, activation, alpha, learning_rate_init, early_stopping, n_iter_no_change, validation_fraction, tol, solver, batch_size, shuffle, beta_1, beta_2, epsilon ]) validation_fraction_cond = InCondition(validation_fraction, early_stopping, ["True"]) cs.add_conditions([validation_fraction_cond]) # We always use early stopping # n_iter_no_change_cond = InCondition(n_iter_no_change, early_stopping, ["valid", "train"]) # tol_cond = InCondition(n_iter_no_change, early_stopping, ["valid", "train"]) # cs.add_conditions([n_iter_no_change_cond, tol_cond]) cs.add_conditions([validation_fraction_cond])
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() # Parameterized Hyperparameters max_depth = UniformIntegerHyperparameter( name="max_depth", lower=1, upper=20, default_value=3 ) learning_rate = UniformFloatHyperparameter( name="learning_rate", lower=0.001, upper=1, default_value=0.1, log=True, ) n_estimators = Constant("n_estimators", 512) booster = CategoricalHyperparameter( "booster", ["gbtree", "dart"] ) subsample = UniformFloatHyperparameter( name="subsample", lower=0.01, upper=1.0, default_value=1.0, log=False ) min_child_weight = UniformIntegerHyperparameter( name="min_child_weight", lower=0, upper=20, default_value=1, log=False ) colsample_bytree = UniformFloatHyperparameter( name="colsample_bytree", lower=0.1, upper=1.0, default_value=1, ) colsample_bylevel = UniformFloatHyperparameter( name="colsample_bylevel", lower=0.1, upper=1.0, default_value=1, ) reg_alpha = UniformFloatHyperparameter( name="reg_alpha", lower=1e-10, upper=1e-1, log=True, default_value=1e-10) reg_lambda = UniformFloatHyperparameter( name="reg_lambda", lower=1e-10, upper=1e-1, log=True, default_value=1e-10) # DART Hyperparameters sample_type = CategoricalHyperparameter( 'sample_type', ['uniform', 'weighted'], default_value='uniform', ) normalize_type = CategoricalHyperparameter( 'normalize_type', ['tree', 'forest'], default_value='tree', ) rate_drop = UniformFloatHyperparameter( 'rate_drop', 1e-10, 1-(1e-10), default_value=0.5, ) # Unparameterized Hyperparameters # https://xgboost.readthedocs.io/en/latest//parameter.html # minimum loss reduction required to make a further partition on a # leaf node of the tree gamma = UnParametrizedHyperparameter( name="gamma", value=0) # absolute regularization (in contrast to eta), comparable to # gradient clipping in deep learning - according to the internet this # is most important for unbalanced data max_delta_step = UnParametrizedHyperparameter( name="max_delta_step", value=0) base_score = UnParametrizedHyperparameter( name="base_score", value=0.5) scale_pos_weight = UnParametrizedHyperparameter( name="scale_pos_weight", value=1) cs.add_hyperparameters([ # Active max_depth, learning_rate, n_estimators, booster, subsample, colsample_bytree, colsample_bylevel, reg_alpha, reg_lambda, # DART sample_type, normalize_type, rate_drop, # Inactive min_child_weight, max_delta_step, gamma, base_score, scale_pos_weight ]) sample_type_condition = EqualsCondition( sample_type, booster, 'dart', ) normalize_type_condition = EqualsCondition( normalize_type, booster, 'dart', ) rate_drop_condition = EqualsCondition( rate_drop, booster, 'dart', ) cs.add_conditions([ sample_type_condition, normalize_type_condition, rate_drop_condition, ]) return cs
def get_benchmark_configspace(benchmark_id): if benchmark_id == 'fcnet': cs = ConfigurationSpace() learning_rate = UniformFloatHyperparameter("learning_rate", 1e-4, 1e-2, default_value=1e-3, q=2e-4) momentum = UniformFloatHyperparameter("momentum", 0., .5, default_value=0., q=.1) lr_decay = UniformFloatHyperparameter("lr_decay", .7, .99, default_value=9e-1, q=3e-2) n_layer1 = UniformIntegerHyperparameter("n_layer1", 32, 256, default_value=96, q=8) n_layer2 = UniformIntegerHyperparameter("n_layer2", 64, 256, default_value=128, q=8) batch_size = UniformIntegerHyperparameter("batch_size", 32, 128, default_value=64, q=8) dropout1 = UniformFloatHyperparameter("kb_1", .3, .9, default_value=.5, q=.1) dropout2 = UniformFloatHyperparameter("kb_2", .3, .9, default_value=.5, q=.1) kernel_regularizer = UniformFloatHyperparameter("k_reg", 1e-9, 1e-4, default_value=1e-6, q=5e-7, log=True) cs.add_hyperparameters([ learning_rate, momentum, lr_decay, n_layer1, n_layer2, batch_size, dropout1, dropout2, kernel_regularizer ]) elif benchmark_id in ['covtype', 'higgs']: cs = ConfigurationSpace() # n_estimators = UniformFloatHyperparameter("n_estimators", 100, 600, default_value=200, q=10) eta = UniformFloatHyperparameter("eta", 0.01, 0.9, default_value=0.3, q=0.01) min_child_weight = UniformFloatHyperparameter("min_child_weight", 0, 10, default_value=1, q=0.1) max_depth = UniformIntegerHyperparameter("max_depth", 1, 12, default_value=6) subsample = UniformFloatHyperparameter("subsample", 0.1, 1, default_value=1, q=0.1) gamma = UniformFloatHyperparameter("gamma", 0, 10, default_value=0, q=0.1) colsample_bytree = UniformFloatHyperparameter("colsample_bytree", 0.1, 1, default_value=1., q=0.1) alpha = UniformFloatHyperparameter("alpha", 0, 10, default_value=0., q=0.1) _lambda = UniformFloatHyperparameter("lambda", 1, 10, default_value=1, q=0.1) cs.add_hyperparameters([ eta, min_child_weight, max_depth, subsample, gamma, colsample_bytree, alpha, _lambda ]) elif benchmark_id in ['covtype_svm', 'mnist_svm']: C = UniformFloatHyperparameter("C", 1e-3, 1e5, log=True, default_value=1.0) kernel = CategoricalHyperparameter("kernel", choices=["rbf", "poly", "sigmoid"], default_value="rbf") degree = UniformIntegerHyperparameter("degree", 2, 5, default_value=3) gamma = UniformFloatHyperparameter("gamma", 1e-5, 10, log=True, default_value=0.1) coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0) tol = UniformFloatHyperparameter("tol", 1e-5, 1e-1, default_value=1e-3, log=True) # cache size is not a hyperparameter, but an argument to the program! max_iter = UnParametrizedHyperparameter("max_iter", 10000) cs = ConfigurationSpace() cs.add_hyperparameters( [C, kernel, degree, gamma, coef0, tol, max_iter]) degree_depends_on_poly = EqualsCondition(degree, kernel, "poly") coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"]) cs.add_condition(degree_depends_on_poly) cs.add_condition(coef0_condition) elif benchmark_id == 'cifar': cs = ConfigurationSpace() # padding_size = CategoricalHyperparameter('padding_size', [1, 2, 3], default_value=2) # batch_size = CategoricalHyperparameter('train_batch_size', [256]) batch_size = UniformIntegerHyperparameter("train_batch_size", 32, 256, default_value=64, q=8) init_lr = UniformFloatHyperparameter('init_lr', lower=1e-3, upper=0.3, default_value=0.1, log=True) # lr_decay_factor = UniformFloatHyperparameter('lr_decay_factor', lower=0.01, upper=0.2, default_value=0.1, # log=True) lr_decay_factor = UnParametrizedHyperparameter('lr_decay_factor', 0.1) weight_decay = UniformFloatHyperparameter('weight_decay', lower=1e-5, upper=1e-2, default_value=0.0002, log=True) momentum = UniformFloatHyperparameter("momentum", 0.5, .99, default_value=0.9) nesterov = CategoricalHyperparameter('nesterov', ['True', 'False'], default_value='True') cs.add_hyperparameters([ nesterov, batch_size, init_lr, lr_decay_factor, weight_decay, momentum ]) elif benchmark_id == 'convnet': cs = ConfigurationSpace() learning_rate = UniformFloatHyperparameter("learning_rate", 1e-5, 5e-2, default_value=1e-4, q=3e-5, log=True) batch_size = UniformIntegerHyperparameter("batch_size", 16, 128, q=16, default_value=32) momentum = UniformFloatHyperparameter("momentum", 0., .5, default_value=0., q=.1) lr_decay = UniformFloatHyperparameter("lr_decay", .7, .99, default_value=9e-1, q=3e-2) dropout_value = UniformFloatHyperparameter("dropout", .1, .7, default_value=.5, q=.1) cs.add_hyperparameters( [learning_rate, batch_size, momentum, lr_decay, dropout_value]) num_pooling_layer = UniformIntegerHyperparameter("n_pooling_layer", 2, 3, default_value=2) num_conv_layer1 = UniformIntegerHyperparameter("n_conv_layer1", 16, 64, default_value=32, q=2) num_conv_layer2 = UniformIntegerHyperparameter("n_conv_layer2", 32, 96, default_value=64, q=2) num_conv_layer3 = UniformIntegerHyperparameter("n_conv_layer3", 32, 96, default_value=64, q=2) num_fully_layer = UniformIntegerHyperparameter("n_fully_unit", 128, 512, default_value=256, q=64) cs.add_hyperparameters([ num_pooling_layer, num_conv_layer1, num_conv_layer2, num_conv_layer3, num_fully_layer ]) for i in [1, 2, 3]: kernel_init_stddev = UniformFloatHyperparameter( "kernel_init_stddev%d" % i, 1e-3, 5e-2, default_value=1e-2, q=2e-3) kernel_regularizer = UniformFloatHyperparameter( "kernel_regularizer%d" % i, 1e-9, 1e-4, default_value=1e-6, q=5e-7, log=True) cs.add_hyperparameters([kernel_init_stddev, kernel_regularizer]) if i == 3: k_init_cond = InCondition(child=kernel_init_stddev, parent=num_pooling_layer, values=[3]) k_reg_cond = InCondition(child=kernel_regularizer, parent=num_pooling_layer, values=[3]) cs.add_conditions([k_init_cond, k_reg_cond]) return cs elif 'sys' in benchmark_id: from mfes.evaluate_function.sys.combined_evaluator import get_combined_cs from solnml.datasets.utils import load_data tmp_node = load_data('balloon', data_dir='../soln-ml/', task_type=0, datanode_returned=True) cs = get_combined_cs(tmp_node) return cs else: raise ValueError('Invalid benchmark id: %s!' % benchmark_id) return cs
def get_hyperparameter_search_space( dataset_properties: Optional[Dict[str, BaseDatasetPropertiesType]] = None, n_components: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter='n_components', value_range=(0.5, 0.9), default_value=0.5, ), kernel: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter='kernel', value_range=('poly', 'rbf', 'sigmoid', 'cosine'), default_value='rbf', ), gamma: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter='gamma', value_range=(3.0517578125e-05, 8), default_value=0.01, log=True), degree: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter='degree', value_range=(2, 5), default_value=3, log=True), coef0: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter='coef0', value_range=(-1, 1), default_value=0, ) ) -> ConfigurationSpace: cs = ConfigurationSpace() if dataset_properties is not None: n_features = len( dataset_properties['numerical_columns']) if isinstance( dataset_properties['numerical_columns'], List) else 0 if n_features == 1: log = False else: log = n_components.log n_components = HyperparameterSearchSpace( hyperparameter='n_components', value_range=(floor( float(n_components.value_range[0]) * n_features), ceil( float(n_components.value_range[1]) * n_features)), default_value=ceil( float(n_components.default_value) * n_features), log=log) else: n_components = HyperparameterSearchSpace( hyperparameter='n_components', value_range=(10, 2000), default_value=100, log=n_components.log) add_hyperparameter(cs, n_components, UniformIntegerHyperparameter) kernel_hp = get_hyperparameter(kernel, CategoricalHyperparameter) gamma = get_hyperparameter(gamma, UniformFloatHyperparameter) coef0 = get_hyperparameter(coef0, UniformFloatHyperparameter) cs.add_hyperparameters([kernel_hp, gamma, coef0]) if "poly" in kernel_hp.choices: degree = get_hyperparameter(degree, UniformIntegerHyperparameter) cs.add_hyperparameters([degree]) degree_depends_on_poly = EqualsCondition(degree, kernel_hp, "poly") cs.add_conditions([degree_depends_on_poly]) kernels = [] if "sigmoid" in kernel_hp.choices: kernels.append("sigmoid") if "poly" in kernel_hp.choices: kernels.append("poly") coef0_condition = InCondition(coef0, kernel_hp, kernels) kernels = [] if "rbf" in kernel_hp.choices: kernels.append("rbf") if "poly" in kernel_hp.choices: kernels.append("poly") gamma_condition = InCondition(gamma, kernel_hp, kernels) cs.add_conditions([coef0_condition, gamma_condition]) return cs
def get_hyperparameter_search_space(dataset_properties=None, optimizer='smac'): if optimizer == 'smac': cs = ConfigurationSpace() hidden_size = UniformIntegerHyperparameter("hidden_size", 100, 500, default_value=200) activation = CategoricalHyperparameter("activation", ["identity", "logistic", "tanh", "relu"], default_value="relu") solver = CategoricalHyperparameter("solver", ["sgd", "adam"], default_value="adam") alpha = UniformFloatHyperparameter( "alpha", 1e-7, 1., log=True, default_value=0.0001) learning_rate = CategoricalHyperparameter( "learning_rate", ["adaptive", "invscaling", "constant"], default_value="constant") learning_rate_init = UniformFloatHyperparameter( "learning_rate_init", 1e-4, 3e-1, default_value=0.001, log=True) tol = UniformFloatHyperparameter("tol", 1e-5, 1e-1, log=True, default_value=1e-4) momentum = UniformFloatHyperparameter("momentum", 0.6, 1, q=0.05, default_value=0.9) nesterovs_momentum = CategoricalHyperparameter("nesterovs_momentum", [True, False], default_value=True) beta1 = UniformFloatHyperparameter("beta1", 0.6, 1, default_value=0.9) power_t = UniformFloatHyperparameter("power_t", 1e-5, 1, log=True, default_value=0.5) cs.add_hyperparameters( [hidden_size, activation, solver, alpha, learning_rate, learning_rate_init, tol, momentum, nesterovs_momentum, beta1, power_t]) learning_rate_condition = EqualsCondition(learning_rate, solver, "sgd") momentum_condition = EqualsCondition(momentum, solver, "sgd") nesterovs_momentum_condition = EqualsCondition(nesterovs_momentum, solver, "sgd") beta1_condition = EqualsCondition(beta1, solver, "adam") power_t_condition = EqualsCondition(power_t, learning_rate, "invscaling") cs.add_conditions([learning_rate_condition, momentum_condition, nesterovs_momentum_condition, beta1_condition, power_t_condition]) return cs elif optimizer == 'tpe': space = {'hidden_size': hp.randint("mlp_hidden_size", 400) + 100, 'activation': hp.choice('mlp_activation', ["identity", "logistic", "tanh", "relu"]), 'solver': hp.choice('mlp_solver', [("sgd", {'learning_rate': hp.choice('mlp_learning_rate', [("adaptive", {}), ("constant", {}), ("invscaling", { 'power_t': hp.uniform('mlp_power_t', 1e-5, 1)})]), 'momentum': hp.uniform('mlp_momentum', 0.6, 1), 'nesterovs_momentum': hp.choice('mlp_nesterovs_momentum', [True, False])}), ("adam", {'beta1': hp.uniform('mlp_beta1', 0.6, 1)})]), 'alpha': hp.loguniform('mlp_alpha', np.log(1e-7), np.log(1e-1)), 'learning_rate_init': hp.loguniform('mlp_learning_rate_init', np.log(1e-6), np.log(1e-1)), 'tol': hp.loguniform('mlp_tol', np.log(1e-5), np.log(1e-1))} return space
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() Window_size = UniformIntegerHyperparameter( name="Window_size", lower=5, upper=50, default_value=20) Difference = CategoricalHyperparameter( name="Difference", choices=["True", "False"], default_value="True") tsfresh_feature = CategoricalHyperparameter( name="tsfresh_feature", choices=["True", "False"], default_value="True") C = UniformFloatHyperparameter( name="C", lower=0.03125, upper=32768, log=True, default_value=1.0) epsilon = UniformFloatHyperparameter(name="epsilon", lower=0.001, upper=1, default_value=0.1, log=True) kernel = CategoricalHyperparameter( name="kernel", choices=['linear', 'poly', 'rbf', 'sigmoid'], default_value="rbf") degree = UniformIntegerHyperparameter( name="degree", lower=2, upper=5, default_value=3) gamma = CategoricalHyperparameter("gamma", ["auto", "value"], default_value="auto") gamma_value = UniformFloatHyperparameter( name="gamma_value", lower=0.0001, upper=8, default_value=1) # TODO this is totally ad-hoc coef0 = UniformFloatHyperparameter( name="coef0", lower=-1, upper=1, default_value=0) # probability is no hyperparameter, but an argument to the SVM algo shrinking = CategoricalHyperparameter( name="shrinking", choices=["True", "False"], default_value="True") tol = UniformFloatHyperparameter( name="tol", lower=1e-5, upper=1e-1, default_value=1e-3, log=True) max_iter = UnParametrizedHyperparameter("max_iter", 200000) cs.add_hyperparameters([Window_size, Difference, tsfresh_feature,C, kernel, degree, gamma, gamma_value, coef0, shrinking, tol, max_iter, epsilon]) degree_depends_on_kernel = InCondition(child=degree, parent=kernel, values=["poly"]) gamma_depends_on_kernel = InCondition(child=gamma, parent=kernel, values=["rbf", "poly", "sigmoid"]) coef0_depends_on_kernel = InCondition(child=coef0, parent=kernel, values=["poly", "sigmoid"]) gamma_value_depends_on_gamma = InCondition(child=gamma_value, parent=gamma, values=["value"]) cs.add_conditions([degree_depends_on_kernel, gamma_depends_on_kernel, coef0_depends_on_kernel, gamma_value_depends_on_gamma]) return cs