def rf_reg_configs(): def run(dataset, seed, cfg): cfg = {k: cfg[k] for k in cfg} cfg["criterion"] = criterion[cfg["criterion"]] clf = RandomForestRegressor(random_state=seed, **cfg) scores = cross_val_score(clf, dataset.data, dataset.target, cv=5, scoring="r2") cfg = {k: cfg[k] for k in cfg} return 1 - np.mean(scores) criterion = ["mse", "mae"] cs = ConfigurationSpace() cs.add_hyperparameters([ UniformIntegerHyperparameter("n_estimators", 1, 200), UniformIntegerHyperparameter("criterion", 0, 1), UniformIntegerHyperparameter("max_depth", 10, 1000), UniformIntegerHyperparameter("min_samples_leaf", 1, 200), UniformIntegerHyperparameter("min_samples_split", 2, 100), UniformFloatHyperparameter("min_weight_fraction_leaf", 0.0, 0.5), UniformFloatHyperparameter("min_impurity_decrease", 0.0, 0.9), ]) return (cs, run, "Random Forest")
def tree_configs(scale=1.0): def run(dataset, seed, cfg): cfg = {k: cfg[k] for k in cfg} if scale > 0.4: cfg["criterion"] = criterion[cfg["criterion"]] cfg["splitter"] = splitter[cfg["splitter"]] clf = DecisionTreeClassifier(random_state=seed, **cfg) scores = cross_val_score(clf, dataset.data, dataset.target, cv=5) return 1 - np.mean(scores) criterion = ["gini", "entropy", "gini", "entropy", "gini", "entropy"] splitter = ["best", "random", "best", "random", "best", "random"] cs = ConfigurationSpace() cs.add_hyperparameters([ UniformIntegerHyperparameter("max_depth", 10, 1500 * scale), UniformIntegerHyperparameter("min_samples_leaf", 1, 400 * scale), UniformIntegerHyperparameter("min_samples_split", 2, 300 * scale), UniformFloatHyperparameter("min_weight_fraction_leaf", 0.0, 0.5 * scale), UniformFloatHyperparameter("min_impurity_decrease", 0.0, 1.0 * scale), ]) if scale > 0.4: cs.add_hyperparameters([ UniformIntegerHyperparameter("criterion", 0, 5), UniformIntegerHyperparameter("splitter", 0, 5), ]) return (cs, run, "Desicion Tree, %.2lf" % scale)
def mlp_configs(): def run(dataset, seed, cfg): cfg = {k: cfg[k] for k in cfg} cfg["activation"] = activation[cfg["activation"]] cfg["solver"] = solver[cfg["solver"]] clf = MLPRegressor(random_state=seed, **cfg) scores = cross_val_score(clf, dataset.data, dataset.target, cv=5, scoring="r2") cfg = {k: cfg[k] for k in cfg} return 1 - np.mean(scores) activation = ["identity", "logistic", "tanh", "relu"] solver = ["lbfgs", "sgd", "adam"] cs = ConfigurationSpace() cs.add_hyperparameters([ UniformIntegerHyperparameter("activation", 0, 3), UniformIntegerHyperparameter("solver", 0, 2), UniformFloatHyperparameter("tol", 1e-7, 1e-1), UniformFloatHyperparameter("alpha", 1e-7, 1e-1), UniformIntegerHyperparameter("max_iter", 10, 1000), ]) return (cs, run, "MLP")
def config_space(self): """Decision tree hyperparameter space.""" criterion = CategoricalHyperparameter( 'criterion', ['gini', 'entropy'], default_value='gini' ) # NOTE: Default value = 'none' is translated to None in base class. # ConfigSpace does not allow for None values as default. max_depth = CategoricalHyperparameter( 'max_depth', [5, 10, 20, 'none'], default_value='none' ) # NOTE: Default value = 'none' is translated to None in base class. # ConfigSpace does not allow for None values as default. max_features = CategoricalHyperparameter( 'max_features', ['auto', 'sqrt', 'log2', 'none'], default_value='none' ) min_samples_leaf = UniformIntegerHyperparameter( 'min_samples_leaf', lower=2, upper=5, default_value=3 ) # Add hyperparameters to config space. config = ConfigurationSpace() config.seed(self.random_state) config.add_hyperparameters( ( criterion, max_depth, max_features, min_samples_leaf ) ) return config
def test_save_load_configspace(self): """Check if inputreader can load different config-spaces""" cs = ConfigurationSpace() hyp = UniformFloatHyperparameter('A', 0.0, 1.0, default_value=0.5) cs.add_hyperparameters([hyp]) output_writer = OutputWriter() input_reader = InputReader() # pcs_new output_writer.save_configspace(cs, self.pcs_fn, 'pcs_new') restored_cs = input_reader.read_pcs_file(self.pcs_fn) self.assertEqual(cs, restored_cs) restored_cs = input_reader.read_pcs_file(self.pcs_fn, self.logger) self.assertEqual(cs, restored_cs) # json output_writer.save_configspace(cs, self.json_fn, 'json') restored_cs = input_reader.read_pcs_file(self.json_fn) self.assertEqual(cs, restored_cs) restored_cs = input_reader.read_pcs_file(self.json_fn, self.logger) self.assertEqual(cs, restored_cs) # pcs with open(self.pcs_fn, 'w') as fh: fh.write(pcs.write(cs)) restored_cs = input_reader.read_pcs_file(self.pcs_fn) self.assertEqual(cs, restored_cs) restored_cs = input_reader.read_pcs_file(self.pcs_fn) self.assertEqual(cs, restored_cs) restored_cs = input_reader.read_pcs_file(self.pcs_fn, self.logger) self.assertEqual(cs, restored_cs)
def getPCS(self): ''' maxIter: [1,100]最大迭代次数,默认50 regParam :[0,0.2] 正则化参数,默认0 tol:[1e-6,1e-1] 迭代算法收敛性,默认 1e-6 ''' # Build Configuration Space which defines all parameters and their # ranges cs = ConfigurationSpace() maxIter = UniformIntegerHyperparameter("maxIter", 1, 100, default_value=50) regParam = UniformFloatHyperparameter("regParam", 0, 0.4, default_value=1e-04) tol = UniformFloatHyperparameter("tol", 1e-06, 1e-01, default_value=1e-06) cs.add_hyperparameters([maxIter, regParam, tol]) return cs
def _get_cfg(self): cs = ConfigurationSpace() max_depth = UniformIntegerHyperparameter("max_depth", 3, 16, default_value=3) min_child_weight = UniformIntegerHyperparameter("min_child_weight", 1, 15, default_value=1) gamma = UniformFloatHyperparameter("gamma", 0.0, 0.4, default_value=0.0) subsample = UniformFloatHyperparameter("subsample", 0.6, 0.9, default_value=0.6) colsample_bytree = UniformFloatHyperparameter("colsample_bytree", 0.6, 0.9, default_value=0.6) cs.add_hyperparameters( [max_depth, min_child_weight, gamma, subsample, colsample_bytree]) return cs
def smac_em(self): clu_cs = ConfigurationSpace() n_init = UniformIntegerHyperparameter("n_init", 1, 15) n_com = UniformIntegerHyperparameter("n_components", 2, 10) reg_c = UniformFloatHyperparameter("min_covar", 1e-6, 0.1) max_iter = UniformIntegerHyperparameter("n_iters", 10, 1000) tr = UniformFloatHyperparameter("threshold", 1e-6, 0.1) clu_cs.add_hyperparameters([n_init, tr, reg_c, n_com, max_iter]) clu_scenario = Scenario({ "run_obj": "quality", # we optimize quality (alternatively runtime) # "runcount-limit": Constants.num_eval, # maximum function evaluations "cs": clu_cs, # configuration space "deterministic": "true", "tuner-timeout": constants.em_timeout, "wallclock_limit": constants.em_timeout, "cutoff_time": constants.em_timeout, "runcount-limit": 1 }) print('Run EM SMAC ' + self.name) smac = SMAC(scenario=clu_scenario, tae_runner=self.run_em) parameters = smac.optimize() value = smac.get_runhistory().get_cost(parameters) return value, parameters
def smac_gm(self): clu_cs = ConfigurationSpace() cov_t = CategoricalHyperparameter( "covariance_type", ["full", "tied", "diag", "spherical"]) tol = UniformFloatHyperparameter("tol", 1e-6, 0.1) reg_c = UniformFloatHyperparameter("reg_covar", 1e-10, 0.1) n_com = UniformIntegerHyperparameter("n_components", 2, 10) max_iter = UniformIntegerHyperparameter("max_iter", 10, 1000) clu_cs.add_hyperparameters([cov_t, tol, reg_c, n_com, max_iter]) clu_scenario = Scenario({ "run_obj": "quality", # we optimize quality (alternatively runtime) # "runcount-limit": Constants.num_eval, # maximum function evaluations "cs": clu_cs, # configuration space "deterministic": "true", "tuner-timeout": constants.timeout, "wallclock_limit": constants.timeout, "cutoff_time": constants.timeout }) print('Run GM SMAC ' + self.name) smac = SMAC(scenario=clu_scenario, tae_runner=self.run_gm) parameters = smac.optimize() value = smac.get_runhistory().get_cost(parameters) return value, parameters
def getPCS(self): ''' maxDepth [3,12] 默认5 minInstancesPerNode:[1,100] 默认10 minInfoGain=:[0,0.1] 默认0 maxIter=[ 1,50] ,默认20 stepSize [0.001,1]默认0.1 ''' # Build Configuration Space which defines all parameters and their # ranges cs = ConfigurationSpace() maxDepth = UniformIntegerHyperparameter("maxDepth", 3, 12, default_value=5) maxIter = UniformIntegerHyperparameter("maxIter", 1, 50, default_value=20) minInstancesPerNode = UniformIntegerHyperparameter( "minInstancesPerNode", 1, 100, default_value=10) minInfoGain = UniformFloatHyperparameter("minInfoGain", 1e-06, 1e-01, default_value=1e-06) stepSize = UniformFloatHyperparameter("stepSize", 0.001, 1, default_value=0.1) cs.add_hyperparameters( [maxDepth, maxIter, minInstancesPerNode, minInfoGain, stepSize]) return cs
def config_space(self): """Decision tree hyperparameter space.""" n_estimators = UniformIntegerHyperparameter( 'n_estimators', lower=10, upper=200, default_value=100 ) criterion = CategoricalHyperparameter( 'criterion', ['gini', 'entropy'], default_value='gini' ) max_depth = CategoricalHyperparameter( 'max_depth', [5, 10, 20, 'none'], default_value='none' ) max_features = CategoricalHyperparameter( 'max_features', ['auto', 'sqrt', 'log2', 'none'], default_value='none' ) # Add hyperparameters to config space. config = ConfigurationSpace() config.seed(self.random_state) config.add_hyperparameters( ( n_estimators, criterion, max_depth, max_features ) ) return config
def getPCS(self): ''' maxDepth :[3,12] 树深度 默认5 minInstancesPerNode:[1,100] 默认10 minInfoGain:[0,0.1] 默认0 impurity :["gini", "entropy"],默认”gini” ''' # Build Configuration Space which defines all parameters and their # ranges cs = ConfigurationSpace() maxDepth = UniformIntegerHyperparameter("maxDepth", 3, 12, default_value=5) minInstancesPerNode = UniformIntegerHyperparameter( "minInstancesPerNode", 1, 100, default_value=10) minInfoGain = UniformFloatHyperparameter("minInfoGain", 1e-06, 1e-01, default_value=1e-06) impurity = CategoricalHyperparameter("impurity", ["gini", "entropy"], default_value="gini") cs.add_hyperparameters( [maxDepth, minInstancesPerNode, minInfoGain, impurity]) return cs
def getPCS(self): ''' maxDepth [3,12] 默认5 minInstancesPerNode:[1,100] 默认10 minInfoGain=:[0,0.1] 默认0 subsamplingRate=[0.01,1], 默认1 numTrees:[1,100] 默认 50 ''' # Build Configuration Space which defines all parameters and their # ranges cs = ConfigurationSpace() maxDepth = UniformIntegerHyperparameter("maxDepth", 3, 12, default_value=5) minInstancesPerNode = UniformIntegerHyperparameter( "minInstancesPerNode", 1, 100, default_value=10) minInfoGain = UniformFloatHyperparameter("minInfoGain", 1e-06, 1e-01, default_value=1e-06) numTrees = UniformIntegerHyperparameter("numTrees", 1, 100, default_value=50) cs.add_hyperparameters( [maxDepth, minInstancesPerNode, minInfoGain, numTrees]) return cs
def _get_cfg(self): n_features = self.X_train.shape[1] cs = ConfigurationSpace() max_features = UniformIntegerHyperparameter("max_features", 1, n_features, default_value=1) min_samples_split = UniformIntegerHyperparameter("min_samples_split", 2, 50, default_value=2) min_samples_leaf = UniformIntegerHyperparameter("min_samples_leaf", 1, 50, default_value=1) min_weight_fraction_leaf = UniformFloatHyperparameter( "min_weight_fraction_leaf", 0.0, 0.5, default_value=0.0) max_leaf_nodes = UniformIntegerHyperparameter("max_leaf_nodes", 10, 1000, default_value=100) cs.add_hyperparameters([ max_features, min_samples_split, min_samples_leaf, min_weight_fraction_leaf, max_leaf_nodes ]) return cs
def smac(): # Build Configuration Space which defines all parameters and their ranges configuration_space = ConfigurationSpace() #Define initial ranges num_of_hidden_nodes = UniformIntegerHyperparameter("num_of_hidden_nodes", 1, max_hidden_nodes, default_value=1) decay = UniformFloatHyperparameter("decay", 0, 0.1, default_value=0) configuration_space.add_hyperparameters([num_of_hidden_nodes, decay]) # creating the scenario object scenario = Scenario({ "run_obj": "quality", "runcount-limit": hyperparameter_tuning_configs.SMAC_RUNCOUNT_LIMIT, "cs": configuration_space, "deterministic": "true", "abort_on_first_run_crash": "false" }) # optimize using an SMAC object smac = SMAC(scenario=scenario, rng=np.random.RandomState(1), tae_runner=train_model) incumbent = smac.optimize() return incumbent.get_dictionary()
def get_smac_func_and_space(func, cp_domain): """ Returns a function to be passed to SMAC. """ from smac.configspace import ConfigurationSpace from ConfigSpace.hyperparameters import CategoricalHyperparameter, \ UniformFloatHyperparameter, UniformIntegerHyperparameter smac_configs = [] num_dims_per_domain = [] disc_spaces = [] cp_dom_types = [] # Iterate through each domain counter = 0 for dom in cp_domain.list_of_domains: dom_type = dom.get_type() cp_dom_types.append(dom_type) if dom_type == 'euclidean': num_dims_per_domain.append(dom.get_dim()) for bds in dom.bounds: disc_spaces.append(None) counter, var_label = _get_var_label_with_counter(counter) smac_configs.append( UniformFloatHyperparameter( var_label, bds[0], bds[1], default_value=(bds[0] + bds[1]) / 2)) elif dom_type == 'integral': num_dims_per_domain.append(dom.get_dim()) for bds in dom.bounds: disc_spaces.append(None) counter, var_label = _get_var_label_with_counter(counter) smac_configs.append( UniformIntegerHyperparameter( var_label, bds[0], bds[1], default_value=(bds[0] + bds[1]) // 2)) elif dom_type in ['prod_discrete', 'prod_discrete_numeric']: num_dims_per_domain.append(dom.get_dim()) for lois in dom.list_of_list_of_items: curr_disc_to_idx_converter = DiscItemsToIndexConverter(lois) disc_spaces.append(curr_disc_to_idx_converter) counter, var_label = _get_var_label_with_counter(counter) smac_configs.append( CategoricalHyperparameter( var_label, curr_disc_to_idx_converter.indices, default_value=curr_disc_to_idx_converter.indices[0])) smac_space = ConfigurationSpace() smac_space.add_hyperparameters(smac_configs) # The convert back function smac_convert_pt_back = lambda x: _convert_alphabetical_dict_repr_to_cp_domain_repr( x, num_dims_per_domain, cp_dom_types, disc_spaces) # Then the function smac_func_to_min = lambda x: -func(smac_convert_pt_back(x))[0] return smac_func_to_min, smac_space, smac_convert_pt_back
def _get_cfg(self): n_features = self.X_train.shape[1] cs = ConfigurationSpace() max_depth = UniformIntegerHyperparameter("max_depth", 5, 16, default_value=5) min_samples_split = UniformIntegerHyperparameter("min_samples_split", 200, 1000, default_value=200) min_samples_leaf = UniformIntegerHyperparameter("min_samples_leaf", 30, 70, default_value=30) max_features = UniformIntegerHyperparameter("max_features", 1, n_features, default_value=1) subsample = UniformFloatHyperparameter("subsample", 0.6, 0.9, default_value=0.6) cs.add_hyperparameters([max_depth, min_samples_split, min_samples_leaf, max_features, subsample]) return cs
def main_loop(problem): logging.basicConfig(level=logging.INFO) # logging.DEBUG for debug output cs = ConfigurationSpace() n_neighbors = UniformIntegerHyperparameter("n_neighbors", 2,10, default_value=5) cs.add_hyperparameter(n_neighbors) weights = CategoricalHyperparameter("weights", ["uniform","distance"], default_value="uniform") algorithm = CategoricalHyperparameter("algorithm", ["ball_tree", "kd_tree","brute","auto"], default_value="auto") cs.add_hyperparameters([weights, algorithm]) leaf_size = UniformIntegerHyperparameter("leaf_size", 1, 100, default_value=50) cs.add_hyperparameter(leaf_size) use_leaf_size= InCondition(child=leaf_size, parent=algorithm, values=["ball_tree","kd_tree"]) cs.add_condition(use_leaf_size) p = UniformIntegerHyperparameter("p", 1,3, default_value=2) cs.add_hyperparameter(p) # Scenario object max_eval=100000 scenario = Scenario({"run_obj": "quality", # we optimize quality (alternatively runtime) "runcount-limit": max_eval, # maximum function evaluations "cs": cs, # configuration space "shared_model": True, "output_dir": "/home/naamah/Documents/CatES/result_All/smac/KNN/run_{}_{}_{}".format(max_eval, datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d_%H:%M:%S'), problem) # "output_dir": "/home/naamah/Documents/CatES/result_All/smac/KNN/{}/run_{}_{}".format(problem,max_eval, datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d_%H:%M:%S_%f')), # "input_psmac_dirs":"/home/naamah/Documents/CatES/result_All/", # "deterministic": "true" }) # Example call of the function # It returns: Status, Cost, Runtime, Additional Infos def_value = svm_from_cfg(cs.get_default_configuration()) print("Default Value: %.2f" % (def_value)) # Optimize, using a SMAC-object print("Optimizing! Depending on your machine, this might take a few minutes.") smac = SMAC(scenario=scenario,tae_runner=svm_from_cfg) incumbent = smac.optimize() inc_value = svm_from_cfg(incumbent) print("Optimized Value: %.2f" % (inc_value)) return (incumbent) # main_loop()
def getPCS(self): ''' smoothing: [0.01,100] 默认1 ''' # Build Configuration Space which defines all parameters and their # ranges cs = ConfigurationSpace() smoothing = UniformFloatHyperparameter( "smoothing", 0.01, 100, default_value=1) cs.add_hyperparameters( [smoothing]) return cs
def config_space(self): """Logistic regression hyperparameter space.""" C_param = UniformFloatHyperparameter( 'C', lower=1e-8, upper=1000.0, default_value=1.0 ) penalty = CategoricalHyperparameter( 'penalty', ['l1', 'l2'], default_value='l1' ) # Add hyperparameters to config space. config = ConfigurationSpace() config.seed(self.random_state) config.add_hyperparameters((C_param, penalty)) return config
def config_space(self): """Returns the MI hyperparameter configuration space.""" num_neighbors = UniformIntegerHyperparameter('num_neighbors', lower=10, upper=100, default_value=20) num_features = UniformIntegerHyperparameter('num_features', lower=2, upper=50, default_value=20) config = ConfigurationSpace() config.seed(self.random_state) config.add_hyperparameters((num_neighbors, num_features)) return config
def create_configspace(): cs = ConfigurationSpace() n_estimators = UniformFloatHyperparameter("n_estimators", 100, 600, default_value=200, q=50) eta = UniformFloatHyperparameter("eta", 0.025, 0.3, default_value=0.3, q=0.025) min_child_weight = UniformIntegerHyperparameter("min_child_weight", 1, 10, default_value=1) max_depth = UniformIntegerHyperparameter("max_depth", 1, 14, default_value=6) subsample = UniformFloatHyperparameter("subsample", 0.5, 1, default_value=1, q=0.05) gamma = UniformFloatHyperparameter("gamma", 0, 1, default_value=0, q=0.1) colsample_bytree = UniformFloatHyperparameter("colsample_bytree", 0.5, 1, default_value=1., q=0.05) alpha = UniformFloatHyperparameter("alpha", 0, 10, default_value=0., q=1.) _lambda = UniformFloatHyperparameter("lambda", 1, 2, default_value=1, q=0.1) scale_pos_weight = CategoricalHyperparameter("scale_pos_weight", [0.01, 0.1, 1., 10, 100], default_value=1.) cs.add_hyperparameters([ n_estimators, eta, min_child_weight, max_depth, subsample, gamma, colsample_bytree, alpha, _lambda, scale_pos_weight ]) return cs
def _get_acm_cs(self): """ returns a configuration space designed for querying ~smac.optimizer.smbo._component_builder Returns ------- ConfigurationSpace """ cs = ConfigurationSpace() cs.seed(self.rng.randint(0,2**20)) model = CategoricalHyperparameter("model", choices=("RF", "GP")) num_trees = Constant("num_trees", value=10) bootstrap = CategoricalHyperparameter("do_bootstrapping", choices=(True, False), default_value=True) ratio_features = CategoricalHyperparameter("ratio_features", choices=(3 / 6, 4 / 6, 5 / 6, 1), default_value=1) min_split = UniformIntegerHyperparameter("min_samples_to_split", lower=1, upper=10, default_value=2) min_leaves = UniformIntegerHyperparameter("min_samples_in_leaf", lower=1, upper=10, default_value=1) cs.add_hyperparameters([model, num_trees, bootstrap, ratio_features, min_split, min_leaves]) inc_num_trees = InCondition(num_trees, model, ["RF"]) inc_bootstrap = InCondition(bootstrap, model, ["RF"]) inc_ratio_features = InCondition(ratio_features, model, ["RF"]) inc_min_split = InCondition(min_split, model, ["RF"]) inc_min_leavs = InCondition(min_leaves, model, ["RF"]) cs.add_conditions([inc_num_trees, inc_bootstrap, inc_ratio_features, inc_min_split, inc_min_leavs]) acq = CategoricalHyperparameter("acq_func", choices=("EI", "LCB", "PI", "LogEI")) par_ei = UniformFloatHyperparameter("par_ei", lower=-10, upper=10) par_pi = UniformFloatHyperparameter("par_pi", lower=-10, upper=10) par_logei = UniformFloatHyperparameter("par_logei", lower=0.001, upper=100, log=True) par_lcb = UniformFloatHyperparameter("par_lcb", lower=0.0001, upper=0.9999) cs.add_hyperparameters([acq, par_ei, par_pi, par_logei, par_lcb]) inc_par_ei = InCondition(par_ei, acq, ["EI"]) inc_par_pi = InCondition(par_pi, acq, ["PI"]) inc_par_logei = InCondition(par_logei, acq, ["LogEI"]) inc_par_lcb = InCondition(par_lcb, acq, ["LCB"]) cs.add_conditions([inc_par_ei, inc_par_pi, inc_par_logei, inc_par_lcb]) return cs
def getPCS(self): ''' maxIter: [1,100]最大迭代次数,默认50 regParam :[0,0.2] 正则化参数,默认0 tol:[1e-6,1e-1] 迭代算法收敛性,默认 1e-6 family ,link, variancePower 对应关系 • “gaussian” -> “identity”, “log”, “inverse” • “binomial” -> “logit”, “probit”, “cloglog” • “poisson” -> “log”, “identity”, “sqrt” • “gamma” -> “inverse”, “identity”, “log” • “tweedie” -> power link function specified through “linkPower”. The default link power in the tweedie family is 1 - variancePower. ''' # Build Configuration Space which defines all parameters and their # ranges cs = ConfigurationSpace() maxIter = UniformIntegerHyperparameter("maxIter", 1, 100, default_value=50) regParam = UniformFloatHyperparameter("regParam", 0, 0.4, default_value=1e-04) tol = UniformFloatHyperparameter("tol", 1e-06, 1e-01, default_value=1e-06) family = CategoricalHyperparameter("family", ["gaussian", "poisson"], default_value="gaussian") gaussianLink = CategoricalHyperparameter( "gaussianLink", ["identity", "log", "inverse"], default_value="identity") poissonLink = CategoricalHyperparameter("poissonLink", ["log", "identity", "sqrt"], default_value="log") cs.add_hyperparameters( [maxIter, regParam, tol, family, gaussianLink, poissonLink]) cs.add_condition( InCondition(child=gaussianLink, parent=family, values=["gaussian"])) cs.add_condition( InCondition(child=poissonLink, parent=family, values=["poisson"])) return cs
def best_hyperparams_smac(): iteration = 1 cs = ConfigurationSpace() cs.add_hyperparameters(SVD_SMAC_SPACE.values()) scenario = Scenario({ "run_obj": "quality", # we optimize quality (alternatively runtime) "runcount-limit": 100, # max. number of function evaluations; for this example set to a low number "cs": cs, # configuration space "deterministic": "true" }) smac = SMAC4HPO( scenario=scenario, rng=np.random.RandomState(42), tae_runner=_hyperopt, ) smac.optimize()
def main(): cs = ConfigurationSpace() cell_size = CategoricalHyperparameter("cell_size", [128], default_value=128) # kick up to 256 n_cell = CategoricalHyperparameter("n_cell", [2], default_value=2) dropout = CategoricalHyperparameter("dropout", [0.5], default_value=0.5) activation = CategoricalHyperparameter("activation", ['sigmoid'], default_value='sigmoid') optimizer = CategoricalHyperparameter("optimizer", ['adam'], default_value='adam') optimizer_lr = CategoricalHyperparameter( "optimizer_lr", [.001, .003, .006, .01, 0.03, 0.1], default_value=.01) learning_decay_rate = UniformFloatHyperparameter("learning_decay_rate", 0, 0.9, default_value=.6) nn_type = CategoricalHyperparameter("nn_type", ['RNN', 'LSTM', 'GRU'], default_value='LSTM') epochs = CategoricalHyperparameter("epochs", [10], default_value=10) cs.add_hyperparameters([ cell_size, n_cell, dropout, nn_type, activation, optimizer, optimizer_lr, learning_decay_rate, epochs ]) scenario = Scenario({ "run_obj": "quality", "runcount-limit": 32, "cs": cs, "deterministic": "true" }) scenario.output_dir_for_this_run = "C:\\NNwork\\HFSF\\SMAC3out" scenario.output_dir = "C:\\NNwork\\HFSF\\SMAC3out" smac = SMAC(scenario=scenario, rng=np.random.RandomState(23), tae_runner=rnn_from_cfg) best_model = smac.optimize() print_incumb(best_model) np.save("C:\\NNwork\\HFSF\\SMAC3out\\best.cfg", best_model)
def smac_opt(): # Import ConfigSpace and different types of parameters from smac.configspace import ConfigurationSpace from ConfigSpace.hyperparameters import UniformFloatHyperparameter # Import SMAC-utilities from smac.scenario.scenario import Scenario from smac.facade.smac_hpo_facade import SMAC4HPO n_params = 5 def fun_to_optimize(x): from jnius import autoclass RunStrategy = autoclass('RunStrategy') params = [x[f'x{i}'] for i in range(0, n_params)] print(f'params:{params}') ret = -RunStrategy.runStrategyWithConfiguration(params, 20) print(ret) return ret cs = ConfigurationSpace() hyper_params = [ UniformFloatHyperparameter(f"x{i}", 0, 100, default_value=1) for i in range(0, n_params) ] cs.add_hyperparameters(hyper_params) # Scenario object scenario = Scenario({ "run_obj": "quality", # we optimize quality (alternatively runtime) "runcount-limit": 999999, # max. number of function evaluations; for this example set to a low number "cs": cs, # configuration space "deterministic": "false" }) # Optimize, using a SMAC-object smac = SMAC4HPO(scenario=scenario, rng=np.random.RandomState(42), tae_runner=fun_to_optimize) smac.optimize()
def bayesian_optimize(self): """conduct Bayesian optimization on the hyperparameters, starting at current values""" if self.algorithm in ['GNB','Perceptron']: return self else: cs = ConfigurationSpace() cs.add_hyperparameters(list(getattr(util, self.algorithm + '_range')(self.hyperparameters).values())) if self.algorithm == 'kNN': if self.hyperparameters['k'] == 1: num = 3 else: num = 5 else: num = 10 scenario = Scenario({'run_obj': 'quality', 'runcount-limit': num, 'cs': cs, 'deterministic': 'true', 'memory_limit': None}) smac = SMAC(scenario=scenario, rng=np.random.RandomState(100), tae_runner=self.error_function) try: incumbent = smac.optimize() finally: incumbent = smac.solver.incumbent self.error = smac.get_tae_runner().run(incumbent, 1)[1] self.hyperparameters = incumbent.get_dictionary() self.bayesian_optimized = True return self
def config_space(self): """SVC hyperparameter space.""" C_param = UniformFloatHyperparameter( 'C', lower=1e-8, upper=100.0, default_value=1.0 ) shrinking = CategoricalHyperparameter( 'shrinking', [True, False], default_value=True ) kernel = CategoricalHyperparameter( 'kernel', ['linear', 'rbf', 'poly', 'sigmoid'], ) degree = UniformIntegerHyperparameter( 'degree', lower=1, upper=5, default_value=2 ) coef0 = UniformFloatHyperparameter( 'coef0', lower=0.0, upper=10.0, default_value=0.0 ) # Add hyperparameters to config space. config = ConfigurationSpace() config.seed(self.random_state) config.add_hyperparameters( ( C_param, shrinking, kernel, degree, coef0, ) ) # Conditionals on hyperparameters specific to kernels. config.add_conditions( ( InCondition(child=degree, parent=kernel, values=['poly']), InCondition( child=coef0, parent=kernel, values=['poly', 'sigmoid'] ) ) ) return config
def config_space(self): """XGBoost hyperparameter space.""" # The mumber of Decision Trees. n_estimators = UniformIntegerHyperparameter( 'n_estimators', lower=10, upper=200, default_value=100 ) # The maximum depth of each decision tree. Generally, boosting # algorithms are configured with weak learners = shallow decision trees. max_depth = UniformIntegerHyperparameter( 'max_depth', lower=5, upper=500, default_value=100 ) # L1 regularization term on weights. reg_alpha = UniformFloatHyperparameter( 'reg_alpha', lower=1e-8, upper=100, default_value=1e-3 ) # L2 regularization term on weights. reg_lambda = UniformFloatHyperparameter( 'reg_lambda', lower=1e-8, upper=100, default_value=1e-3 ) learning_rate = UniformFloatHyperparameter( 'learning_rate', lower=1e-8, upper=50, default_value=0.01 ) min_data_in_leaf = UniformIntegerHyperparameter( 'min_data_in_leaf', lower=2, upper=5, default_value=3 ) # Add hyperparameters to config space. config = ConfigurationSpace() config.seed(self.random_state) config.add_hyperparameters( ( n_estimators, min_data_in_leaf, max_depth, reg_alpha, reg_lambda, learning_rate ) ) return config