def get_random_forest_default_search_space(): cs = ConfigSpace.ConfigurationSpace() imputation = ConfigSpace.CategoricalHyperparameter( 'imputation__strategy', ['mean', 'median', 'most_frequent']) n_estimators = ConfigSpace.Constant("classifier__n_estimators", 100) criterion = ConfigSpace.CategoricalHyperparameter("classifier__criterion", ["gini", "entropy"], default_value="gini") # The maximum number of features used in the forest is calculated as m^max_features, where # m is the total number of features, and max_features is the hyperparameter specified below. # The default is 0.5, which yields sqrt(m) features as max_features in the estimator. This # corresponds with Geurts' heuristic. max_features = ConfigSpace.UniformFloatHyperparameter( "classifier__max_features", 0., 1., default_value=0.5) # max_depth = ConfigSpace.UnParametrizedHyperparameter("classifier__max_depth", None) min_samples_split = ConfigSpace.UniformIntegerHyperparameter( "classifier__min_samples_split", 2, 20, default_value=2) min_samples_leaf = ConfigSpace.UniformIntegerHyperparameter( "classifier__min_samples_leaf", 1, 20, default_value=1) min_weight_fraction_leaf = ConfigSpace.UnParametrizedHyperparameter( "classifier__min_weight_fraction_leaf", 0.) # max_leaf_nodes = ConfigSpace.UnParametrizedHyperparameter("classifier__max_leaf_nodes", None) bootstrap = ConfigSpace.CategoricalHyperparameter("classifier__bootstrap", ["True", "False"], default_value="True") cs.add_hyperparameters([ imputation, n_estimators, criterion, max_features, min_samples_split, min_samples_leaf, min_weight_fraction_leaf, bootstrap ]) return cs
def __init__(self, development_stage=-1): super().__init__() self.development_stage = development_stage self.add_hyperparameter( CS.UniformIntegerHyperparameter("dropout_1", lower=0, upper=2) ) self.add_hyperparameter( CS.UniformIntegerHyperparameter("dropout_2", lower=0, upper=2) ) self.add_hyperparameter( CS.UniformIntegerHyperparameter("init_lr", lower=0, upper=5) ) self.add_hyperparameter( CS.CategoricalHyperparameter("activation_fn_1", choices=["relu", "tanh"]) ) self.add_hyperparameter( CS.CategoricalHyperparameter("activation_fn_2", choices=["relu", "tanh"]) ) if development_stage == 0: self.budget = 50 self.add_hyperparameter( CS.UniformIntegerHyperparameter("batch_size", lower=0, upper=3) ) else: self.budget = 100
def get_configuration_space(): cs = ConfigSpace.ConfigurationSpace() cs.add_hyperparameter( ConfigSpace.OrdinalHyperparameter("n_units_1", [16, 32, 64, 128, 256, 512])) cs.add_hyperparameter( ConfigSpace.OrdinalHyperparameter("n_units_2", [16, 32, 64, 128, 256, 512])) cs.add_hyperparameter( ConfigSpace.OrdinalHyperparameter("dropout_1", [0.0, 0.3, 0.6])) cs.add_hyperparameter( ConfigSpace.OrdinalHyperparameter("dropout_2", [0.0, 0.3, 0.6])) cs.add_hyperparameter( ConfigSpace.CategoricalHyperparameter("activation_fn_1", ["tanh", "relu"])) cs.add_hyperparameter( ConfigSpace.CategoricalHyperparameter("activation_fn_2", ["tanh", "relu"])) cs.add_hyperparameter( ConfigSpace.OrdinalHyperparameter( "init_lr", [5 * 1e-4, 1e-3, 5 * 1e-3, 1e-2, 5 * 1e-2, 1e-1])) cs.add_hyperparameter( ConfigSpace.CategoricalHyperparameter("lr_schedule", ["cosine", "const"])) cs.add_hyperparameter( ConfigSpace.OrdinalHyperparameter("batch_size", [8, 16, 32, 64])) return cs
def get_hyperparameter_search_space(seed) -> ConfigSpaceWrapper: """ The SVM configuration space based on the search space from auto-sklearn: https://github.com/automl/auto-sklearn/blob/master/autosklearn/pipeline/components/classification/libsvm_svc.py Parameters ---------- seed: int Random seed that will be used to sample random configurations Returns ------- cs: ConfigSpace.ConfigurationSpace The configuration space object """ cs = ConfigSpace.ConfigurationSpace('sklearn.svm.SVC', seed) C = ConfigSpace.UniformFloatHyperparameter(name='C', lower=0.03125, upper=32768, log=True, default_value=1.0) kernel = ConfigSpace.CategoricalHyperparameter( name='kernel', choices=['rbf', 'poly', 'sigmoid'], default_value='rbf') degree = ConfigSpace.UniformIntegerHyperparameter(name='degree', lower=1, upper=5, default_value=3) gamma = ConfigSpace.UniformFloatHyperparameter(name='gamma', lower=3.0517578125e-05, upper=8, log=True, default_value=0.1) coef0 = ConfigSpace.UniformFloatHyperparameter(name='coef0', lower=-1, upper=1, default_value=0) shrinking = ConfigSpace.CategoricalHyperparameter(name='shrinking', choices=[True, False], default_value=True) tol = ConfigSpace.UniformFloatHyperparameter(name='tol', lower=1e-5, upper=1e-1, default_value=1e-3, log=True) max_iter = ConfigSpace.UnParametrizedHyperparameter('max_iter', -1) hyperparameters = [ C, kernel, degree, gamma, coef0, shrinking, tol, max_iter ] degree_depends_on_poly = ConfigSpace.EqualsCondition( degree, kernel, 'poly') coef0_condition = ConfigSpace.InCondition(coef0, kernel, ['poly', 'sigmoid']) conditions = [degree_depends_on_poly, coef0_condition] return ConfigSpaceWrapper(cs, hyperparameters, conditions)
def get_hyperparameter_search_space(seed=None): """ Neural Network search space based on a best effort using the scikit-learn implementation. Note that for state of the art performance, other packages could be preferred. Parameters ---------- seed: int Random seed that will be used to sample random configurations Returns ------- cs: ConfigSpace.ConfigurationSpace The configuration space object """ cs = ConfigSpace.ConfigurationSpace('ResNet18_classifier', seed) # batch_size = ConfigSpace.UniformIntegerHyperparameter( # name='batch_size', lower=1, upper=256, log=True, default_value=128) # learning_rate = ConfigSpace.CategoricalHyperparameter( # name='learning_rate', choices=['constant', 'invscaling', 'adaptive'], default_value='constant') learning_rate = ConfigSpace.UniformFloatHyperparameter( name='learning_rate', lower=1e-6, upper=1e-1, log=True, default_value=1e-2) epochs = ConfigSpace.UniformIntegerHyperparameter(name='epochs', lower=1, upper=50, default_value=20) batch_size = ConfigSpace.CategoricalHyperparameter( name='batch_size', choices=[1, 2, 4, 8, 16, 32, 64, 128, 256, 512], default_value=128) shuffle = ConfigSpace.CategoricalHyperparameter(name='shuffle', choices=[True, False], default_value=True) momentum = ConfigSpace.UniformFloatHyperparameter(name='momentum', lower=0, upper=1, default_value=0.9) weight_decay = ConfigSpace.UniformFloatHyperparameter(name='weight_decay', lower=1e-6, upper=1e-2, log=True, default_value=5e-4) cs.add_hyperparameters([ batch_size, learning_rate, epochs, shuffle, momentum, weight_decay, ]) return cs
def get_hyperparameter_search_space(seed=None): """ Neural Network search space based on a best effort using the scikit-learn implementation. Note that for state of the art performance, other packages could be preferred. Parameters ---------- seed: int Random seed that will be used to sample random configurations Returns ------- cs: ConfigSpace.ConfigurationSpace The configuration space object """ cs = ConfigSpace.ConfigurationSpace('ResNet18_classifier', seed) learning_rate_init = ConfigSpace.UniformFloatHyperparameter( name='learning_rate_init', lower=1e-6, upper=1e-1, log=True, default_value=1e-2) epochs = ConfigSpace.UniformIntegerHyperparameter( name='epochs', lower=1, upper=50, default_value=20) batch_size = ConfigSpace.CategoricalHyperparameter( name='batch_size', choices=[32, 64, 128, 256, 512], default_value=128) momentum = ConfigSpace.UniformFloatHyperparameter( name='momentum', lower=0, upper=1, default_value=0.9) weight_decay = ConfigSpace.UniformFloatHyperparameter( name='weight_decay', lower=1e-6, upper=1e-2, log=True, default_value=5e-4) lr_decay = ConfigSpace.UniformIntegerHyperparameter( name='learning_rate_decay', lower=2, upper=1000, log=True, default_value=10) patience = ConfigSpace.UniformIntegerHyperparameter( name='patience', lower=2, upper=200, log=False, default_value=10) tolerance = ConfigSpace.UniformFloatHyperparameter( name='tolerance', lower=1e-5, upper=1e-2, log=True, default_value=1e-4) resize_crop = ConfigSpace.CategoricalHyperparameter( name='resize_crop', choices=[True, False], default_value=False) h_flip = ConfigSpace.CategoricalHyperparameter( name='horizontal_flip', choices=[True, False], default_value=False) v_flip = ConfigSpace.CategoricalHyperparameter( name='vertical_flip', choices=[True, False], default_value=False) shuffle = ConfigSpace.CategoricalHyperparameter( name='shuffle', choices=[True, False], default_value=True) cs.add_hyperparameters([ batch_size, learning_rate_init, epochs, momentum, weight_decay, lr_decay, patience, tolerance, resize_crop, h_flip, v_flip, shuffle, ]) return cs
def get_config_space(): config_space = CS.ConfigurationSpace() config_space.add_hyperparameter( CS.UniformFloatHyperparameter('momentum', lower=0, upper=1, default_value=0.9)) config_space.add_hyperparameter( CS.UniformFloatHyperparameter('initial_lr', lower=1e-3, upper=1, default_value=1e-1, log=True)) config_space.add_hyperparameter( CS.UniformIntegerHyperparameter('lr_step_length', lower=1, upper=8, default_value=4)) config_space.add_hyperparameter( CS.UniformFloatHyperparameter('lr_decay_factor', lower=1e-4, upper=1, default_value=1e-1, log=True)) config_space.add_hyperparameter( CS.UniformFloatHyperparameter('dropout_rate', lower=0, upper=0.5, default_value=0.5, log=False)) config_space.add_hyperparameter( CS.CategoricalHyperparameter('num_filters_1', choices=[4, 8, 16, 32], default_value=8)) config_space.add_hyperparameter( CS.CategoricalHyperparameter('num_filters_2', choices=[4, 8, 16, 32], default_value=8)) config_space.add_hyperparameter( CS.CategoricalHyperparameter('kernel_size', choices=[3, 5, 7], default_value=3)) config_space.add_hyperparameter( CS.UniformIntegerHyperparameter('stride', lower=1, upper=5, default_value=1)) config_space.add_hyperparameter( CS.UniformIntegerHyperparameter('max_pool_size', lower=2, upper=4, default_value=2)) config_space.add_hyperparameter( CS.CategoricalHyperparameter('num_fc_units', choices=[8, 16, 32, 64])) return (config_space)
def main(args): cfg = setup(args) search_space = CS.ConfigurationSpace() search_space.add_hyperparameters([ CS.UniformFloatHyperparameter(name="lr", lower=1e-6, upper=1e-3), CS.UniformFloatHyperparameter(name="wd", lower=0, upper=1e-3), CS.UniformFloatHyperparameter(name="wd_bias", lower=0, upper=1e-3), CS.CategoricalHyperparameter(name="bsz", choices=[64, 96, 128, 160, 224, 256]), CS.CategoricalHyperparameter(name="num_inst", choices=[2, 4, 8, 16, 32]), CS.UniformIntegerHyperparameter(name="delay_iters", lower=20, upper=60), CS.UniformFloatHyperparameter(name="ce_scale", lower=0.1, upper=1.0), CS.UniformIntegerHyperparameter(name="circle_scale", lower=8, upper=256), CS.UniformFloatHyperparameter(name="circle_margin", lower=0.1, upper=0.5), CS.CategoricalHyperparameter(name="autoaug_enabled", choices=[True, False]), CS.CategoricalHyperparameter(name="cj_enabled", choices=[True, False]), ]) exp_metrics = dict(metric="score", mode="max") bohb_hyperband = HyperBandForBOHB( time_attr="training_iteration", max_t=7, **exp_metrics, ) bohb_search = TuneBOHB(search_space, max_concurrent=4, **exp_metrics) reporter = CLIReporter(parameter_columns=["bsz", "num_inst", "lr"], metric_columns=["r1", "map", "training_iteration"]) analysis = tune.run(partial(train_reid_tune, cfg), resources_per_trial={ "cpu": 10, "gpu": 1 }, search_alg=bohb_search, num_samples=args.num_samples, scheduler=bohb_hyperband, progress_reporter=reporter, local_dir=cfg.OUTPUT_DIR, keep_checkpoints_num=4, name="bohb") best_trial = analysis.get_best_trial("map", "max", "last") logger.info("Best trial config: {}".format(best_trial.config)) logger.info("Best trial final validation mAP: {}, Rank-1: {}".format( best_trial.last_result["map"], best_trial.last_result["r1"]))
def get_hyperparameter_search_space(seed) -> ConfigSpaceWrapper: """ The random forest configuration space based on the search space from auto-sklearn: https://github.com/automl/auto-sklearn/blob/master/autosklearn/pipeline/components/classification/random_forest.py Parameters ---------- seed: int Random seed that will be used to sample random configurations Returns ------- cs: ConfigSpace.ConfigurationSpace The configuration space object """ cs = ConfigSpace.ConfigurationSpace( 'sklearn.ensemble.RandomForestClassifier', seed) # TODO: parameterize the number of estimators? n_estimators = ConfigSpace.Constant(name='n_estimators', value=100) criterion = ConfigSpace.CategoricalHyperparameter( name='criterion', choices=['gini', 'entropy'], default_value='gini') max_features = ConfigSpace.UniformFloatHyperparameter(name='max_features', lower=0., upper=1., default_value=0.5) # max_depth = ConfigSpace.UnParametrizedHyperparameter( # name='randomforestclassifier__max_depth', value=None) min_samples_split = ConfigSpace.UniformIntegerHyperparameter( name='min_samples_split', lower=2, upper=20, default_value=2) min_samples_leaf = ConfigSpace.UniformIntegerHyperparameter( name='min_samples_leaf', lower=1, upper=20, default_value=1) min_weight_fraction_leaf = ConfigSpace.UnParametrizedHyperparameter( name='min_weight_fraction_leaf', value=0.) # max_leaf_nodes = ConfigSpace.UnParametrizedHyperparameter( # name='randomforestclassifier__max_leaf_nodes', value=None) bootstrap = ConfigSpace.CategoricalHyperparameter(name='bootstrap', choices=[True, False], default_value=True) hyperparameters = [ n_estimators, criterion, max_features, # max_depth, min_samples_split, min_samples_leaf, min_weight_fraction_leaf, # max_leaf_nodes, bootstrap ] return ConfigSpaceWrapper(cs, hyperparameters, None)
def get_configuration_space(self): cs = ConfigSpace.ConfigurationSpace() for node in list(self.num_parents_per_node.keys())[1:-1]: cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("choice_block_{}_op".format(node), [CONV1X1, CONV3X3, MAXPOOL3X3])) for choice_block_index, num_parents in list(self.num_parents_per_node.items())[1:]: cs.add_hyperparameter( ConfigSpace.CategoricalHyperparameter( "choice_block_{}_parents".format(choice_block_index), parent_combinations(node=choice_block_index, num_parents=num_parents))) return cs
def get_configuration_space(): cs = ConfigSpace.ConfigurationSpace() ops_choices = ['conv1x1-bn-relu', 'conv3x3-bn-relu', 'maxpool3x3'] cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_0", ops_choices)) cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_1", ops_choices)) cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_2", ops_choices)) cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_3", ops_choices)) cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("op_node_4", ops_choices)) cs.add_hyperparameter(ConfigSpace.UniformIntegerHyperparameter("num_edges", 0, MAX_EDGES)) for i in range(VERTICES * (VERTICES - 1) // 2): cs.add_hyperparameter(ConfigSpace.UniformFloatHyperparameter("edge_%d" % i, 0, 1)) return cs
def get_hyperparameter_search_space(seed) -> ConfigSpaceWrapper: """ The Bernoulli NB configuration space based on the search space from auto-sklearn: https://github.com/automl/auto-sklearn/blob/master/autosklearn/pipeline/components/classification/bernoulli_nb.py Parameters ---------- seed: int Random seed that will be used to sample random configurations Returns ------- cs: ConfigSpace.ConfigurationSpace The configuration space object """ cs = ConfigSpace.ConfigurationSpace('sklearn.naive_bayes.BernoulliNB', seed) # the smoothing parameter is a non-negative float # I will limit it to 1000 and put it on a logarithmic scale. (SF) # Please adjust that, if you know a proper range, this is just a guess. alpha = ConfigSpace.UniformFloatHyperparameter(name='alpha', lower=1e-2, upper=100, default_value=1, log=True) fit_prior = ConfigSpace.CategoricalHyperparameter(name='fit_prior', choices=[True, False], default_value=True) hyperparameters = [alpha, fit_prior] return ConfigSpaceWrapper(cs, hyperparameters, None)
def add_hyperparameters(self): HPs=[] HPs.append( CS.CategoricalHyperparameter('cat1', choices=['foo', 'bar', 'baz'])) HPs.append( CS.UniformFloatHyperparameter('cont1', lower=0, upper=1)) HPs.append( CS.UniformIntegerHyperparameter('int1', lower=-2, upper=2)) HPs.append( CS.OrdinalHyperparameter('ord1', ['cold', 'mild', 'warm', 'hot'])) self.configspace.add_hyperparameters(HPs)
def get_configuration_space( seed: Union[int, None] = None) -> CS.ConfigurationSpace: """ Return the CS representation of the search space. From https://github.com/D-X-Y/AutoDL-Projects/blob/master/exps/algos/BOHB.py Author: https://github.com/D-X-Y [[email protected]] Parameters ---------- seed : int, None Random seed for the configuration space. Returns ------- CS.ConfigurationSpace - Containing the benchmark's hyperparameter """ seed = seed if seed is not None else np.random.randint(1, 100000) cs = CS.ConfigurationSpace(seed=seed) search_space = NasBench201BaseBenchmark.get_search_spaces( 'cell', 'nas-bench-201') hps = [ CS.CategoricalHyperparameter(f'{i}<-{j}', search_space) for i in range(1, MAX_NODES) for j in range(i) ] cs.add_hyperparameters(hps) return cs
def resolve_value( par: str, domain: Domain) -> ConfigSpace.hyperparameters.Hyperparameter: quantize = None sampler = domain.get_sampler() if isinstance(sampler, Quantized): quantize = sampler.q sampler = sampler.sampler if isinstance(domain, Float): if isinstance(sampler, LogUniform): lower = domain.lower upper = domain.upper if quantize: lower = math.ceil(domain.lower / quantize) * quantize upper = math.floor(domain.upper / quantize) * quantize return ConfigSpace.UniformFloatHyperparameter(par, lower=lower, upper=upper, q=quantize, log=True) elif isinstance(sampler, Uniform): lower = domain.lower upper = domain.upper if quantize: lower = math.ceil(domain.lower / quantize) * quantize upper = math.floor(domain.upper / quantize) * quantize return ConfigSpace.UniformFloatHyperparameter(par, lower=lower, upper=upper, q=quantize, log=False) elif isinstance(sampler, Normal): return ConfigSpace.NormalFloatHyperparameter( par, mu=sampler.mean, sigma=sampler.sd, q=quantize, log=False) elif isinstance(domain, Integer): if isinstance(sampler, Uniform): lower = domain.lower upper = domain.upper if quantize: lower = math.ceil(domain.lower / quantize) * quantize upper = math.floor(domain.upper / quantize) * quantize return ConfigSpace.UniformIntegerHyperparameter( par, lower=lower, upper=upper, q=quantize, log=False) elif isinstance(domain, Categorical): if isinstance(sampler, Uniform): return ConfigSpace.CategoricalHyperparameter( par, choices=domain.categories) raise ValueError("TuneBOHB does not support parameters of type " "`{}` with samplers of type `{}`".format( type(domain).__name__, type(domain.sampler).__name__))
def get_size_config_space(search_space): cs = ConfigSpace.ConfigurationSpace() for ilayer in range(search_space['numbers']): node_str = 'layer-{:}'.format(ilayer) cs.add_hyperparameter( ConfigSpace.CategoricalHyperparameter(node_str, search_space['candidates'])) return cs
def get_configuration_space(max_nodes, search_space): cs = ConfigSpace.ConfigurationSpace() for i in range(1, max_nodes): for j in range(i): node_str = '{:}<-{:}'.format(i, j) cs.add_hyperparameter( ConfigSpace.CategoricalHyperparameter(node_str, search_space)) return cs
def get_configuration_space(): # for unpruned graph cs = ConfigSpace.ConfigurationSpace() ops_choices = ['nor_conv_3x3', 'nor_conv_1x1', 'avg_pool_3x3', 'skip_connect', 'none'] for i in range(6): cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("edge_%d" % i, ops_choices)) return cs
def get_topology_config_space(search_space, max_nodes=4): cs = ConfigSpace.ConfigurationSpace() #edge2index = {} for i in range(1, max_nodes): for j in range(i): node_str = '{:}<-{:}'.format(i, j) cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter(node_str, search_space)) return cs
def testConvertBOHB(self): from ray.tune.suggest.bohb import TuneBOHB import ConfigSpace config = { "a": tune.sample.Categorical([2, 3, 4]).uniform(), "b": { "x": tune.sample.Integer(0, 5).quantized(2), "y": 4, "z": tune.sample.Float(1e-4, 1e-2).loguniform() } } converted_config = TuneBOHB.convert_search_space(config) bohb_config = ConfigSpace.ConfigurationSpace() bohb_config.add_hyperparameters([ ConfigSpace.CategoricalHyperparameter("a", [2, 3, 4]), ConfigSpace.UniformIntegerHyperparameter("b/x", lower=0, upper=4, q=2), ConfigSpace.UniformFloatHyperparameter("b/z", lower=1e-4, upper=1e-2, log=True) ]) converted_config.seed(1234) bohb_config.seed(1234) searcher1 = TuneBOHB(space=converted_config) searcher2 = TuneBOHB(space=bohb_config) config1 = searcher1.suggest("0") config2 = searcher2.suggest("0") self.assertEqual(config1, config2) self.assertIn(config1["a"], [2, 3, 4]) self.assertIn(config1["b"]["x"], list(range(5))) self.assertLess(1e-4, config1["b"]["z"]) self.assertLess(config1["b"]["z"], 1e-2) searcher = TuneBOHB(metric="a", mode="max") analysis = tune.run(_mock_objective, config=config, search_alg=searcher, num_samples=1) trial = analysis.trials[0] self.assertIn(trial.config["a"], [2, 3, 4]) self.assertEqual(trial.config["b"]["y"], 4) mixed_config = { "a": tune.uniform(5, 6), "b": tune.uniform(8, 9) # Cannot mix ConfigSpace and Dict } searcher = TuneBOHB(space=mixed_config, metric="a", mode="max") config = searcher.suggest("0") self.assertTrue(5 <= config["a"] <= 6) self.assertTrue(8 <= config["b"] <= 9)
def get_neural_network_default_search_space(): config_space = ConfigSpace.ConfigurationSpace() config_space.add_hyperparameter( ConfigSpace.CategoricalHyperparameter('imputation:strategy', ['mean', 'median', 'most_frequent'])) config_space.add_hyperparameter(ConfigSpace.CategoricalHyperparameter('classifier:__choice__', ['neural_network'])) config_space.add_hyperparameter( ConfigSpace.UniformIntegerHyperparameter('classifier:neural_network:hidden_layer_sizes', 32, 1024)) config_space.add_hyperparameter( ConfigSpace.UniformIntegerHyperparameter('classifier:neural_network:num_hidden_layers', 1, 5)) config_space.add_hyperparameter( ConfigSpace.UniformFloatHyperparameter('classifier:neural_network:learning_rate_init', 0.00001, 1, log=True)) config_space.add_hyperparameter( ConfigSpace.UniformFloatHyperparameter('classifier:neural_network:alpha', 0.0000001, 0.0001, log=True)) # config_space.add_hyperparameter(ConfigSpace.UniformFloatHyperparameter('classifier:neural_network:beta_1', 0, 1)) # config_space.add_hyperparameter(ConfigSpace.UniformFloatHyperparameter('classifier:neural_network:beta_2', 0, 1)) # config_space.add_hyperparameter(ConfigSpace.UniformIntegerHyperparameter('classifier:neural_network:max_iter', 2, 1000)) config_space.add_hyperparameter(ConfigSpace.UniformFloatHyperparameter('classifier:neural_network:momentum', 0.1, 0.9)) return config_space
def setUp(self): root = logging.getLogger() root.setLevel(logging.INFO) self.cs = ConfigSpace.ConfigurationSpace( 'sklearn.ensemble.GradientBoostingClassifier', 0) self.cs.add_hyperparameter( ConfigSpace.CategoricalHyperparameter(name='algorithm', choices=['a1', 'a2', 'a3']))
def setUp(self): self.configspace = CS.ConfigurationSpace() self.HPs = [] self.HPs.append( CS.CategoricalHyperparameter('parent', [1,2,3])) self.HPs.append( CS.CategoricalHyperparameter('child1_x1', ['foo','bar'])) self.HPs.append( CS.UniformFloatHyperparameter('child2_x1', lower=-1, upper=1)) self.HPs.append( CS.UniformIntegerHyperparameter('child3_x1', lower=-2, upper=5)) self.configspace.add_hyperparameters(self.HPs) self.conditions = [] self.conditions += [CS.EqualsCondition(self.HPs[1], self.HPs[0], 1)] self.conditions += [CS.EqualsCondition(self.HPs[2], self.HPs[0], 2)] self.conditions += [CS.EqualsCondition(self.HPs[3], self.HPs[0], 3)] [self.configspace.add_condition(cond) for cond in self.conditions]
def get_libsvm_svc_default_search_space(): imputation = ConfigSpace.CategoricalHyperparameter('imputation__strategy', ['mean', 'median', 'most_frequent']) C = ConfigSpace.UniformFloatHyperparameter("classifier__C", 0.03125, 32768, log=True, default_value=1.0) kernel = ConfigSpace.CategoricalHyperparameter(name="classifier__kernel", choices=["rbf", "poly", "sigmoid"], default_value="rbf") degree = ConfigSpace.UniformIntegerHyperparameter("classifier__degree", 1, 5, default_value=3) gamma = ConfigSpace.UniformFloatHyperparameter("classifier__gamma", 3.0517578125e-05, 8, log=True, default_value=0.1) coef0 = ConfigSpace.UniformFloatHyperparameter("classifier__coef0", -1, 1, default_value=0) shrinking = ConfigSpace.CategoricalHyperparameter("classifier__shrinking", ["True", "False"], default_value="True") tol = ConfigSpace.UniformFloatHyperparameter("classifier__tol", 1e-5, 1e-1, default_value=1e-3, log=True) max_iter = ConfigSpace.UnParametrizedHyperparameter("classifier__max_iter", -1) cs = ConfigSpace.ConfigurationSpace() cs.add_hyperparameters([imputation, C, kernel, degree, gamma, coef0, shrinking, tol, max_iter]) degree_depends_on_poly = ConfigSpace.EqualsCondition(degree, kernel, "poly") coef0_condition = ConfigSpace.InCondition(coef0, kernel, ["poly", "sigmoid"]) cs.add_condition(degree_depends_on_poly) cs.add_condition(coef0_condition) return cs
def get_configuration_space(self): cs = ConfigSpace.ConfigurationSpace() OPS = PRIMITIVES[0:-1] for cell in ['normal', 'reduce']: for node in range(2, 6): for prev in range(0, node): cs.add_hyperparameter( ConfigSpace.CategoricalHyperparameter( "{}_{}_{}".format(cell, node, prev), OPS)) return cs
def get_config_space(self): cs = CS.ConfigurationSpace() cs.add_hyperparameter( CS.UniformIntegerHyperparameter("n_units_1", lower=0, upper=5)) cs.add_hyperparameter( CS.UniformIntegerHyperparameter("n_units_2", lower=0, upper=5)) cs.add_hyperparameter( CS.UniformIntegerHyperparameter("dropout_1", lower=0, upper=2)) cs.add_hyperparameter( CS.UniformIntegerHyperparameter("dropout_2", lower=0, upper=2)) cs.add_hyperparameter( CS.CategoricalHyperparameter("activation_fn_1", ["tanh", "relu"])) cs.add_hyperparameter( CS.CategoricalHyperparameter("activation_fn_2", ["tanh", "relu"])) cs.add_hyperparameter( CS.UniformIntegerHyperparameter("init_lr", lower=0, upper=5)) cs.add_hyperparameter( CS.CategoricalHyperparameter("lr_schedule", ["cosine", "const"])) cs.add_hyperparameter( CS.UniformIntegerHyperparameter("batch_size", lower=0, upper=3)) return cs
def setUp(self): self.configspace = CS.ConfigurationSpace(43) HPs=[] HPs.append( CS.CategoricalHyperparameter('cat1', choices=['foo', 'bar', 'baz'])) self.configspace.add_hyperparameters(HPs) x_train_confs = [ self.configspace.sample_configuration() for i in range(self.n_train)] self.x_train = np.array( [c.get_array() for c in x_train_confs]).squeeze() x_test_confs = [ self.configspace.sample_configuration() for i in range(self.n_test)] self.x_test= np.array( [c.get_array() for c in x_train_confs]).squeeze()
def setUp(self): self.X = np.loadtxt('toy_data_set_features.csv', delimiter=',') self.y = np.loadtxt('toy_data_set_responses.csv', delimiter=',') self.cfs = cfs.ConfigurationSpace() f1 = cfs.UniformFloatHyperparameter('x1', 0, 100) f2 = cfs.CategoricalHyperparameter('x2', [0, 1, 2]) self.cfs.add_hyperparameter(f1) self.cfs.add_hyperparameter(f2)
def get_configspace(self): class MyConstrainedConfigurationSpace(ConfigSpace.ConfigurationSpace): def __init__(self, tp, computer, t): super(MyConstrainedConfigurationSpace, self).__init__() self.t = t self.tp = tp self.computer = computer def sample_configuration(self, size=1): cond = False cpt = 0 while (not cond): cpt += 1 if (size == 1): accepted_configurations = [super(MyConstrainedConfigurationSpace, self).sample_configuration(size=size)] else: accepted_configurations = super(MyConstrainedConfigurationSpace, self).sample_configuration(size=size) for config in accepted_configurations: t = self.t # print(config,type(config)) x = [config[p] for p in self.tp.parameter_space.dimension_names] kwargs = {d.name: x[i] for (i, d) in enumerate(self.tp.parameter_space)} kwargs2 = {d.name: t[i] for (i, d) in enumerate(self.tp.input_space)} kwargs2.update(kwargs) check_constraints = functools.partial(self.computer.evaluate_constraints, self.tp, inputs_only = False, kwargs = kwargs) cond = check_constraints(kwargs2) if (not cond): break if (size == 1): return accepted_configurations[0] else: return accepted_configurations #config_space = ConfigSpace.ConfigurationSpace() config_space = MyConstrainedConfigurationSpace(self.tp, self.computer, self.t) for n,p in zip(self.tp.parameter_space.dimension_names,self.tp.parameter_space.dimensions): if (isinstance(p, Real)): config_space.add_hyperparameter(ConfigSpace.UniformFloatHyperparameter(n, lower = p.bounds[0], upper = p.bounds[1])) elif (isinstance(p, Integer)): config_space.add_hyperparameter(ConfigSpace.UniformIntegerHyperparameter(n, lower = p.bounds[0], upper = p.bounds[1])) elif (isinstance(p, Categorical)): config_space.add_hyperparameter(ConfigSpace.CategoricalHyperparameter(n, choices = list(p.bounds))) else: raise Exception("Unknown parameter type") return(config_space)
def get_configspace(): """Builds the config space as described in the header docstring.""" cs = CS.ConfigurationSpace() lr = CS.UniformFloatHyperparameter('lr', lower=1e-5, upper=1e-2, default_value=1e-4, log=True) optimizer = CS.CategoricalHyperparameter('optimizer', ['adam', 'sgd']) momentum = CS.UniformFloatHyperparameter('momentum', lower=0., upper=1.00, default_value=0.9) epsilon = CS.UniformFloatHyperparameter('epsilon', lower=1e-2, upper=1., default_value=0.1) bs = CS.UniformIntegerHyperparameter('bs', lower=4, upper=256) first_layer = CS.UniformIntegerHyperparameter('first_layer', lower=16, upper=64) second_layer = CS.UniformIntegerHyperparameter('second_layer', lower=8, upper=64) leaky1 = CS.CategoricalHyperparameter('leaky1', [True, False]) leaky2 = CS.CategoricalHyperparameter('leaky2', [True, False]) leaky3 = CS.CategoricalHyperparameter('leaky3', [True, False]) cs.add_hyperparameters([ lr, optimizer, momentum, epsilon, bs, first_layer, second_layer, leaky1, leaky2, leaky3 ]) cs.add_condition(CS.EqualsCondition(momentum, optimizer, 'sgd')) cs.add_condition(CS.EqualsCondition(epsilon, optimizer, 'adam')) return cs