def test_uniformfloat_to_integer(self): f1 = UniformFloatHyperparameter("param", 1, 10, q=0.1, log=True) with pytest.warns(UserWarning, match="Setting quantization < 1 for Integer " "Hyperparameter 'param' has no effect"): f2 = f1.to_integer() # TODO is this a useful rounding? # TODO should there be any rounding, if e.g. lower=0.1 self.assertEqual("param, Type: UniformInteger, Range: [1, 10], " "Default: 3, on log-scale", str(f2))
def test_uniformfloat_to_integer(self): f1 = UniformFloatHyperparameter("param", 1, 10, q=0.1, log=True) with warnings.catch_warnings(): f2 = f1.to_integer() warnings.simplefilter("ignore") # TODO is this a useful rounding? # TODO should there be any rounding, if e.g. lower=0.1 self.assertEqual("param, Type: UniformInteger, Range: [1, 10], " "Default: 3, on log-scale", str(f2))
def test_log_space_conversion(self): lower, upper = 1e-5, 1e5 hyper = UniformFloatHyperparameter('test', lower=lower, upper=upper, log=True) self.assertTrue(hyper.is_legal(hyper._transform(1.))) lower, upper = 1e-10, 1e10 hyper = UniformFloatHyperparameter('test', lower=lower, upper=upper, log=True) self.assertTrue(hyper.is_legal(hyper._transform(1.)))
def NB_from_cfg(params): clf = nb(**params) return 1 - cross_val_score(clf, X, y, cv=5).mean() #logger = logging.getLogger("SVMExample") logging.basicConfig(level=logging.INFO) # logging.DEBUG for debug output # Build Configuration Space which defines all parameters and their ranges cs = ConfigurationSpace() # We define a few possible types of SVM-kernels and add them as "kernel" to our cs aplha = UniformFloatHyperparameter("alpha", 0.0, 5.0, default_value=1.0) fit_prior = CategoricalHyperparameter("fit_prior", [True, False], default_value=True) cs.add_hyperparameters([aplha, fit_prior]) # Scenario object scenario = Scenario({"run_obj": "quality", # we optimize quality (alternatively runtime) "runcount-limit": 500, # max. number of function evaluations; for this example set to a low number "cs": cs, # configuration space "deterministic": "true" }) # Example call of the function # It returns: Status, Cost, Runtime, Additional Infos def_value = NB_from_cfg(cs.get_default_configuration())
def test_uniformfloat_is_legal(self): lower = 0.1 upper = 10 f1 = UniformFloatHyperparameter("param", lower, upper, q=0.1, log=True) self.assertTrue(f1.is_legal(3.0)) self.assertTrue(f1.is_legal(3)) self.assertFalse(f1.is_legal(-0.1)) self.assertFalse(f1.is_legal(10.1)) self.assertFalse(f1.is_legal("AAA")) self.assertFalse(f1.is_legal(dict())) # Test legal vector values self.assertTrue(f1.is_legal_vector(1.0)) self.assertTrue(f1.is_legal_vector(0.0)) self.assertTrue(f1.is_legal_vector(0)) self.assertTrue(f1.is_legal_vector(0.3)) self.assertFalse(f1.is_legal_vector(-0.1)) self.assertFalse(f1.is_legal_vector(1.1)) self.assertRaises(TypeError, f1.is_legal_vector, "Hahaha")
# get instance data, target = generate_instances(int(instance[0]), int(instance[1])) cv = StratifiedKFold(n_splits=4, random_state=seed, shuffle=True) # to make CV splits consistent scores = cross_val_score(clf, data, target, cv=cv) return 1 - np.mean(scores) if __name__ == "__main__": # Build Configuration Space which defines all parameters and their ranges cs = ConfigurationSpace() # We define a few possible parameters for the SGD classifier alpha = UniformFloatHyperparameter( "alpha", 0, 1, default_value=1.0) l1_ratio = UniformFloatHyperparameter( "l1_ratio", 0, 1, default_value=0.5) learning_rate = CategoricalHyperparameter( "learning_rate", choices=['constant', 'invscaling', 'adaptive'], default_value='constant') eta0 = UniformFloatHyperparameter( "eta0", 0.00001, 1, default_value=0.1, log=True) # Add the parameters to configuration space cs.add_hyperparameters([alpha, l1_ratio, learning_rate, eta0]) # SMAC scenario object scenario = Scenario({ "run_obj": "quality", # we optimize quality (alternative to runtime) "wallclock-limit": 100, # max duration to run the optimization (in seconds) "cs": cs, # configuration space "deterministic": True,
kf = KFold(X.shape[0], n_folds=4) # build Configuration Space which defines all parameters and their ranges # to illustrate different parameter types, # we use continuous, integer and categorical parameters cs = ConfigurationSpace() do_bootstrapping = CategoricalHyperparameter("do_bootstrapping", ["true", "false"], default="true") cs.add_hyperparameter(do_bootstrapping) num_trees = UniformIntegerHyperparameter("num_trees", 10, 50, default=10) cs.add_hyperparameter(num_trees) frac_points_per_tree = UniformFloatHyperparameter("frac_points_per_tree", 0.001, 1, default=1) cs.add_hyperparameter(frac_points_per_tree) ratio_features = UniformFloatHyperparameter("ratio_features", 0.001, 1, default=1) cs.add_hyperparameter(ratio_features) min_samples_to_split = UniformIntegerHyperparameter("min_samples_to_split", 2, 20, default=2) cs.add_hyperparameter(min_samples_to_split)
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() loss = CategoricalHyperparameter("loss", ["least_squares"], default_value="least_squares") learning_rate = UniformFloatHyperparameter(name="learning_rate", lower=0.01, upper=1, default_value=0.1, log=True) max_iter = UniformIntegerHyperparameter("max_iter", 32, 512, default_value=100) min_samples_leaf = UniformIntegerHyperparameter( name="min_samples_leaf", lower=1, upper=200, default_value=20, log=True) max_depth = UnParametrizedHyperparameter(name="max_depth", value="None") max_leaf_nodes = UniformIntegerHyperparameter(name="max_leaf_nodes", lower=3, upper=2047, default_value=31, log=True) max_bins = Constant("max_bins", 256) l2_regularization = UniformFloatHyperparameter( name="l2_regularization", lower=1E-10, upper=1, default_value=1E-10, log=True) early_stop = CategoricalHyperparameter( name="early_stop", choices=["off", "train", "valid"], default_value="off") tol = UnParametrizedHyperparameter(name="tol", value=1e-7) scoring = UnParametrizedHyperparameter(name="scoring", value="loss") n_iter_no_change = UniformIntegerHyperparameter( name="n_iter_no_change", lower=1, upper=20, default_value=10) validation_fraction = UniformFloatHyperparameter( name="validation_fraction", lower=0.01, upper=0.4, default_value=0.1) cs.add_hyperparameters([ loss, learning_rate, max_iter, min_samples_leaf, max_depth, max_leaf_nodes, max_bins, l2_regularization, early_stop, tol, scoring, n_iter_no_change, validation_fraction ]) n_iter_no_change_cond = InCondition(n_iter_no_change, early_stop, ["valid", "train"]) validation_fraction_cond = EqualsCondition(validation_fraction, early_stop, "valid") cs.add_conditions([n_iter_no_change_cond, validation_fraction_cond]) return cs
def get_configspace(): cs = ConfigurationSpace() x1 = UniformFloatHyperparameter("x1", -5, 10, default_value=0) x2 = UniformFloatHyperparameter("x2", 0, 15, default_value=0) cs.add_hyperparameters([x1, x2]) return cs
def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, min_num_gropus: int = 1, max_num_groups: int = 9, min_blocks_per_groups: int = 1, max_blocks_per_groups: int = 4, min_num_units: int = 10, max_num_units: int = 1024, ) -> ConfigurationSpace: cs = ConfigurationSpace() # The number of groups that will compose the resnet. That is, # a group can have N Resblock. The M number of this N resblock # repetitions is num_groups num_groups = UniformIntegerHyperparameter( "num_groups", lower=min_num_gropus, upper=max_num_groups, default_value=5) activation = CategoricalHyperparameter( "activation", choices=list(_activations.keys()) ) cs.add_hyperparameters([num_groups, activation]) # We can have dropout in the network for # better generalization use_dropout = CategoricalHyperparameter( "use_dropout", choices=[True, False]) cs.add_hyperparameters([use_dropout]) use_shake_shake = CategoricalHyperparameter("use_shake_shake", choices=[True, False]) use_shake_drop = CategoricalHyperparameter("use_shake_drop", choices=[True, False]) shake_drop_prob = UniformFloatHyperparameter( "max_shake_drop_probability", lower=0.0, upper=1.0) cs.add_hyperparameters([use_shake_shake, use_shake_drop, shake_drop_prob]) cs.add_condition(CS.EqualsCondition(shake_drop_prob, use_shake_drop, True)) # It is the upper bound of the nr of groups, # since the configuration will actually be sampled. for i in range(0, max_num_groups + 1): n_units = UniformIntegerHyperparameter( "num_units_%d" % i, lower=min_num_units, upper=max_num_units, ) blocks_per_group = UniformIntegerHyperparameter( "blocks_per_group_%d" % i, lower=min_blocks_per_groups, upper=max_blocks_per_groups) cs.add_hyperparameters([n_units, blocks_per_group]) if i > 1: cs.add_condition(CS.GreaterThanCondition(n_units, num_groups, i - 1)) cs.add_condition(CS.GreaterThanCondition(blocks_per_group, num_groups, i - 1)) this_dropout = UniformFloatHyperparameter( "dropout_%d" % i, lower=0.0, upper=1.0 ) cs.add_hyperparameters([this_dropout]) dropout_condition_1 = CS.EqualsCondition(this_dropout, use_dropout, True) if i > 1: dropout_condition_2 = CS.GreaterThanCondition(this_dropout, num_groups, i - 1) cs.add_condition(CS.AndConjunction(dropout_condition_1, dropout_condition_2)) else: cs.add_condition(dropout_condition_1) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() loss = CategoricalHyperparameter("loss", [ "squared_loss", "huber", "epsilon_insensitive", "squared_epsilon_insensitive" ], default="squared_loss") penalty = CategoricalHyperparameter("penalty", ["l1", "l2", "elasticnet"], default="l2") alpha = UniformFloatHyperparameter("alpha", 10e-7, 1e-1, log=True, default=0.01) l1_ratio = UniformFloatHyperparameter("l1_ratio", 1e-9, 1., log=True, default=0.15) fit_intercept = UnParametrizedHyperparameter("fit_intercept", "True") n_iter = UniformIntegerHyperparameter("n_iter", 5, 1000, log=True, default=20) epsilon = UniformFloatHyperparameter("epsilon", 1e-5, 1e-1, default=1e-4, log=True) learning_rate = CategoricalHyperparameter( "learning_rate", ["optimal", "invscaling", "constant"], default="optimal") eta0 = UniformFloatHyperparameter("eta0", 10**-7, 0.1, default=0.01) power_t = UniformFloatHyperparameter("power_t", 1e-5, 1, default=0.5) average = CategoricalHyperparameter("average", ["False", "True"], default="False") cs.add_hyperparameters([ loss, penalty, alpha, l1_ratio, fit_intercept, n_iter, epsilon, learning_rate, eta0, power_t, average ]) # TODO add passive/aggressive here, although not properly documented? elasticnet = EqualsCondition(l1_ratio, penalty, "elasticnet") epsilon_condition = InCondition( epsilon, loss, ["huber", "epsilon_insensitive", "squared_epsilon_insensitive"]) # eta0 seems to be always active according to the source code; when # learning_rate is set to optimial, eta0 is the starting value: # https://github.com/scikit-learn/scikit-learn/blob/0.15.X/sklearn/linear_model/sgd_fast.pyx # eta0_and_inv = EqualsCondition(eta0, learning_rate, "invscaling") #eta0_and_constant = EqualsCondition(eta0, learning_rate, "constant") #eta0_condition = OrConjunction(eta0_and_inv, eta0_and_constant) power_t_condition = EqualsCondition(power_t, learning_rate, "invscaling") cs.add_conditions([elasticnet, epsilon_condition, power_t_condition]) return cs
def test_normalfloat_to_uniformfloat(self): f1 = NormalFloatHyperparameter("param", 0, 10, q=0.1) f1_expected = UniformFloatHyperparameter("param", -30, 30, q=0.1) f1_actual = f1.to_uniform() self.assertEqual(f1_expected, f1_actual)
def test_uniformfloat(self): # TODO test non-equality # TODO test sampling from a log-distribution which has a negative # lower value! f1 = UniformFloatHyperparameter("param", 0, 10) f1_ = UniformFloatHyperparameter("param", 0, 10) self.assertEqual(f1, f1_) self.assertEqual( "param, Type: UniformFloat, Range: [0.0, 10.0], " "Default: 5.0", str(f1)) # Test attributes are accessible self.assertEqual(f1.name, "param") self.assertAlmostEqual(f1.lower, 0.0) self.assertAlmostEqual(f1.upper, 10.0) self.assertEqual(f1.q, None) self.assertEqual(f1.log, False) self.assertAlmostEqual(f1.default_value, 5.0) self.assertAlmostEqual(f1.normalized_default_value, 0.5) f2 = UniformFloatHyperparameter("param", 0, 10, q=0.1) f2_ = UniformFloatHyperparameter("param", 0, 10, q=0.1) self.assertEqual(f2, f2_) self.assertEqual( "param, Type: UniformFloat, Range: [0.0, 10.0], " "Default: 5.0, Q: 0.1", str(f2)) f3 = UniformFloatHyperparameter("param", 0.00001, 10, log=True) f3_ = UniformFloatHyperparameter("param", 0.00001, 10, log=True) self.assertEqual(f3, f3_) self.assertEqual( "param, Type: UniformFloat, Range: [1e-05, 10.0], Default: 0.01, " "on log-scale", str(f3)) f4 = UniformFloatHyperparameter("param", 0, 10, default_value=1.0) f4_ = UniformFloatHyperparameter("param", 0, 10, default_value=1.0) # Test that a int default is converted to float f4__ = UniformFloatHyperparameter("param", 0, 10, default_value=1) self.assertEqual(f4, f4_) self.assertEqual(type(f4.default_value), type(f4__.default_value)) self.assertEqual( "param, Type: UniformFloat, Range: [0.0, 10.0], Default: 1.0", str(f4)) f5 = UniformFloatHyperparameter("param", 0.1, 10, q=0.1, log=True, default_value=1.0) f5_ = UniformFloatHyperparameter("param", 0.1, 10, q=0.1, log=True, default_value=1.0) self.assertEqual(f5, f5_) self.assertEqual( "param, Type: UniformFloat, Range: [0.1, 10.0], Default: 1.0, " "on log-scale, Q: 0.1", str(f5)) self.assertNotEqual(f1, f2) self.assertNotEqual(f1, "UniformFloat") # test that meta-data is stored correctly f_meta = UniformFloatHyperparameter("param", 0.1, 10, q=0.1, log=True, default_value=1.0, meta=dict(self.meta_data)) self.assertEqual(f_meta.meta, self.meta_data)
def test_uniformfloat_is_legal(self): lower = 0.1 upper = 10 f1 = UniformFloatHyperparameter("param", lower, upper, q=0.1, log=True) self.assertTrue(f1.is_legal(3.0)) self.assertTrue(f1.is_legal(3)) self.assertFalse(f1.is_legal(-0.1)) self.assertFalse(f1.is_legal(10.1)) self.assertFalse(f1.is_legal("AAA")) self.assertFalse(f1.is_legal(dict())) # Test legal vector values self.assertTrue(f1.is_legal_vector(1.0)) self.assertTrue(f1.is_legal_vector(0.0)) self.assertTrue(f1.is_legal_vector(0)) self.assertTrue(f1.is_legal_vector(0.3)) self.assertFalse(f1.is_legal_vector(-0.1)) self.assertFalse(f1.is_legal_vector(1.1)) self.assertRaises(TypeError, f1.is_legal_vector, "Hahaha")
def test_photon_implementation_simple(self): # PHOTON implementation self.pipe.add(PipelineElement('StandardScaler')) self.pipe += PipelineElement( 'PCA', hyperparameters={'n_components': IntegerRange(5, 30)}) self.pipe += PipelineElement('SVC', hyperparameters={ 'kernel': Categorical(["rbf", 'poly']), 'C': FloatRange(0.5, 200) }, gamma='auto') self.X, self.y = self.simple_classification() self.pipe.fit(self.X, self.y) # direct AUTO ML implementation # Build Configuration Space which defines all parameters and their ranges cs = ConfigurationSpace() n_components = UniformIntegerHyperparameter( "PCA__n_components", 5, 30) cs.add_hyperparameter(n_components) kernel = CategoricalHyperparameter("SVC__kernel", ["rbf", 'poly']) cs.add_hyperparameter(kernel) c = UniformFloatHyperparameter("SVC__C", 0.5, 200) cs.add_hyperparameter(c) # Scenario object scenario = Scenario({ "run_obj": "quality", "cs": cs, "deterministic": "true", "wallclock_limit": self.time_limit, "limit_resources": False, 'abort_on_first_run_crash': False }) # Optimize, using a SMAC directly smac = SMAC4HPO(scenario=scenario, rng=42, tae_runner=self.objective_function_simple) _ = smac.optimize() runhistory_photon = self.smac_helper["data"].solver.runhistory runhistory_original = smac.solver.runhistory x_ax = range( 1, min(len(runhistory_original._cost_per_config.keys()), len(runhistory_photon._cost_per_config.keys())) + 1) y_ax_original = [ runhistory_original._cost_per_config[tmp] for tmp in x_ax ] y_ax_photon = [ runhistory_photon._cost_per_config[tmp] for tmp in x_ax ] y_ax_original_inc = [min(y_ax_original[:tmp + 1]) for tmp in x_ax] y_ax_photon_inc = [min(y_ax_photon[:tmp + 1]) for tmp in x_ax] plot = False if plot: plt.figure(figsize=(10, 7)) plt.plot(x_ax, y_ax_original, 'g', label='Original') plt.plot(x_ax, y_ax_photon, 'b', label='PHOTON') plt.plot(x_ax, y_ax_photon_inc, 'r', label='PHOTON Incumbent') plt.plot(x_ax, y_ax_original_inc, 'k', label='Original Incumbent') plt.title('Photon Prove') plt.xlabel('X') plt.ylabel('Y') plt.legend(loc='best') plt.savefig("smac.png") min_len = min(len(y_ax_original), len(y_ax_photon)) self.assertLessEqual( np.max( np.abs( np.array(y_ax_original[:min_len]) - np.array(y_ax_photon[:min_len]))), 0.01)
def test_photon_implementation_switch(self): # PHOTON implementation self.pipe.add(PipelineElement('StandardScaler')) self.pipe += PipelineElement( 'PCA', hyperparameters={'n_components': IntegerRange(5, 30)}) estimator_siwtch = Switch("Estimator") estimator_siwtch += PipelineElement('SVC', hyperparameters={ 'kernel': Categorical( ["rbf", 'poly']), 'C': FloatRange(0.5, 200) }, gamma='auto') estimator_siwtch += PipelineElement('RandomForestClassifier', hyperparameters={ 'criterion': Categorical( ['gini', 'entropy']), 'min_samples_split': IntegerRange(2, 4) }) self.pipe += estimator_siwtch self.X, self.y = self.simple_classification() self.pipe.fit(self.X, self.y) # direct AUTO ML implementation # Build Configuration Space which defines all parameters and their ranges cs = ConfigurationSpace() n_components = UniformIntegerHyperparameter( "PCA__n_components", 5, 30) cs.add_hyperparameter(n_components) switch = CategoricalHyperparameter("Estimator_switch", ['svc', 'rf']) cs.add_hyperparameter(switch) kernel = CategoricalHyperparameter("SVC__kernel", ["rbf", 'poly']) cs.add_hyperparameter(kernel) c = UniformFloatHyperparameter("SVC__C", 0.5, 200) cs.add_hyperparameter(c) use_svc_c = InCondition(child=kernel, parent=switch, values=["svc"]) use_svc_kernel = InCondition(child=c, parent=switch, values=["svc"]) criterion = CategoricalHyperparameter( "RandomForestClassifier__criterion", ['gini', 'entropy']) cs.add_hyperparameter(criterion) minsplit = UniformIntegerHyperparameter( "RandomForestClassifier__min_samples_split", 2, 4) cs.add_hyperparameter(minsplit) use_rf_crit = InCondition(child=criterion, parent=switch, values=["rf"]) use_rf_minsplit = InCondition(child=minsplit, parent=switch, values=["rf"]) cs.add_conditions( [use_svc_c, use_svc_kernel, use_rf_crit, use_rf_minsplit]) # Scenario object scenario = Scenario({ "run_obj": "quality", "cs": cs, "deterministic": "true", "wallclock_limit": self.time_limit, "limit_resources": False, 'abort_on_first_run_crash': False }) # Optimize, using a SMAC directly smac = SMAC4HPO(scenario=scenario, rng=42, tae_runner=self.objective_function_switch) _ = smac.optimize() runhistory_photon = self.smac_helper["data"].solver.runhistory runhistory_original = smac.solver.runhistory x_ax = range( 1, min(len(runhistory_original._cost_per_config.keys()), len(runhistory_photon._cost_per_config.keys())) + 1) y_ax_original = [ runhistory_original._cost_per_config[tmp] for tmp in x_ax ] y_ax_photon = [ runhistory_photon._cost_per_config[tmp] for tmp in x_ax ] min_len = min(len(y_ax_original), len(y_ax_photon)) self.assertLessEqual( np.max( np.abs( np.array(y_ax_original[:min_len]) - np.array(y_ax_photon[:min_len]))), 0.01)
def test_against_smac(self): # PHOTON implementation self.pipe.add(PipelineElement('StandardScaler')) # then do feature selection using a PCA, specify which values to try in the hyperparameter search self.pipe += PipelineElement( 'PCA', hyperparameters={'n_components': IntegerRange(5, 30)}) # engage and optimize the good old SVM for Classification self.pipe += PipelineElement( 'SVC', hyperparameters={ 'kernel': Categorical(["linear", "rbf", 'poly', 'sigmoid']), 'C': FloatRange(0.5, 200) }, gamma='auto') self.X, self.y = self.simple_classification() self.pipe.fit(self.X, self.y) # AUTO ML direct # Build Configuration Space which defines all parameters and their ranges cs = ConfigurationSpace() # We define a few possible types of SVM-kernels and add them as "kernel" to our cs n_components = UniformIntegerHyperparameter("PCA__n_components", 5, 30) # , default_value=5) cs.add_hyperparameter(n_components) kernel = CategoricalHyperparameter( "SVC__kernel", ["linear", "rbf", 'poly', 'sigmoid']) #, default_value="linear") cs.add_hyperparameter(kernel) c = UniformFloatHyperparameter("SVC__C", 0.5, 200) #, default_value=1) cs.add_hyperparameter(c) # Scenario object scenario = Scenario({ "run_obj": "quality", # we optimize quality (alternatively runtime) "runcount-limit": 800, # maximum function evaluations "cs": cs, # configuration space "deterministic": "true", "shared_model": "false", # !!!! "wallclock_limit": self.time_limit }) # Optimize, using a SMAC-object print( "Optimizing! Depending on your machine, this might take a few minutes." ) smac = SMAC4BO(scenario=scenario, rng=np.random.RandomState(42), tae_runner=self.objective_function) self.traurig = smac incumbent = smac.optimize() inc_value = self.objective_function(incumbent) print(incumbent) print(inc_value) runhistory_photon = self.smac_helper["data"].solver.runhistory runhistory_original = smac.solver.runhistory x_ax = range( 1, min(len(runhistory_original.cost_per_config.keys()), len(runhistory_photon.cost_per_config.keys())) + 1) y_ax_original = [ runhistory_original.cost_per_config[tmp] for tmp in x_ax ] y_ax_photon = [runhistory_photon.cost_per_config[tmp] for tmp in x_ax] y_ax_original_inc = [min(y_ax_original[:tmp + 1]) for tmp in x_ax] y_ax_photon_inc = [min(y_ax_photon[:tmp + 1]) for tmp in x_ax] plt.figure(figsize=(10, 7)) plt.plot(x_ax, y_ax_original, 'g', label='Original') plt.plot(x_ax, y_ax_photon, 'b', label='PHOTON') plt.plot(x_ax, y_ax_photon_inc, 'r', label='PHOTON Incumbent') plt.plot(x_ax, y_ax_original_inc, 'k', label='Original Incumbent') plt.title('Photon Prove') plt.xlabel('X') plt.ylabel('Y') plt.legend(loc='best') plt.show() def neighbours(items, fill=None): before = itertools.chain([fill], items) after = itertools.chain( items, [fill]) # You could use itertools.zip_longest() later instead. next(after) for a, b, c in zip(before, items, after): yield [value for value in (a, b, c) if value is not fill] print("---------------") original_pairing = [ sum(values) / len(values) for values in neighbours(y_ax_original) ] bias_term = np.mean([ abs(y_ax_original_inc[t] - y_ax_photon_inc[t]) for t in range(len(y_ax_photon_inc)) ]) photon_pairing = [ sum(values) / len(values) - bias_term for values in neighbours(y_ax_photon) ] counter = 0 for i, x in enumerate(x_ax): if abs(original_pairing[i] - photon_pairing[i]) > 0.05: counter += 1 self.assertLessEqual(counter / len(x_ax), 0.15)
def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, min_num_gropus: int = 1, max_num_groups: int = 9, min_blocks_per_groups: int = 1, max_blocks_per_groups: int = 4, min_num_units: int = 10, max_num_units: int = 1024, ) -> ConfigurationSpace: cs = ConfigurationSpace() # Support for different shapes resnet_shape = CategoricalHyperparameter( 'resnet_shape', choices=[ 'funnel', 'long_funnel', 'diamond', 'hexagon', 'brick', 'triangle', 'stairs' ] ) cs.add_hyperparameter(resnet_shape) # The number of groups that will compose the resnet. That is, # a group can have N Resblock. The M number of this N resblock # repetitions is num_groups num_groups = UniformIntegerHyperparameter( "num_groups", lower=min_num_gropus, upper=max_num_groups, default_value=5) blocks_per_group = UniformIntegerHyperparameter( "blocks_per_group", lower=min_blocks_per_groups, upper=max_blocks_per_groups) activation = CategoricalHyperparameter( "activation", choices=list(_activations.keys()) ) output_dim = UniformIntegerHyperparameter( "output_dim", lower=min_num_units, upper=max_num_units ) cs.add_hyperparameters([num_groups, blocks_per_group, activation, output_dim]) # We can have dropout in the network for # better generalization use_dropout = CategoricalHyperparameter( "use_dropout", choices=[True, False]) cs.add_hyperparameters([use_dropout]) use_shake_shake = CategoricalHyperparameter("use_shake_shake", choices=[True, False]) use_shake_drop = CategoricalHyperparameter("use_shake_drop", choices=[True, False]) shake_drop_prob = UniformFloatHyperparameter( "max_shake_drop_probability", lower=0.0, upper=1.0) cs.add_hyperparameters([use_shake_shake, use_shake_drop, shake_drop_prob]) cs.add_condition(CS.EqualsCondition(shake_drop_prob, use_shake_drop, True)) max_units = UniformIntegerHyperparameter( "max_units", lower=min_num_units, upper=max_num_units, ) cs.add_hyperparameters([max_units]) max_dropout = UniformFloatHyperparameter( "max_dropout", lower=0.0, upper=1.0 ) cs.add_hyperparameters([max_dropout]) cs.add_condition(CS.EqualsCondition(max_dropout, use_dropout, True)) return cs
def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() # Parameterized Hyperparameters max_depth = UniformIntegerHyperparameter( name="max_depth", lower=1, upper=10, default_value=3 ) learning_rate = UniformFloatHyperparameter( name="learning_rate", lower=0.01, upper=1, default_value=0.1, log=True ) n_estimators = Constant("n_estimators", 512) booster = CategoricalHyperparameter( "booster", ["gbtree", "dart"] ) subsample = UniformFloatHyperparameter( name="subsample", lower=0.01, upper=1.0, default_value=1.0, log=False ) min_child_weight = UniformIntegerHyperparameter( name="min_child_weight", lower=1e-10, upper=20, default_value=1, log=False ) colsample_bytree = UniformFloatHyperparameter( name="colsample_bytree", lower=0.1, upper=1.0, default_value=1, ) colsample_bylevel = UniformFloatHyperparameter( name="colsample_bylevel", lower=0.1, upper=1.0, default_value=1, ) reg_alpha = UniformFloatHyperparameter( name="reg_alpha", lower=1e-10, upper=1e-1, log=True, default_value=1e-10) reg_lambda = UniformFloatHyperparameter( name="reg_lambda", lower=1e-10, upper=1e-1, log=True, default_value=1e-10) # DART Hyperparameters sample_type = CategoricalHyperparameter( 'sample_type', ['uniform', 'weighted'], default_value='uniform', ) normalize_type = CategoricalHyperparameter( 'normalize_type', ['tree', 'forest'], default_value='tree', ) rate_drop = UniformFloatHyperparameter( 'rate_drop', 1e-10, 1 - (1e-10), default_value=0.5, ) # Unparameterized Hyperparameters # https://xgboost.readthedocs.io/en/latest//parameter.html # minimum loss reduction required to make a further partition on a # leaf node of the tree gamma = UnParametrizedHyperparameter( name="gamma", value=0) # absolute regularization (in contrast to eta), comparable to # gradient clipping in deep learning - according to the internet this # is most important for unbalanced data max_delta_step = UnParametrizedHyperparameter( name="max_delta_step", value=0) base_score = UnParametrizedHyperparameter( name="base_score", value=0.5) scale_pos_weight = UnParametrizedHyperparameter( name="scale_pos_weight", value=1) cs.add_hyperparameters([ # Active max_depth, learning_rate, n_estimators, booster, subsample, colsample_bytree, colsample_bylevel, reg_alpha, reg_lambda, # DART sample_type, normalize_type, rate_drop, # Inactive min_child_weight, max_delta_step, gamma, base_score, scale_pos_weight ]) sample_type_condition = EqualsCondition( sample_type, booster, 'dart', ) normalize_type_condition = EqualsCondition( normalize_type, booster, 'dart', ) rate_drop_condition = EqualsCondition( rate_drop, booster, 'dart', ) cs.add_conditions([ sample_type_condition, normalize_type_condition, rate_drop_condition, ]) return cs
def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, min_mlp_layers: int = 1, max_mlp_layers: int = 15, dropout: bool = True, min_num_units: int = 10, max_num_units: int = 1024, ) -> ConfigurationSpace: cs = ConfigurationSpace() # The number of hidden layers the network will have. # Layer blocks are meant to have the same architecture, differing only # by the number of units num_groups = UniformIntegerHyperparameter( "num_groups", min_mlp_layers, max_mlp_layers, default_value=5) activation = CategoricalHyperparameter( "activation", choices=list(_activations.keys()) ) cs.add_hyperparameters([num_groups, activation]) # We can have dropout in the network for # better generalization if dropout: use_dropout = CategoricalHyperparameter( "use_dropout", choices=[True, False]) cs.add_hyperparameters([use_dropout]) for i in range(1, max_mlp_layers + 1): n_units_hp = UniformIntegerHyperparameter("num_units_%d" % i, lower=min_num_units, upper=max_num_units, default_value=20) cs.add_hyperparameter(n_units_hp) if i > min_mlp_layers: # The units of layer i should only exist # if there are at least i layers cs.add_condition( CS.GreaterThanCondition( n_units_hp, num_groups, i - 1 ) ) if dropout: dropout_hp = UniformFloatHyperparameter( "dropout_%d" % i, lower=0.0, upper=0.8, default_value=0.5 ) cs.add_hyperparameter(dropout_hp) dropout_condition_1 = CS.EqualsCondition(dropout_hp, use_dropout, True) if i > min_mlp_layers: dropout_condition_2 = CS.GreaterThanCondition(dropout_hp, num_groups, i - 1) cs.add_condition(CS.AndConjunction(dropout_condition_1, dropout_condition_2)) else: cs.add_condition(dropout_condition_1) return cs
dense_width = CategoricalHyperparameter("dense_width", [64, 128], default_value=128) dense_length = UniformIntegerHyperparameter("dense_length", 1, 3, default_value=2) optimizer = CategoricalHyperparameter("optimizer", ['adam', 'sgd', 'nadam', 'RMSprop'], default_value='RMSprop') optimizer_lr = CategoricalHyperparameter("optimizer_lr", [.0001, .0003, .001, .003, .01], default_value=.0003) learning_decay_rate = UniformFloatHyperparameter("learning_decay_rate", 0, 0.9, default_value=.6) cs.add_hyperparameters([ first_kernel_size, conv_filters, n_conv, dropout, activation, dense_width, dense_length, optimizer, optimizer_lr, learning_decay_rate ]) scenario = Scenario({ "run_obj": "quality", "runcount-limit": 128, "cs": cs, "deterministic": "true" }) scenario.output_dir_for_this_run = "C:\\NNwork\\SMAC3out" scenario.output_dir = "C:\\NNwork\\SMAC3out"
def get_hyperparameter_search_space(dataset_properties=None, optimizer='smac'): if optimizer == 'smac': cs = ConfigurationSpace() n_estimators = UniformIntegerHyperparameter(name="n_estimators", lower=50, upper=500, default_value=50, log=False) learning_rate = UniformFloatHyperparameter(name="learning_rate", lower=0.01, upper=2, default_value=0.1, log=True) algorithm = CategoricalHyperparameter(name="algorithm", choices=["SAMME.R", "SAMME"], default_value="SAMME.R") max_depth = UniformIntegerHyperparameter(name="max_depth", lower=1, upper=10, default_value=1, log=False) sampling_strategy = CategoricalHyperparameter( name="sampling_strategy", choices=["majority", "not minority", "not majority", "all"], default_value="not minority") replacement = CategoricalHyperparameter("replacement", ["True", "False"], default_value="False") cs.add_hyperparameters([ n_estimators, learning_rate, algorithm, max_depth, sampling_strategy, replacement ]) return cs elif optimizer == 'tpe': from hyperopt import hp space = { 'n_estimators': hp.randint('rb_n_estimators', 451) + 50, 'learning_rate': hp.loguniform('rb_learning_rate', np.log(0.01), np.log(2)), 'algorithm': hp.choice('rb_algorithm', ["SAMME.R", "SAMME"]), 'max_depth': hp.randint('rb_max_depth', 10) + 1, 'sampling_strategy': hp.choice('rb_sampling_strategy', ["majority", "not minority", "not majority", "all"]), 'replacement': hp.choice('rb_replacement', ["True", "False"]) } init_trial = { 'n_estimators': 50, 'learning_rate': 0.1, 'algorithm': "SAMME.R", 'max_depth': 1, 'sampling_strategy': "not minority", 'replacement': "False", } return space
def get_configspace_instance(algo_id='random_forest'): cs = ConfigurationSpace() if algo_id == 'random_forest': criterion = CategoricalHyperparameter("criterion", ["gini", "entropy"], default_value="gini") # The maximum number of features used in the forest is calculated as m^max_features, where # m is the total number of features, and max_features is the hyperparameter specified below. # The default is 0.5, which yields sqrt(m) features as max_features in the estimator. This # corresponds with Geurts' heuristic. max_features = UniformFloatHyperparameter("max_features", 0., 1., default_value=0.5) max_depth = UnParametrizedHyperparameter("max_depth", "None") min_samples_split = UniformIntegerHyperparameter("min_samples_split", 2, 20, default_value=2) min_samples_leaf = UniformIntegerHyperparameter("min_samples_leaf", 1, 20, default_value=1) min_weight_fraction_leaf = UnParametrizedHyperparameter( "min_weight_fraction_leaf", 0.) max_leaf_nodes = UnParametrizedHyperparameter("max_leaf_nodes", "None") min_impurity_decrease = UnParametrizedHyperparameter( 'min_impurity_decrease', 0.0) bootstrap = CategoricalHyperparameter("bootstrap", ["True", "False"], default_value="True") cs.add_hyperparameters([ criterion, max_features, max_depth, min_samples_split, min_samples_leaf, min_weight_fraction_leaf, max_leaf_nodes, bootstrap, min_impurity_decrease ]) elif algo_id == 'liblinear_svc': penalty = CategoricalHyperparameter("penalty", ["l1", "l2"], default_value="l2") loss = CategoricalHyperparameter("loss", ["hinge", "squared_hinge"], default_value="squared_hinge") dual = CategoricalHyperparameter("dual", ['True', 'False'], default_value='True') # This is set ad-hoc tol = UniformFloatHyperparameter("tol", 1e-5, 1e-1, default_value=1e-4, log=True) C = UniformFloatHyperparameter("C", 0.03125, 32768, log=True, default_value=1.0) multi_class = Constant("multi_class", "ovr") # These are set ad-hoc fit_intercept = Constant("fit_intercept", "True") intercept_scaling = Constant("intercept_scaling", 1) cs.add_hyperparameters([ penalty, loss, dual, tol, C, multi_class, fit_intercept, intercept_scaling ]) penalty_and_loss = ForbiddenAndConjunction( ForbiddenEqualsClause(penalty, "l1"), ForbiddenEqualsClause(loss, "hinge")) constant_penalty_and_loss = ForbiddenAndConjunction( ForbiddenEqualsClause(dual, "False"), ForbiddenEqualsClause(penalty, "l2"), ForbiddenEqualsClause(loss, "hinge")) penalty_and_dual = ForbiddenAndConjunction( ForbiddenEqualsClause(dual, "True"), ForbiddenEqualsClause(penalty, "l1")) cs.add_forbidden_clause(penalty_and_loss) cs.add_forbidden_clause(constant_penalty_and_loss) cs.add_forbidden_clause(penalty_and_dual) elif algo_id == 'lightgbm': n_estimators = UniformFloatHyperparameter("n_estimators", 100, 1000, default_value=500, q=50) num_leaves = UniformIntegerHyperparameter("num_leaves", 31, 2047, default_value=128) max_depth = Constant('max_depth', 15) learning_rate = UniformFloatHyperparameter("learning_rate", 1e-3, 0.3, default_value=0.1, log=True) min_child_samples = UniformIntegerHyperparameter("min_child_samples", 5, 30, default_value=20) subsample = UniformFloatHyperparameter("subsample", 0.7, 1, default_value=1, q=0.1) colsample_bytree = UniformFloatHyperparameter("colsample_bytree", 0.7, 1, default_value=1, q=0.1) cs.add_hyperparameters([ n_estimators, num_leaves, max_depth, learning_rate, min_child_samples, subsample, colsample_bytree ]) elif algo_id == 'adaboost': n_estimators = UniformIntegerHyperparameter(name="n_estimators", lower=50, upper=500, default_value=50, log=False) learning_rate = UniformFloatHyperparameter(name="learning_rate", lower=0.01, upper=2, default_value=0.1, log=True) algorithm = CategoricalHyperparameter(name="algorithm", choices=["SAMME.R", "SAMME"], default_value="SAMME.R") max_depth = UniformIntegerHyperparameter(name="max_depth", lower=2, upper=8, default_value=3, log=False) cs.add_hyperparameters( [n_estimators, learning_rate, algorithm, max_depth]) elif algo_id == 'lda': shrinkage = CategoricalHyperparameter("shrinkage", ["None", "auto", "manual"], default_value="None") shrinkage_factor = UniformFloatHyperparameter("shrinkage_factor", 0., 1., 0.5) n_components = UniformIntegerHyperparameter('n_components', 1, 250, default_value=10) tol = UniformFloatHyperparameter("tol", 1e-5, 1e-1, default_value=1e-4, log=True) cs.add_hyperparameters( [shrinkage, shrinkage_factor, n_components, tol]) cs.add_condition(EqualsCondition(shrinkage_factor, shrinkage, "manual")) elif algo_id == 'extra_trees': criterion = CategoricalHyperparameter("criterion", ["gini", "entropy"], default_value="gini") # The maximum number of features used in the forest is calculated as m^max_features, where # m is the total number of features, and max_features is the hyperparameter specified below. # The default is 0.5, which yields sqrt(m) features as max_features in the estimator. This # corresponds with Geurts' heuristic. max_features = UniformFloatHyperparameter("max_features", 0., 1., default_value=0.5) max_depth = UnParametrizedHyperparameter(name="max_depth", value="None") min_samples_split = UniformIntegerHyperparameter("min_samples_split", 2, 20, default_value=2) min_samples_leaf = UniformIntegerHyperparameter("min_samples_leaf", 1, 20, default_value=1) min_weight_fraction_leaf = UnParametrizedHyperparameter( 'min_weight_fraction_leaf', 0.) max_leaf_nodes = UnParametrizedHyperparameter("max_leaf_nodes", "None") min_impurity_decrease = UnParametrizedHyperparameter( 'min_impurity_decrease', 0.0) bootstrap = CategoricalHyperparameter("bootstrap", ["True", "False"], default_value="False") cs.add_hyperparameters([ criterion, max_features, max_depth, min_samples_split, min_samples_leaf, min_weight_fraction_leaf, max_leaf_nodes, min_impurity_decrease, bootstrap ]) elif algo_id == 'resnet': batch_size = UniformIntegerHyperparameter("train_batch_size", 32, 256, default_value=64, q=8) init_lr = UniformFloatHyperparameter('init_lr', lower=1e-3, upper=0.3, default_value=0.1, log=True) lr_decay_factor = UnParametrizedHyperparameter('lr_decay_factor', 0.1) weight_decay = UniformFloatHyperparameter('weight_decay', lower=1e-5, upper=1e-2, default_value=0.0002, log=True) momentum = UniformFloatHyperparameter("momentum", 0.5, .99, default_value=0.9) nesterov = CategoricalHyperparameter('nesterov', ['True', 'False'], default_value='True') cs.add_hyperparameters([ batch_size, init_lr, lr_decay_factor, weight_decay, momentum, nesterov ]) elif algo_id == 'nas': operation = 6 benchmark201_choices = [ 'none', 'skip_connect', 'nor_conv_1x1', 'nor_conv_3x3', 'avg_pool_3x3' ] for i in range(operation): cs.add_hyperparameter( CategoricalHyperparameter( 'op_%d' % i, choices=benchmark201_choices, default_value=benchmark201_choices[1])) return cs else: raise ValueError('Invalid algorithm - %s' % algo_id) return cs
import json import random import requests from ConfigSpace.read_and_write import json as config_json from ConfigSpace.hyperparameters import UniformFloatHyperparameter from openbox.config_space import ConfigurationSpace from openbox.config_space.util import convert_configurations_to_array user_id = 18 cs = ConfigurationSpace() x1 = UniformFloatHyperparameter("x1", -5, 10, default_value=0) x2 = UniformFloatHyperparameter("x2", 0, 15, default_value=0) cs.add_hyperparameters([x1, x2]) config_space_array = config_json.write(cs) res = requests.post('http://127.0.0.1:8001/bo_advice/task_register/', data={ 'id': user_id, 'config_space_array': config_space_array }) print('-----------------') print(res) print('-----------------') print(res.text) print('-----------------')
#logger = logging.getLogger("SVMExample") logging.basicConfig(level=logging.INFO) # logging.DEBUG for debug output # Build Configuration Space which defines all parameters and their ranges cs = ConfigurationSpace() # We define a few possible types of SVM-kernels and add them as "kernel" to our cs penalty = CategoricalHyperparameter("penalty", ["l1", "l2", "elasticnet", "none"], default_value="l2") dual = CategoricalHyperparameter("dual", [True, False], default_value=False) tol = UniformFloatHyperparameter("tol", 0.00001, 0.1, default_value=0.0001) C = UniformFloatHyperparameter("C", 0.0, 5.0, default_value=1.0) fit_intercept = CategoricalHyperparameter("fit_intercept", [True, False], default_value=False) intercept_scaling = UniformFloatHyperparameter("intercept_scaling", 0.0, 5.0, default_value=1.0) solver = CategoricalHyperparameter( "solver", ["newton-cg", "lbfgs", "liblinear", "sag", "saga"], default_value="liblinear")
def fmin_smac( func: typing.Callable, x0: typing.List[float], bounds: typing.List[typing.Iterable[float]], maxfun: int = -1, rng: typing.Union[np.random.RandomState, int] = None, scenario_args: typing.Mapping[str, typing.Any] = None, tae_runner_kwargs: typing.Optional[typing.Dict[str, typing.Any]] = None, **kwargs: typing.Any) -> typing.Tuple[Configuration, float, SMAC4HPO]: """ Minimize a function func using the SMAC4HPO facade (i.e., a modified version of SMAC). This function is a convenience wrapper for the SMAC4HPO class. Parameters ---------- func : typing.Callable Function to minimize. x0 : typing.List[float] Initial guess/default configuration. bounds : typing.List[typing.List[float]] ``(min, max)`` pairs for each element in ``x``, defining the bound on that parameters. maxfun : int, optional Maximum number of function evaluations. rng : np.random.RandomState, optional Random number generator used by SMAC. scenario_args: typing.Mapping[str,typing.Any] Arguments passed to the scenario See smac.scenario.scenario.Scenario **kwargs: Arguments passed to the optimizer class See ~smac.facade.smac_facade.SMAC Returns ------- x : list Estimated position of the minimum. f : float Value of `func` at the minimum. s : :class:`smac.facade.smac_hpo_facade.SMAC4HPO` SMAC objects which enables the user to get e.g., the trajectory and runhistory. """ # create configuration space cs = ConfigurationSpace() # Adjust zero padding tmplt = 'x{0:0' + str(len(str(len(bounds)))) + 'd}' for idx, (lower_bound, upper_bound) in enumerate(bounds): parameter = UniformFloatHyperparameter(name=tmplt.format(idx + 1), lower=lower_bound, upper=upper_bound, default_value=x0[idx]) cs.add_hyperparameter(parameter) # create scenario scenario_dict = { "run_obj": "quality", "cs": cs, "deterministic": "true", "initial_incumbent": "DEFAULT", } if scenario_args is not None: scenario_dict.update(scenario_args) if maxfun > 0: scenario_dict["runcount_limit"] = maxfun scenario = Scenario(scenario_dict) # Handle optional tae arguments if tae_runner_kwargs is not None: if 'ta' not in tae_runner_kwargs: tae_runner_kwargs.update({'ta': func}) else: tae_runner_kwargs = {'ta': func} smac = SMAC4HPO(scenario=scenario, tae_runner=ExecuteTAFuncArray, tae_runner_kwargs=tae_runner_kwargs, rng=rng, **kwargs) smac.logger = logging.getLogger(smac.__module__ + "." + smac.__class__.__name__) incumbent = smac.optimize() config_id = smac.solver.runhistory.config_ids[incumbent] run_key = RunKey(config_id, None, 0) incumbent_performance = smac.solver.runhistory.data[run_key] incumbent = np.array( [incumbent[tmplt.format(idx + 1)] for idx in range(len(bounds))], dtype=np.float) return incumbent, incumbent_performance.cost, smac
from mosaic.external.ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import CategoricalHyperparameter, \ UniformFloatHyperparameter, UniformIntegerHyperparameter from ConfigSpace.conditions import InCondition from mosaic.mosaic import Search # Build Configuration Space which defines all parameters and their ranges cs = ConfigurationSpace() # We define a few possible types of SVM-kernels and add them as "kernel" to our cs kernel = CategoricalHyperparameter("kernel", ["linear", "rbf", "poly", "sigmoid"], default_value="poly") cs.add_hyperparameter(kernel) # There are some hyperparameters shared by all kernels C = UniformFloatHyperparameter("C", 0.001, 1000.0, default_value=1.0) shrinking = CategoricalHyperparameter("shrinking", ["true", "false"], default_value="true") cs.add_hyperparameters([C, shrinking]) # Others are kernel-specific, so we can add conditions to limit the searchspace degree = UniformIntegerHyperparameter( "degree", 1, 5, default_value=3) # Only used by kernel poly coef0 = UniformFloatHyperparameter("coef0", 0.0, 10.0, default_value=0.0) # poly, sigmoid cs.add_hyperparameters([degree, coef0]) use_degree = InCondition(child=degree, parent=kernel, values=["poly"]) use_coef0 = InCondition(child=coef0, parent=kernel, values=["poly", "sigmoid"]) cs.add_conditions([use_degree, use_coef0]) # This also works for parameters that are a mix of categorical and values from a range of numbers # For example, gamma can be either "auto" or a fixed float gamma = CategoricalHyperparameter(
#!pip install smac --no-cache # Import ConfigSpace and different types of parameters from smac.configspace import ConfigurationSpace from ConfigSpace.hyperparameters import CategoricalHyperparameter, \ UniformFloatHyperparameter, UniformIntegerHyperparameter from ConfigSpace.conditions import InCondition # Import SMAC-utilities from smac.tae.execute_func import ExecuteTAFuncDict from smac.scenario.scenario import Scenario from smac.facade.smac_facade import SMAC cs = ConfigurationSpace() lr = UniformFloatHyperparameter("lr", 0.0001, 0.1, default_value=0.001) cs.add_hyperparameter(lr) batch_size = CategoricalHyperparameter("batch_size", [128, 256], default_value=128) cs.add_hyperparameter(batch_size) # Scenario object scenario = Scenario({"run_obj": "quality", # we optimize quality (alternatively runtime) "runcount-limit": 5, # maximum function evaluations "cs": cs, # configuration space "deterministic": "true", #"abort_on_first_run_crash": "false" }) # Optimize, using a SMAC-object print("Optimizing! Depending on your machine, this might take a few minutes.")
from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import UniformFloatHyperparameter, \ UniformIntegerHyperparameter, CategoricalHyperparameter, \ UnParametrizedHyperparameter, Constant from automl.utl import json_utils C = UniformFloatHyperparameter("C", 1e-5, 10, 1.0, log=True) fit_intercept = UnParametrizedHyperparameter("fit_intercept", "True") loss = CategoricalHyperparameter("loss", ["hinge", "squared_hinge"], default_value="hinge") tol = UniformFloatHyperparameter("tol", 1e-5, 1e-1, default_value=1e-4, log=True) # Note: Average could also be an Integer if > 1 average = CategoricalHyperparameter('average', ['False', 'True'], default_value='False') cs = ConfigurationSpace() cs.add_hyperparameters([loss, fit_intercept, tol, C, average]) json_utils.write_cs_to_json_file(cs, "PassiveAggressiveClassifier")
from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import UniformFloatHyperparameter, \ UniformIntegerHyperparameter, CategoricalHyperparameter, \ UnParametrizedHyperparameter, Constant from automl.utl import json_utils cs = ConfigurationSpace() # the smoothing parameter is a non-negative float # I will limit it to 1000 and put it on a logarithmic scale. (SF) # Please adjust that, if you know a proper range, this is just a guess. alpha = UniformFloatHyperparameter(name="alpha", lower=1e-2, upper=100, default_value=1, log=True) fit_prior = CategoricalHyperparameter(name="fit_prior", choices=["True", "False"], default_value="True") cs.add_hyperparameters([alpha, fit_prior]) json_utils.write_cs_to_json_file(cs, "BernoulliNB")
def get_cs(): cs = ConfigurationSpace() root = CategoricalHyperparameter("root", choices=["l1", "ln"]) x1 = CategoricalHyperparameter("x1", choices=["l1", "ln"]) x2 = CategoricalHyperparameter("x2", choices=["l1", "ln"]) x3 = CategoricalHyperparameter("x3", choices=["l1", "ln"]) x4 = CategoricalHyperparameter("x4", choices=["l1", "ln"]) x5 = CategoricalHyperparameter("x5", choices=["l1", "ln"]) x6 = CategoricalHyperparameter("x6", choices=["l1", "ln"]) # r1 is the data associated in x1 r1_1 = UniformFloatHyperparameter("r1_1", lower=0.01, upper=0.99, log=False) r1_2 = UniformFloatHyperparameter("r1_2", lower=0.01, upper=0.99, log=False) r1_3 = UniformFloatHyperparameter("r1_3", lower=0.01, upper=0.99, log=False) r2_1 = UniformFloatHyperparameter("r2_1", lower=0.01, upper=0.99, log=False) r2_2 = UniformFloatHyperparameter("r2_2", lower=0.01, upper=0.99, log=False) r2_3 = UniformFloatHyperparameter("r2_3", lower=0.01, upper=0.99, log=False) r3_1 = UniformFloatHyperparameter("r3_1", lower=0.01, upper=0.99, log=False) r3_2 = UniformFloatHyperparameter("r3_2", lower=0.01, upper=0.99, log=False) r3_3 = Constant("r3_3", 0.5) r4_1 = UniformFloatHyperparameter("r4_1", lower=0.01, upper=0.99, log=False) r4_2 = UniformFloatHyperparameter("r4_2", lower=0.01, upper=0.99, log=False) r4_3 = UniformFloatHyperparameter("r4_3", lower=0.01, upper=0.99, log=False) r5_1 = UniformFloatHyperparameter("r5_1", lower=0.01, upper=0.99, log=False) r5_2 = UniformFloatHyperparameter("r5_2", lower=0.01, upper=0.99, log=False) r5_3 = UniformFloatHyperparameter("r5_3", lower=0.01, upper=0.99, log=False) r6_1 = UniformFloatHyperparameter("r6_1", lower=0.01, upper=0.99, log=False) r6_2 = UniformFloatHyperparameter("r6_2", lower=0.01, upper=0.99, log=False) r6_3 = UniformFloatHyperparameter("r6_3", lower=0.01, upper=0.99, log=False) r7_1 = UniformFloatHyperparameter("r7_1", lower=0.01, upper=0.99, log=False) r7_2 = UniformFloatHyperparameter("r7_2", lower=0.01, upper=0.99, log=False) r7_3 = Constant("r7_3", 0.5) r8_1 = UniformFloatHyperparameter("r8_1", lower=0.01, upper=0.99, log=False) r8_2 = UniformFloatHyperparameter("r8_2", lower=0.01, upper=0.99, log=False) r8_3 = Constant("r8_3", 0.5) r9_1 = UniformFloatHyperparameter("r9_1", lower=0.01, upper=0.99, log=False) r9_2 = UniformFloatHyperparameter("r9_2", lower=0.01, upper=0.99, log=False) r9_3 = Constant("r9_3", 0.5) r10_1 = UniformFloatHyperparameter("r10_1", lower=0.01, upper=0.99, log=False) r10_2 = UniformFloatHyperparameter("r10_2", lower=0.01, upper=0.99, log=False) r10_3 = Constant("r10_3", 0.5) r11_1 = UniformFloatHyperparameter("r11_1", lower=0.01, upper=0.99, log=False) r11_2 = UniformFloatHyperparameter("r11_2", lower=0.01, upper=0.99, log=False) r11_3 = Constant("r11_3", 0.5) r12_1 = UniformFloatHyperparameter("r12_1", lower=0.01, upper=0.99, log=False) r12_2 = UniformFloatHyperparameter("r12_2", lower=0.01, upper=0.99, log=False) r12_3 = Constant("r12_3", 0.5) r13_1 = UniformFloatHyperparameter("r13_1", lower=0.01, upper=0.99, log=False) r13_2 = UniformFloatHyperparameter("r13_2", lower=0.01, upper=0.99, log=False) r13_3 = Constant("r13_3", 0.5) r14_1 = UniformFloatHyperparameter("r14_1", lower=0.01, upper=0.99, log=False) r14_2 = UniformFloatHyperparameter("r14_2", lower=0.01, upper=0.99, log=False) r14_3 = Constant("r14_3", 0.5) cs.add_hyperparameters([ root, x1, x2, x3, x4, x5, x6, r1_1, r1_2, r1_3, r2_1, r2_2, r2_3, r3_1, r3_2, r3_3, r4_1, r4_2, r4_3, r5_1, r5_2, r5_3, r6_1, r6_2, r6_3, r7_1, r7_2, r7_3, r8_1, r8_2, r8_3, r9_1, r9_2, r9_3, r10_1, r10_2, r10_3, r11_1, r11_2, r11_3, r12_1, r12_2, r12_3, r13_1, r13_2, r13_3, r14_1, r14_2, r14_3, ]) # add condition cs.add_condition(InCondition(x1, root, ["l1"])) cs.add_condition(InCondition(x2, root, ["ln"])) cs.add_condition(InCondition(r1_1, root, ["l1"])) cs.add_condition(InCondition(r1_2, root, ["l1"])) cs.add_condition(InCondition(r1_3, root, ["l1"])) cs.add_condition(InCondition(r2_1, root, ["ln"])) cs.add_condition(InCondition(r2_2, root, ["ln"])) cs.add_condition(InCondition(r2_3, root, ["ln"])) cs.add_condition(InCondition(x3, x1, ["l1"])) cs.add_condition(InCondition(x4, x1, ["ln"])) cs.add_condition(InCondition(r3_1, x1, ["l1"])) cs.add_condition(InCondition(r3_2, x1, ["l1"])) cs.add_condition(InCondition(r3_3, x1, ["l1"])) cs.add_condition(InCondition(r4_1, x1, ["ln"])) cs.add_condition(InCondition(r4_2, x1, ["ln"])) cs.add_condition(InCondition(r4_3, x1, ["ln"])) cs.add_condition(InCondition(x5, x2, ["l1"])) cs.add_condition(InCondition(x6, x2, ["ln"])) cs.add_condition(InCondition(r5_1, x2, ["l1"])) cs.add_condition(InCondition(r5_2, x2, ["l1"])) cs.add_condition(InCondition(r5_3, x2, ["l1"])) cs.add_condition(InCondition(r6_1, x2, ["ln"])) cs.add_condition(InCondition(r6_2, x2, ["ln"])) cs.add_condition(InCondition(r6_3, x2, ["ln"])) cs.add_condition(InCondition(r7_1, x3, ["l1"])) cs.add_condition(InCondition(r7_2, x3, ["l1"])) cs.add_condition(InCondition(r7_3, x3, ["l1"])) cs.add_condition(InCondition(r8_1, x3, ["ln"])) cs.add_condition(InCondition(r8_2, x3, ["ln"])) cs.add_condition(InCondition(r8_3, x3, ["ln"])) cs.add_condition(InCondition(r9_1, x4, ["l1"])) cs.add_condition(InCondition(r9_2, x4, ["l1"])) cs.add_condition(InCondition(r9_3, x4, ["l1"])) cs.add_condition(InCondition(r10_1, x4, ["ln"])) cs.add_condition(InCondition(r10_2, x4, ["ln"])) cs.add_condition(InCondition(r10_3, x4, ["ln"])) cs.add_condition(InCondition(r11_1, x5, ["l1"])) cs.add_condition(InCondition(r11_2, x5, ["l1"])) cs.add_condition(InCondition(r11_3, x5, ["l1"])) cs.add_condition(InCondition(r12_1, x5, ["ln"])) cs.add_condition(InCondition(r12_2, x5, ["ln"])) cs.add_condition(InCondition(r12_3, x5, ["ln"])) cs.add_condition(InCondition(r13_1, x6, ["l1"])) cs.add_condition(InCondition(r13_2, x6, ["l1"])) cs.add_condition(InCondition(r13_3, x6, ["l1"])) cs.add_condition(InCondition(r14_1, x6, ["ln"])) cs.add_condition(InCondition(r14_2, x6, ["ln"])) cs.add_condition(InCondition(r14_3, x6, ["ln"])) return cs
def dqn_bohb_wrapper(**params): # Setup directories where live data is logged logdir = params["logdir"] dqn_output_dir = os.path.join(logdir, 'dqn_output') # if not os.path.isdir(dqn_output_dir): # os.makedirs(dqn_output_dir) params["logdir"] = dqn_output_dir bohb_output_dir = os.path.join(logdir, 'bohb_output') # if not os.path.isdir(bohb_output_dir): # os.makedirs(bohb_output_dir) logging.basicConfig(level=logging.INFO) # logging.DEBUG for debug output logger = logging.getLogger() logger.propagate = False # no duplicate logging outputs fh = logging.FileHandler(os.path.join(logdir, 'bohb.log')) fh.setLevel(logging.INFO) fh.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s:%(name)s: %(message)s')) logger.addHandler(fh) # Build configuration space and define all hyperparameters cs = ConfigurationSpace() epsilon = UniformFloatHyperparameter("epsilon", 0.2, 0.9, default_value=0.6) # initial epsilon epsilon_decay = UniformFloatHyperparameter("epsilon_decay", 0.2, 1, default_value=0.995) # decay rate lr = UniformFloatHyperparameter("lr", 0.0005, 0.01, default_value=0.005) units_shared_layer1 = UniformIntegerHyperparameter("units_layer1", 8, 100, default_value=24) units_shared_layer2 = UniformIntegerHyperparameter("units_layer2", 8, 100, default_value=24) units_policy_layer = UniformIntegerHyperparameter("units_layer3", 8, 100, default_value=24) activ_fcn = CategoricalHyperparameter("activ_fcn", ['relu6', 'elu', 'mixed'], default_value='relu6') gamma = UniformFloatHyperparameter("gamma", 0.6, 0.90, default_value=0.80) tau = UniformFloatHyperparameter("tau", 0.5, 1., default_value=0.7) # update_interval = UniformIntegerHyperparameter("update_interval", 1, 300, default_value=50) if params["architecture"] == 'lstm' or (params["architecture"] == 'gru'): trace_length = UniformIntegerHyperparameter("trace_length", 1, 20, default_value=8) # buffer_condition = LessThanCondition(child=trace_length, parent=params["buffer_size"]) # pa["batch_size"] = 5 cs.add_hyperparameters([units_shared_layer1, units_shared_layer2, units_policy_layer, epsilon, epsilon_decay, activ_fcn, lr, gamma, tau, trace_length]) else: params.pop("batch_size") batch_size = UniformIntegerHyperparameter("batch_size", 1, 100, default_value=30) # buffer_condition = LessThanCondition(child=batch_size, parent=params["buffer_size"], value=33) # InCondition(child=batch_size, value=33) cs.add_hyperparameters([units_shared_layer1, units_shared_layer2, units_policy_layer, epsilon, epsilon_decay, activ_fcn, lr, gamma, tau, batch_size]) logger.info('##############################################') logger.info('Run Optimization') logger.info('##############################################') if params["array_id"] == 1: # Setup directories where live data is logged # logdir = params["logdir"] # dqn_output_dir = os.path.join(logdir, 'dqn_output') if not os.path.isdir(dqn_output_dir): os.makedirs(dqn_output_dir) # params["logdir"] = dqn_output_dir # bohb_output_dir = os.path.join(logdir, 'bohb_output') if not os.path.isdir(bohb_output_dir): os.makedirs(bohb_output_dir) # start nameserver NS = hpns.NameServer(run_id=params["instance_id"], nic_name=params["nic_name"], working_directory=bohb_output_dir) ns_host, ns_port = NS.start() # stores information for workers to find in working directory # BOHB is usually so cheap, that we can affort to run a worker on the master node, too. worker = DQNWorker(nameserver=ns_host, nameserver_port=ns_port, run_id=params["instance_id"], **params) worker.run(background=True) # Create scenario object logger.info('##############################################') logger.info('Setup BOHB instance') logger.info('##############################################') logger.info('Output_dir: %s' % bohb_output_dir) HB = BOHB(configspace=cs, run_id=params["instance_id"], eta=3, min_budget=params["min_resource"], max_budget=params["max_resource"], host=ns_host, nameserver=ns_host, nameserver_port=ns_port, ping_interval=3600) res = HB.run(n_iterations=4, min_n_workers=4) # BOHB can wait until a minimum number of workers is online before starting # pickle result here for later analysis with open(os.path.join(bohb_output_dir, 'results.pkl'), 'wb') as f: pickle.dump(res, f) id2config = res.get_id2config_mapping() print('A total of %i unique configurations where sampled.' % len(id2config.keys())) print('A total of %i runs where executed.' % len(res.get_all_runs())) # incumbent_trajectory = res.get_incumbent_trajectory() # import matplotlib.pyplot as plt # plt.plot(incumbent_trajectory['times_finished'], incumbent_trajectory['losses']) # plt.xlabel('wall clock time [s]') # plt.ylabel('incumbent loss') # plt.show() # shutdown all workers HB.shutdown(shutdown_workers=True) # shutdown nameserver NS.shutdown() else: host = hpns.nic_name_to_host(params["nic_name"]) # workers only instantiate the MyWorker, find the nameserver and start serving w = DQNWorker(run_id=params["instance_id"], host=host, **params) w.load_nameserver_credentials(bohb_output_dir) # run worker in the forground, w.run(background=False)
dimensions. The minimium is always at x_i = 1 with a function value of zero. All input parameters are continuous. The search domain for all x's is the interval [-5, 10]. """ x1 = x["x0"] x2 = x["x1"] val = 100. * (x2 - x1**2.)**2. + (1 - x1)**2. return val if __name__ == "__main__": # Build Configuration Space which defines all parameters and their ranges cs = ConfigurationSpace() x0 = UniformFloatHyperparameter("x0", -5, 10, default_value=-3) x1 = UniformFloatHyperparameter("x1", -5, 10, default_value=-4) cs.add_hyperparameters([x0, x1]) # Scenario object scenario = Scenario({ "run_obj": "quality", # we optimize quality (alternatively runtime) "runcount-limit": 10, # max. number of function evaluations "cs": cs, # configuration space "deterministic": True }) # Use 'gp' or 'gp_mcmc' here model_type = 'gp' # Example call of the function
plt.ylabel("Accuracy (%)") plt.title("Validation Accuracy vs Number of Iterations: Bagging") for label, x, y in zip(hyper_parameters, iterations, accuracies): plt.text(x,y,label,fontsize=8,horizontalalignment='center',verticalalignment='center') save_file_name = "Accuracy_vs_Number_of_Iterations_Bagging.png" plt.savefig(save_file_name) #plt.show() # In[ ]: cs = ConfigurationSpace() C = UniformFloatHyperparameter("C", 0.001, 1000.0, default_value=1.0) cs.add_hyperparameter(C) scenario = Scenario({"run_obj": "quality", # we optimize quality (alternatively runtime) "runcount-limit": 50, # maximum function evaluations "cs": cs, # configuration space "deterministic": "true" }) # In[ ]: smac = SMAC(scenario=scenario, rng=np.random.RandomState(42), tae_runner=svm_from_cfg) incumbent = smac.optimize() print(incumbent)