def addSearchSpaceGrid( hp: SearchSpaceGrid, disc: int, parent_disc: Hyperparameter, cs: ConfigurationSpace ) -> None: smac = SearchSpaceGridtoSMAC(hp, disc) for hyp in smac: cs.add_hyperparameter(hyp) cs.add_condition(EqualsCondition(child=hyp, parent=parent_disc, value=disc))
def optimize(): # We load the iris-dataset (a widely used benchmark) iris = datasets.load_iris() #logger = logging.getLogger("SVMExample") logging.basicConfig(level=logging.INFO) # logging.DEBUG for debug output # Build Configuration Space which defines all parameters and their ranges cs = ConfigurationSpace() # We define a few possible types of SVM-kernels and add them as "kernel" to our cs kernel = CategoricalHyperparameter("kernel", ["linear", "rbf", "poly", "sigmoid"], default="poly") cs.add_hyperparameter(kernel) # There are some hyperparameters shared by all kernels C = UniformFloatHyperparameter("C", 0.001, 1000.0, default=1.0) shrinking = CategoricalHyperparameter("shrinking", ["true", "false"], default="true") cs.add_hyperparameters([C, shrinking]) # Others are kernel-specific, so we can add conditions to limit the searchspace degree = UniformIntegerHyperparameter("degree", 1, 5, default=3) # Only used by kernel poly coef0 = UniformFloatHyperparameter("coef0", 0.0, 10.0, default=0.0) # poly, sigmoid cs.add_hyperparameters([degree, coef0]) use_degree = InCondition(child=degree, parent=kernel, values=["poly"]) use_coef0 = InCondition(child=coef0, parent=kernel, values=["poly", "sigmoid"]) cs.add_conditions([use_degree, use_coef0]) # This also works for parameters that are a mix of categorical and values from a range of numbers # For example, gamma can be either "auto" or a fixed float gamma = CategoricalHyperparameter("gamma", ["auto", "value"], default="auto") # only rbf, poly, sigmoid gamma_value = UniformFloatHyperparameter("gamma_value", 0.0001, 8, default=1) cs.add_hyperparameters([gamma, gamma_value]) # We only activate gamma_value if gamma is set to "value" cs.add_condition(InCondition(child=gamma_value, parent=gamma, values=["value"])) # And again we can restrict the use of gamma in general to the choice of the kernel cs.add_condition(InCondition(child=gamma, parent=kernel, values=["rbf", "poly", "sigmoid"])) # Scenario object scenario = Scenario("test/test_files/svm_scenario.txt") # Example call of the function # It returns: Status, Cost, Runtime, Additional Infos def_value = svm_from_cfg(cs.get_default_configuration()) print("Default Value: %.2f" % (def_value)) # Optimize, using a SMAC-object print("Optimizing! Depending on your machine, this might take a few minutes.") smac = SMAC(scenario=scenario, rng=np.random.RandomState(42), tae_runner=svm_from_cfg) incumbent = smac.optimize() inc_value = svm_from_cfg(incumbent) print("Optimized Value: %.2f" % (inc_value))
def main_loop(problem): logging.basicConfig(level=logging.INFO) # logging.DEBUG for debug output cs = ConfigurationSpace() n_neighbors = UniformIntegerHyperparameter("n_neighbors", 2,10, default_value=5) cs.add_hyperparameter(n_neighbors) weights = CategoricalHyperparameter("weights", ["uniform","distance"], default_value="uniform") algorithm = CategoricalHyperparameter("algorithm", ["ball_tree", "kd_tree","brute","auto"], default_value="auto") cs.add_hyperparameters([weights, algorithm]) leaf_size = UniformIntegerHyperparameter("leaf_size", 1, 100, default_value=50) cs.add_hyperparameter(leaf_size) use_leaf_size= InCondition(child=leaf_size, parent=algorithm, values=["ball_tree","kd_tree"]) cs.add_condition(use_leaf_size) p = UniformIntegerHyperparameter("p", 1,3, default_value=2) cs.add_hyperparameter(p) # Scenario object max_eval=100000 scenario = Scenario({"run_obj": "quality", # we optimize quality (alternatively runtime) "runcount-limit": max_eval, # maximum function evaluations "cs": cs, # configuration space "shared_model": True, "output_dir": "/home/naamah/Documents/CatES/result_All/smac/KNN/run_{}_{}_{}".format(max_eval, datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d_%H:%M:%S'), problem) # "output_dir": "/home/naamah/Documents/CatES/result_All/smac/KNN/{}/run_{}_{}".format(problem,max_eval, datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d_%H:%M:%S_%f')), # "input_psmac_dirs":"/home/naamah/Documents/CatES/result_All/", # "deterministic": "true" }) # Example call of the function # It returns: Status, Cost, Runtime, Additional Infos def_value = svm_from_cfg(cs.get_default_configuration()) print("Default Value: %.2f" % (def_value)) # Optimize, using a SMAC-object print("Optimizing! Depending on your machine, this might take a few minutes.") smac = SMAC(scenario=scenario,tae_runner=svm_from_cfg) incumbent = smac.optimize() inc_value = svm_from_cfg(incumbent) print("Optimized Value: %.2f" % (inc_value)) return (incumbent) # main_loop()
def get_variables(problem): cs = ConfigurationSpace() params = [] for i in range(problem.dims()): param = get_variable(problem, i) params.append(param) cs.add_hyperparameter(param) for (i, c) in enumerate(problem.dependencies()): if c is None: continue j = c['on'] s = c['values'] cs.add_condition(InCondition(params[i], params[j], list(s))) return cs
def getPCS(self): ''' maxIter: [1,100]最大迭代次数,默认50 regParam :[0,0.2] 正则化参数,默认0 tol:[1e-6,1e-1] 迭代算法收敛性,默认 1e-6 family ,link, variancePower 对应关系 • “gaussian” -> “identity”, “log”, “inverse” • “binomial” -> “logit”, “probit”, “cloglog” • “poisson” -> “log”, “identity”, “sqrt” • “gamma” -> “inverse”, “identity”, “log” • “tweedie” -> power link function specified through “linkPower”. The default link power in the tweedie family is 1 - variancePower. ''' # Build Configuration Space which defines all parameters and their # ranges cs = ConfigurationSpace() maxIter = UniformIntegerHyperparameter("maxIter", 1, 100, default_value=50) regParam = UniformFloatHyperparameter("regParam", 0, 0.4, default_value=1e-04) tol = UniformFloatHyperparameter("tol", 1e-06, 1e-01, default_value=1e-06) family = CategoricalHyperparameter("family", ["gaussian", "poisson"], default_value="gaussian") gaussianLink = CategoricalHyperparameter( "gaussianLink", ["identity", "log", "inverse"], default_value="identity") poissonLink = CategoricalHyperparameter("poissonLink", ["log", "identity", "sqrt"], default_value="log") cs.add_hyperparameters( [maxIter, regParam, tol, family, gaussianLink, poissonLink]) cs.add_condition( InCondition(child=gaussianLink, parent=family, values=["gaussian"])) cs.add_condition( InCondition(child=poissonLink, parent=family, values=["poisson"])) return cs
def build_config_space(clustering_ls=["KMeans", "DBSCAN"], dim_reduction_ls=[]): cs = ConfigurationSpace() if len(clustering_ls) > 0: clustering_choice = CategoricalHyperparameter( "clustering_choice", clustering_ls, default_value=clustering_ls[0]) cs.add_hyperparameters([clustering_choice]) if len(dim_reduction_ls) > 0: dim_reduction_choice = CategoricalHyperparameter( "dim_reduction_choice", dim_reduction_ls, default_value=dim_reduction_ls[0]) cs.add_hyperparameters([dim_reduction_choice]) for idx, string in enumerate( itertools.chain(clustering_ls, dim_reduction_ls)): algorithm = Mapper.getClass(string) # encode parameter names encoded_params = [] for param in algorithm.params: encoded_string = StringUtils.encode_parameter( param.name, algorithm.name) param.name = encoded_string # add encoded paramters to configuration space cs.add_hyperparameters(algorithm.params) # define dependency for param in algorithm.params: cs.add_condition( InCondition(child=param, parent=clustering_choice if idx < len(clustering_ls) else dim_reduction_choice, values=[string])) # add forbidden clauses for condition in algorithm.forbidden_clauses: cs.add_forbidden_clause(condition) return cs
def build_config_space(clustering_ls: List[str] = None, feature_selection_ls: List[str] = None): if clustering_ls is None: clustering_ls = ["KMeans", "DBSCAN"] if feature_selection_ls is None: feature_selection_ls = ["NullModel", "NormalizedCut"] assert len(clustering_ls) > 0 assert len(feature_selection_ls) > 0 cs = ConfigurationSpace() clustering_choice = CategoricalHyperparameter( "clustering_choice", clustering_ls, default_value=clustering_ls[0]) feature_selection_choice = CategoricalHyperparameter( "feature_selection_choice", feature_selection_ls, default_value=feature_selection_ls[0]) cs.add_hyperparameters([clustering_choice]) cs.add_hyperparameters([feature_selection_choice]) for idx, string in enumerate( itertools.chain(clustering_ls, feature_selection_ls)): algorithm = Mapper.get_class(string) for param in algorithm.params: encoded_string = encode_parameter(param.name, algorithm.name) param.name = encoded_string cs.add_hyperparameters(algorithm.params) for param in algorithm.params: cs.add_condition( InCondition( child=param, parent=clustering_choice if idx < len(clustering_ls) else feature_selection_choice, values=[string])) for condition in algorithm.forbidden_clauses: cs.add_forbidden_clause(condition) return cs
def config_space(self): """KNN hyperparameter space.""" n_neighbors = UniformIntegerHyperparameter( 'n_neighbors', lower=1, upper=100, default_value=5 ) leaf_size = UniformIntegerHyperparameter( 'leaf_size', lower=1, upper=100, default_value=20 ) metric = CategoricalHyperparameter( 'metric', ['euclidean', 'manhattan', 'chebyshev', 'minkowski'], default_value='euclidean' ) p_param = UniformIntegerHyperparameter('p', 1, 5, default_value=2) # Add hyperparameters to config space. config = ConfigurationSpace() config.seed(self.random_state) config.add_hyperparameters((n_neighbors, leaf_size, metric, p_param)) # Conditionals on hyperparameters specific to kernels. config.add_condition( InCondition(child=p_param, parent=metric, values=['minkowski']) ) return config
def test_impute_inactive_hyperparameters(self): cs = ConfigurationSpace() a = cs.add_hyperparameter(CategoricalHyperparameter('a', [0, 1])) b = cs.add_hyperparameter(CategoricalHyperparameter('b', [0, 1])) c = cs.add_hyperparameter(UniformFloatHyperparameter('c', 0, 1)) cs.add_condition(EqualsCondition(b, a, 1)) cs.add_condition(EqualsCondition(c, a, 0)) cs.seed(1) configs = cs.sample_configuration(size=100) config_array = convert_configurations_to_array(configs) for line in config_array: if line[0] == 0: self.assertTrue(np.isnan(line[1])) elif line[0] == 1: self.assertTrue(np.isnan(line[2])) gp = get_gp(3, np.random.RandomState(1)) config_array = gp._impute_inactive(config_array) for line in config_array: if line[0] == 0: self.assertEqual(line[1], -1) elif line[0] == 1: self.assertEqual(line[2], -1)
def make_cs(): cs = ConfigurationSpace() cs.add_hyperparameter( UniformIntegerHyperparameter("n_estimators", 1, 30, default=10)) max_features = CategoricalHyperparameter('max_features', ['auto', 'value'], default='auto') max_features_value = UniformFloatHyperparameter('max_features_value', .1, 1) cs.add_hyperparameters([max_features, max_features_value]) cs.add_condition( InCondition(child=max_features_value, parent=max_features, values=['value'])) max_depth = CategoricalHyperparameter('max_depth', [None, 'value']) max_depth_value = UniformIntegerHyperparameter("max_depth_value", 1, 10) cs.add_hyperparameters([max_depth, max_depth_value]) cs.add_condition( InCondition(child=max_depth_value, parent=max_depth, values=['value'])) min_samples_split = UniformFloatHyperparameter("min_samples_split", .1, 1) cs.add_hyperparameter(min_samples_split) min_samples_leaf = UniformFloatHyperparameter("min_samples_leaf", .1, .5) cs.add_hyperparameter(min_samples_leaf) min_weight_fraction_leaf = UniformFloatHyperparameter( "min_weight_fraction_leaf", 0, .5) cs.add_hyperparameter(min_weight_fraction_leaf) max_leaf_nodes = CategoricalHyperparameter('max_leaf_nodes', [None, 'value']) max_leaf_nodes_value = UniformIntegerHyperparameter( 'max_leaf_nodes_value', 2, 100) cs.add_hyperparameters([max_leaf_nodes, max_leaf_nodes_value]) cs.add_condition( InCondition(child=max_leaf_nodes_value, parent=max_leaf_nodes, values=['value'])) min_impurity_split = UniformFloatHyperparameter('min_impurity_split', 0, 1) cs.add_hyperparameter(min_impurity_split) bootstrap = CategoricalHyperparameter('bootstrap', [True, False], default=True) cs.add_hyperparameter(bootstrap)
class SMACOptimizer(PhotonBaseOptimizer): def __init__(self, run_obj="quality", wallclock_limit: float = float("inf"), min_r=1, max_r=1, run_limit=5, smac_helper=None ): # optimizer only for test and developing settings yet. if not __found__: raise ModuleNotFoundError( "Module smac not found or not installed as expected. " "Please install the smac_requirements.txt PHOTON provides.") self.wallclock_limit = wallclock_limit self.minR = min_r self.maxR = max_r if self.minR != 1 or self.maxR != 1: raise NotImplementedError( "PHOTONs seed management is not implemented yet. " "At this juncture we can not run a config multiple times correctly." ) self.cspace = ConfigurationSpace() # Hyperparameter space for smac self.runtime = 0 self.run_start = 0 self.run_obj = run_obj self.run_limit = run_limit self.challengers = [] self.old_challengers = None self.ask_list = [] self.switch_optiones = {} self.hyperparameters = [] self.budget_exhausted = False self.constant_dictionary = {} self.smac_helper = smac_helper self.maximize_metric = False @staticmethod def _convert_photon_to_smac_space(hyperparam: object, name: str): """ Helper function: Convert PHOTON hyperparams to smac params. """ if not hyperparam: return None if isinstance(hyperparam, PhotonCategorical): return CategoricalHyperparameter(name, hyperparam.values) elif isinstance(hyperparam, list): return CategoricalHyperparameter(name, hyperparam) elif isinstance(hyperparam, FloatRange): if hyperparam.range_type == 'linspace': return UniformFloatHyperparameter(name, hyperparam.start, hyperparam.stop) elif hyperparam.range_type == 'logspace': raise NotImplementedError( "Logspace in your float hyperparameter is not implemented in SMAC." ) else: return UniformFloatHyperparameter(name, hyperparam.start, hyperparam.stop) elif isinstance(hyperparam, IntegerRange): return UniformIntegerHyperparameter(name, hyperparam.start, hyperparam.stop) def build_smac_space(self, pipeline_elements): """ Build entire smac hyperparam space. """ for pipe_element in pipeline_elements: # build conditions for switch elements # maybe this creepy condition is so that we avoid circular imports! if pipe_element.__class__.__name__ == 'Switch': algorithm_options = {} for algo in pipe_element.elements: algo_params = [] # hyper params corresponding to "algo" for name, value in algo.hyperparameters.items(): smac_param = self._convert_photon_to_smac_space( value, (pipe_element.name + "__" + name)) # or element.name__algo.name__name ??? algo_params.append(smac_param) algorithm_options[(pipe_element.name + "__" + algo.name)] = algo_params algos = CategoricalHyperparameter( name=pipe_element.name + "__algos", choices=algorithm_options.keys()) self.switch_optiones[pipe_element.name + "__algos"] = algorithm_options.keys() self.cspace.add_hyperparameter(algos) for algo, params in algorithm_options.items(): for param in params: cond = InCondition(child=param, parent=algos, values=[algo]) self.cspace.add_hyperparameter(param) self.cspace.add_condition(cond) continue if hasattr(pipe_element, 'hyperparameters'): for name, value in pipe_element.hyperparameters.items(): self.hyperparameters.append(name) # if we only have one value we do not need to optimize if isinstance(value, list) and len(value) < 2: self.constant_dictionary[name] = value[0] continue if isinstance(value, PhotonCategorical) and len(value.values) < 2: self.constant_dictionary[name] = value.values[0] continue smac_param = self._convert_photon_to_smac_space( value, name) if smac_param is not None: self.cspace.add_hyperparameter(smac_param) def prepare(self, pipeline_elements: list, maximize_metric: bool): # build space self.space = ConfigurationSpace() self.build_smac_space(pipeline_elements) self.maximize_metric = maximize_metric self.scenario = Scenario({ "run_obj": self.run_obj, "cs": self.cspace, "deterministic": "true", "wallclock_limit": self.wallclock_limit }) self.smac = SMAC4BO(scenario=self.scenario, rng=np.random.RandomState(42), tae_runner=MyExecuteTARun) if self.smac_helper: self.smac_helper["data"] = self.smac self.optimizer = self.smac.solver self.optimizer.runhistory.overwrite_existing_runs = True self.ask = self.ask_generator() def tell(self, config, performance): if not config: return config = copy.copy(config) for key in self.switch_optiones.keys(): config[key] = [ x for x in self.switch_optiones[key] if any(x in val for val in config.keys()) ][0] if self.maximize_metric: performance = 1 - performance[1] config = Configuration(self.cspace, values=config) self.optimizer.stats.ta_runs += 1 # first incubment setting if self.runtime == 0: self.optimizer.incumbent = Configuration( self.cspace, values=config) #self.challengers[0]) self.optimizer.start() else: if self.optimizer.stats.is_budget_exhausted(): self.budget_exhausted = True self.optimizer.runhistory.add(config=config, cost=performance, time=0, status=StatusType.SUCCESS, seed=0) self.runtime += 1 def ask_generator(self): def init(): if self.runtime == 0: ta_run = MyExecuteTARun(run_limit=self.run_limit, runhistory=self.optimizer.runhistory) self.optimizer.intensifier = Intensifier( tae_runner=ta_run, stats=self.optimizer.stats, traj_logger=None, rng=np.random.RandomState(42), instances=self.scenario. train_insts, #list(self.optimizer.runhistory.ids_config.keys()), minR=1, maxR=1, min_chall=self.scenario.intens_min_chall, instance_specifics=self.scenario.instance_specific) # , # run_limit=self.run_limit) #self.optimizer.stats.start_timing() self.old_challengers = self.optimizer.initial_design.select_configurations( ) #a = self.optimizer.initial_design.select_configurations()[0].get_dictionary() #tmp_dict = x.get_dictionary() self.ask_list = [{ key: x.get_dictionary()[key] for key in x.get_dictionary().keys() if 'algos' not in key } for x in self.optimizer.initial_design.select_configurations( )] self.smac_helper = len(self.ask_list) print("-----SMAC INIT READY----") else: start_time = time.time() X, Y = self.optimizer.rh2EPM.transform( self.optimizer.runhistory) self.optimizer.logger.debug("Search for next configuration") # get all found configurations sorted according to acq challengers = self.optimizer.choose_next(X, Y) time_spent = time.time() - start_time time_left = self.optimizer._get_timebound_for_intensification( time_spent) # first run challenger vs. any other run if isinstance(challengers, list): self.challengers = challengers[:self.run_limit] else: self.challengers = challengers.challengers[:self.run_limit] self.ask_list = [{ key: x.get_dictionary()[key] for key in x.get_dictionary().keys() if 'algos' not in key } for x in self.challengers] if self.old_challengers: self.incumbent, inc_perf = self.optimizer.intensifier.\ intensify(challengers=self.old_challengers, incumbent=self.optimizer.incumbent, run_history=self.optimizer.runhistory, aggregate_func=self.optimizer.aggregate_func, log_traj=False, time_bound=max(self.optimizer.intensifier._min_time, time_left)) self.optimizer.incumbent = self.incumbent self.old_challengers = self.challengers return 0 i = init() while True: if self.budget_exhausted: val = yield {} return else: val = (yield self.ask_list[i]) if len(self.ask_list) - 1 == i: i = init() else: i += 1
use_degree = InCondition(child=degree, parent=kernel, values=["poly"]) use_coef0 = InCondition(child=coef0, parent=kernel, values=["poly", "sigmoid"]) cs.add_conditions([use_degree, use_coef0]) # This also works for parameters that are a mix of categorical and values from a range of numbers # For example, gamma can be either "auto" or a fixed float gamma = CategoricalHyperparameter( "gamma", ["auto", "value"], default_value="auto") # only rbf, poly, sigmoid gamma_value = UniformFloatHyperparameter("gamma_value", 0.0001, 8, default_value=1) cs.add_hyperparameters([gamma, gamma_value]) # We only activate gamma_value if gamma is set to "value" cs.add_condition(InCondition(child=gamma_value, parent=gamma, values=["value"])) # And again we can restrict the use of gamma in general to the choice of the kernel cs.add_condition( InCondition(child=gamma, parent=kernel, values=["rbf", "poly", "sigmoid"])) # Scenario object scenario = Scenario({ "run_obj": "quality", # we optimize quality (alternatively runtime) "runcount-limit": 200, # maximum function evaluations "cs": cs, # configuration space "deterministic": "true" }) # Example call of the function # It returns: Status, Cost, Runtime, Additional Infos def_value = svm_from_cfg(cs.get_default_configuration())
def cs_single(): # Build Configuration Space which defines all parameters and their ranges cs = ConfigurationSpace() root = CategoricalHyperparameter("root", choices=["l1", "ln"]) x1 = CategoricalHyperparameter("x1", choices=["l1", "ln"]) x2 = CategoricalHyperparameter("x2", choices=["l1", "ln"]) x3 = CategoricalHyperparameter("x3", choices=["l1", "ln"]) x4 = CategoricalHyperparameter("x4", choices=["l1", "ln"]) x5 = CategoricalHyperparameter("x5", choices=["l1", "ln"]) x6 = CategoricalHyperparameter("x6", choices=["l1", "ln"]) # r1 is the data associated in x1 r1 = UniformFloatHyperparameter("r1", lower=0.01, upper=0.99, log=False) r2 = UniformFloatHyperparameter("r2", lower=0.01, upper=0.99, log=False) r3 = UniformFloatHyperparameter("r3", lower=0.01, upper=0.99, log=False) r4 = UniformFloatHyperparameter("r4", lower=0.01, upper=0.99, log=False) r5 = UniformFloatHyperparameter("r5", lower=0.01, upper=0.99, log=False) r6 = UniformFloatHyperparameter("r6", lower=0.01, upper=0.99, log=False) r7 = UniformFloatHyperparameter("r7", lower=0.01, upper=0.99, log=False) r8 = UniformFloatHyperparameter("r8", lower=0.01, upper=0.99, log=False) r9 = UniformFloatHyperparameter("r9", lower=0.01, upper=0.99, log=False) r10 = UniformFloatHyperparameter("r10", lower=0.01, upper=0.99, log=False) r11 = UniformFloatHyperparameter("r11", lower=0.01, upper=0.99, log=False) r12 = UniformFloatHyperparameter("r12", lower=0.01, upper=0.99, log=False) r13 = UniformFloatHyperparameter("r13", lower=0.01, upper=0.99, log=False) r14 = UniformFloatHyperparameter("r14", lower=0.01, upper=0.99, log=False) cs.add_hyperparameters([ root, x1, x2, x3, x4, x5, x6, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12, r13, r14, ]) # add condition cs.add_condition(InCondition(x1, root, ["l1"])) cs.add_condition(InCondition(x2, root, ["ln"])) cs.add_condition(InCondition(r1, root, ["l1"])) cs.add_condition(InCondition(r2, root, ["ln"])) cs.add_condition(InCondition(x3, x1, ["l1"])) cs.add_condition(InCondition(x4, x1, ["ln"])) cs.add_condition(InCondition(r3, x1, ["l1"])) cs.add_condition(InCondition(r4, x1, ["ln"])) cs.add_condition(InCondition(x5, x2, ["l1"])) cs.add_condition(InCondition(x6, x2, ["ln"])) cs.add_condition(InCondition(r5, x2, ["l1"])) cs.add_condition(InCondition(r6, x2, ["ln"])) cs.add_condition(InCondition(r7, x3, ["l1"])) cs.add_condition(InCondition(r8, x3, ["ln"])) cs.add_condition(InCondition(r9, x4, ["l1"])) cs.add_condition(InCondition(r10, x4, ["ln"])) cs.add_condition(InCondition(r11, x5, ["l1"])) cs.add_condition(InCondition(r12, x5, ["ln"])) cs.add_condition(InCondition(r13, x6, ["l1"])) cs.add_condition(InCondition(r14, x6, ["ln"])) return cs
def bayesianmcmc(self): from sklearn.decomposition import PCA from sklearn.manifold import Isomap from sklearn import svm from sklearn.ensemble import RandomForestClassifier from mahotas.features import haralick import os from sklearn.model_selection import StratifiedKFold, train_test_split from sklearn import metrics from sklearn.preprocessing import StandardScaler import cv2 from sklearn.neighbors import KNeighborsClassifier def naive_all_features(names): f = [] for i in range(len(names)): I = cv2.imread(names[i]) l = I.shape f1 = [] if I is None or I.size == 0 or np.sum( I[:]) == 0 or I.shape[0] == 0 or I.shape[1] == 0: if len(l) == 3: f1 = np.zeros((1, l[0] * l[1] * l[2])) else: f1 = I.flatten() f1 = np.expand_dims(f1, 0) if i == 0: f = f1 else: f = np.vstack((f, f1)) return f def haralick_all_features(X, distance=1): f = [] for i in range(len(X)): I = cv2.imread(X[i]) if I is None or I.size == 0 or np.sum( I[:]) == 0 or I.shape[0] == 0 or I.shape[1] == 0: h = np.zeros((1, 13)) else: I = cv2.cvtColor(I, cv2.COLOR_BGR2GRAY) h = haralick(I, distance=distance, return_mean=True, ignore_zeros=False) h = np.expand_dims(h, 0) if i == 0: f = h else: f = np.vstack((f, h)) return f def CNN_all_features(names, cnn): from keras.applications.vgg19 import VGG19 from keras.applications.inception_v3 import InceptionV3 from keras.applications.vgg19 import preprocess_input f = [] if cnn == 'VGG': model = VGG19(weights='imagenet') dsize = (224, 224) else: model = InceptionV3(weights='imagenet') dsize = (299, 299) for i in range(len(names)): img = cv2.imread(names[i]) img = cv2.resize(img, dsize=dsize) img = img.astype('float32') x = np.expand_dims(img, axis=0) x = preprocess_input(x) features = model.predict(x) if i == 0: f = features else: f = np.vstack((f, features)) return f def VGG_all_features(names, X): home = os.path.expanduser('~') if os.path.exists(self.data_loc + 'features/bayesian1/VGG_' + self.data_name + '.npz'): f = np.load( open( self.data_loc + 'features/bayesian1/VGG_' + self.data_name + '.npz', 'rb')) return f.f.arr_0[X, :] else: f = CNN_all_features(names, 'VGG') np.savez( open( self.data_loc + 'features/bayesian1/VGG_' + self.data_name + '.npz', 'wb'), f) return f[X, :] def inception_all_features(names, X): home = os.path.expanduser('~') if os.path.exists(self.data_loc + 'features/bayesian1/inception_' + self.data_name + '.npz'): f = np.load( open( self.data_loc + 'features/bayesian1/inception_' + self.data_name + '.npz', 'rb')) return f.f.arr_0[X, :] else: f = CNN_all_features(names, 'inception') np.savez( open( self.data_loc + 'features/bayesian1/inception_' + self.data_name + '.npz', 'wb'), f) return f[X, :] def principal_components(X, whiten=True): pca = PCA(whiten=whiten) maxvar = 0.95 X = np.asarray(X) if len(X.shape) == 1: X = X.reshape(-1, 1) data = X X1 = pca.fit(X) var = pca.explained_variance_ratio_ s1 = 0 for i in range(len(var)): s1 += var[i] s = 0 for i in range(len(var)): s += var[i] if (s * 1.0 / s1) >= maxvar: break pca = PCA(n_components=i + 1) pca.fit(data) return pca def isomap(X, n_neighbors=5, n_components=2): iso = Isomap(n_components=n_components, n_neighbors=n_neighbors) X = np.asarray(X) if len(X.shape) == 1: X = X.reshape(-1, 1) iso.fit(X) return iso def random_forests(X, y, n_estimators, max_features): clf = RandomForestClassifier(n_estimators=n_estimators, max_features=max_features, class_weight='balanced') clf.fit(X, y) return clf def support_vector_machines(X, y, C, gamma): clf = svm.SVC(C=C, gamma=gamma, class_weight='balanced', probability=True) clf.fit(X, y) return clf def knn(X, y, neighbors=1): clf = KNeighborsClassifier(n_neighbors=neighbors) clf.fit(X, y) return clf def pipeline_from_cfg(cfg): cfg = {k: cfg[k] for k in cfg if cfg[k]} # Load the data data_home = self.data_loc + 'datasets/' + self.data_name + '/' l1 = os.listdir(data_home) y = [] names = [] cnt = 0 for z in range(len(l1)): if l1[z][0] == '.': continue l = os.listdir(data_home + l1[z] + '/') y += [cnt] * len(l) cnt += 1 for i in range(len(l)): names.append(data_home + l1[z] + '/' + l[i]) # Train val split X = np.empty((len(y), 1)) indices = np.arange(len(y)) X1, _, y1, y_val, id1, _ = train_test_split(X, y, indices, test_size=0.2, random_state=42, shuffle=True) s = [] val_splits = 3 kf = StratifiedKFold(n_splits=val_splits, random_state=42, shuffle=True) names1 = [] for i in range(len(id1)): names1.append((names[id1[i]])) f11 = [] for idx1, idx2 in kf.split(X1, y1): # Feature extraction ids1 = [] X_train = [] y_train = [] for i in idx1: X_train.append(names1[i]) y_train.append(y1[i]) ids1.append(id1[i]) X_val = [] y_val = [] ids2 = [] for i in idx2: X_val.append(names1[i]) y_val.append(y1[i]) ids2.append(id1[i]) # Feature extraction f_train = [] # f_test = [] f_val = [] if cfg['feature_extraction'] == "haralick": f_val = haralick_all_features(X_val, cfg["haralick_distance"]) f_train = haralick_all_features(X_train, cfg["haralick_distance"]) elif cfg['feature_extraction'] == "VGG": f_val = VGG_all_features(names, ids2) f_train = VGG_all_features(names, ids1) elif cfg['feature_extraction'] == "inception": f_val = inception_all_features(names, ids2) f_train = inception_all_features(names, ids1) elif cfg['feature_extraction'] == "naive_feature_extraction": f_val = naive_all_features(X_val) f_train = naive_all_features(X_train) # Dimensionality reduction if cfg['dimensionality_reduction'] == "PCA": cfg["pca_whiten"] = True if cfg[ "pca_whiten"] == "true" else False dr = principal_components(f_train, cfg["pca_whiten"]) f_train = dr.transform(f_train) f_val = dr.transform(f_val) elif cfg['dimensionality_reduction'] == "ISOMAP": dr = isomap(f_train, cfg["n_neighbors"], cfg["n_components"]) f_train = dr.transform(f_train) f_val = dr.transform(f_val) elif cfg[ 'dimensionality_reduction'] == 'naive_dimensionality_reduction': f_train = f_train f_val = f_val # Pre-processing normalizer = StandardScaler().fit(f_train) f_train = normalizer.transform(f_train) f_val = normalizer.transform(f_val) # Learning algorithms if cfg['learning_algorithm'] == "RF": clf = random_forests(f_train, y_train, cfg["n_estimators"], cfg["max_features"]) elif cfg['learning_algorithm'] == "SVM": clf = support_vector_machines(f_train, y_train, cfg["svm_C"], cfg["svm_gamma"]) elif cfg['learning_algorithm'] == 'naive_learning_algorithm': clf = knn(f_train, y_train) p_pred = clf.predict_proba(f_val) f11.append(metrics.log_loss(y_val, p_pred)) s.append(clf.score(f_val, y_val)) return np.mean(f11) self.potential = [] self.best_pipelines = [] self.times = [] self.error_curves = [] cs = ConfigurationSpace() feature_extraction = CategoricalHyperparameter( "feature_extraction", ["haralick", "VGG", "inception"], default="haralick") cs.add_hyperparameter(feature_extraction) dimensionality_reduction = CategoricalHyperparameter( "dimensionality_reduction", ["PCA", "ISOMAP"], default="PCA") cs.add_hyperparameter(dimensionality_reduction) learning_algorithm = CategoricalHyperparameter("learning_algorithm", ["SVM", "RF"], default="RF") cs.add_hyperparameter(learning_algorithm) haralick_distance = UniformIntegerHyperparameter("haralick_distance", 1, 3, default=1) cs.add_hyperparameter(haralick_distance) cond1 = InCondition(child=haralick_distance, parent=feature_extraction, values=["haralick"]) cs.add_condition(cond1) pca_whiten = CategoricalHyperparameter("pca_whiten", ["true", "false"], default="true") cs.add_hyperparameter(pca_whiten) cs.add_condition( InCondition(child=pca_whiten, parent=dimensionality_reduction, values=["PCA"])) n_neighbors = UniformIntegerHyperparameter("n_neighbors", 3, 7, default=5) n_components = UniformIntegerHyperparameter("n_components", 2, 4, default=2) cs.add_hyperparameters([n_neighbors, n_components]) cs.add_condition( InCondition(child=n_components, parent=dimensionality_reduction, values=["ISOMAP"])) cs.add_condition( InCondition(child=n_neighbors, parent=dimensionality_reduction, values=["ISOMAP"])) svm_C = UniformFloatHyperparameter("svm_C", 0.1, 100.0, default=1.0) cs.add_hyperparameter(svm_C) svm_gamma = UniformFloatHyperparameter("svm_gamma", 0.01, 8, default=1) cs.add_hyperparameter(svm_gamma) cond1 = InCondition(child=svm_C, parent=learning_algorithm, values=["SVM"]) cond2 = InCondition(child=svm_gamma, parent=learning_algorithm, values=["SVM"]) cs.add_conditions([cond1, cond2]) n_estimators = UniformIntegerHyperparameter("n_estimators", 8, 300, default=10) max_features = UniformFloatHyperparameter("max_features", 0.3, 0.8, default=0.5) cs.add_hyperparameters([max_features, n_estimators]) cond1 = InCondition(child=n_estimators, parent=learning_algorithm, values=["RF"]) cond2 = InCondition(child=max_features, parent=learning_algorithm, values=["RF"]) cs.add_conditions([cond1, cond2]) scenario = Scenario({ "run_obj": "quality", "cutoff_time": 100000, "runcount_limit": 10000 * 10, "cs": cs, "maxR": 100000, "wallclock_limit": 1000000, "deterministic": "true" }) smac = SMAC(scenario=scenario, rng=np.random.RandomState(42), tae_runner=pipeline_from_cfg) incumbent, incs, incumbents, incumbents1, times = smac.optimize() inc_value = pipeline_from_cfg(incumbent) self.best_pipelines.append(incumbent) self.potential.append(inc_value) self.incumbents = incumbents self.all_incumbents = incumbents1 self.error_curves.append(incs) self.times = times pickle.dump( self, open( self.results_loc + 'intermediate/SMAC/SMAC_' + self.data_name + '_run_' + str(self.run) + '_full.pkl', 'wb'))
class SMACOptimizer(PhotonBaseOptimizer): def __init__( self, cutoff_time: float = float("inf"), run_obj: str = "quality", runcount_limit: float = float("inf"), tuner_timeout: float = float("inf"), wallclock_limit: float = float("inf"), ): self.optimizer = None self.scenario = None self.hyperparameter_list = [] self.metric_to_optimize = "" if ( run_obj == "runtime" and cutoff_time == float("inf") ) or run_obj == "quality": cutoff_time = None self.ask = self.ask_generator() self.runcount_limit = ( runcount_limit ) # should there be something more? Refer - https://automl.github.io/SMAC3/stable/options.html self.tuner_timeout = tuner_timeout self.wallclock_limit = wallclock_limit self.run_obj = run_obj self.cutoff_time = cutoff_time # self.maximize_metric = True self.memory = {} # stores config : cost self.test = 0 def prepare(self, pipeline_elements: list, maximize_metric: bool): self.hyperparameter_list = [] # build space self.space = ConfigurationSpace() for element in pipeline_elements: # check if Switch object if isinstance(element, Switch): algorithm_options = ( {} ) # mapping algorithm name with their child hyper params for algo in element.elements: algo_params = [] # hyper params corresponding to "algo" for name, value in algo.hyperparameters.items(): smac_param = self._convert_PHOTON_to_smac_space( value, (element.name + "__" + name) ) # or element.name__algo.name__name ??? algo_params.append(smac_param) algorithm_options[(element.name + "__" + algo.name)] = algo_params algos = CategoricalHyperparameter( name=element.name + "__algos", choices=algorithm_options.keys() ) self.space.add_hyperparameter(algos) for algo, params in algorithm_options.items(): for param in params: cond = InCondition(child=param, parent=algos, values=[algo]) self.space.add_hyperparameter(param) self.space.add_condition(cond) else: for name, value in element.hyperparameters.items(): smac_param = self._convert_PHOTON_to_smac_space(value, name) if smac_param is not None: self.space.add_hyperparameter(smac_param) self.scenario = Scenario( { "run_obj": self.run_obj, "cutoff_time": self.cutoff_time, "runcount_limit": self.runcount_limit, "tuner-timeout": self.tuner_timeout, "wallclock_limit": self.wallclock_limit, "cs": self.space, "deterministic": "true", } ) self.smac = SMAC4BO(scenario=self.scenario, rng=np.random.RandomState(42)) self.optimizer = self.smac.solver self.optimizer.stats.start_timing() self.optimizer.incumbent = self.get_default_incumbent() self.flag = ( False ) # False: compute performance of challenger, True: compute performance of incumbent self.ask = self.ask_generator() def get_default_incumbent(self): instantiated_hyperparameters = {} for hp in self.space.get_hyperparameters(): conditions = self.space._get_parent_conditions_of(hp.name) active = True for condition in conditions: parent_names = [ c.parent.name for c in condition.get_descendant_literal_conditions() ] parents = { parent_name: instantiated_hyperparameters[parent_name] for parent_name in parent_names } if not condition.evaluate(parents): # TODO find out why a configuration is illegal! active = False if not active: instantiated_hyperparameters[hp.name] = None elif isinstance(hp, Constant): instantiated_hyperparameters[hp.name] = hp.value else: instantiated_hyperparameters[hp.name] = hp.default_value config = Configuration(self.space, instantiated_hyperparameters) return config def _convert_PHOTON_to_smac_space(self, hyperparam: object, name: str): if not hyperparam: return None self.hyperparameter_list.append(name) if isinstance(hyperparam, PhotonCategorical): return CategoricalHyperparameter(choices=hyperparam.values, name=name) elif isinstance(hyperparam, list): return CategoricalHyperparameter(choices=hyperparam, name=name) elif isinstance(hyperparam, FloatRange): return UniformFloatHyperparameter( lower=hyperparam.start, upper=hyperparam.stop, name=name ) elif isinstance(hyperparam, IntegerRange): return UniformIntegerHyperparameter( lower=hyperparam.start, upper=hyperparam.stop, name=name ) elif isinstance(hyperparam, BooleanSwitch): return CategoricalHyperparameter(choices=["true", "false"], name=name) def ask_generator(self): while True: self.flag = False start_time = time.time() X, Y = self.optimizer.rh2EPM.transform(self.optimizer.runhistory) self.optimizer.logger.debug("Search for next configuration.") # get all configurations sorted according to acquision function challengers = self.optimizer.choose_next(X, Y) self.test += 1 print("TEST # of trains", self.test) time_spent = time.time() - start_time time_left = self.optimizer._get_timebound_for_intensification(time_spent) self.to_run = self.intensify( challengers=challengers, incumbent=self.optimizer.incumbent, run_history=self.optimizer.runhistory, aggregate_func=self.optimizer.aggregate_func, time_bound=max(self.optimizer.intensifier._min_time, time_left), ) if self.flag: if self.optimizer.stats.is_budget_exhausted(): # yield self.optimizer.incumbent.get_dictionary() return None else: yield self.check(self.to_run.get_dictionary()) else: print("Size of challenger list: ", len(self.to_run)) for challenger in self.to_run[: min(len(self.to_run), 25)]: if self.optimizer.stats.is_budget_exhausted(): # yield self.optimizer.incumbent.get_dictionary() return None else: yield self.check(challenger.get_dictionary()) logger.debug( "Remaining budget: %f (wallclock), %f (ta costs), %f (target runs)" % ( self.optimizer.stats.get_remaing_time_budget(), self.optimizer.stats.get_remaining_ta_budget(), self.optimizer.stats.get_remaining_ta_runs(), ) ) self.optimizer.stats.print_stats(debug_out=True) def check(self, config): config_dict = {} self.flag0 = False for k, val in config.items(): if not "algos" in k: config_dict[k] = val else: self.temp = (k, val) self.flag0 = True return config_dict def intensify( self, challengers, incumbent, run_history, aggregate_func, time_bound ): instance_specifics = {} _num_run = 0 _chall_indx = 0 _ta_time = 0 _min_time = 10 ** -5 start_time = time.time() to_run = None if time_bound < _min_time: raise ValueError("time_bound must be >= %f" % (_min_time)) to_run_configs = [] for challenger in challengers: if challenger == incumbent: continue if self.hashable_dict(incumbent.get_dictionary()) not in self.memory: self.flag = True return incumbent to_run = self.race_challenger(challenger=challenger) if to_run is not None: to_run_configs.append(to_run) return to_run_configs def race_challenger(self, challenger): if self.hashable_dict(challenger.get_dictionary()) not in self.memory: return challenger else: return None def compare_configs(self, incumbent, challenger): inc_perf = self.memory[self.hashable_dict(incumbent.get_dictionary())] chal_perf = self.memory[self.hashable_dict(challenger.get_dictionary())] if inc_perf > chal_perf: return incumbent else: return challenger def hashable_dict(self, config: dict): return frozenset(config.items()) def tell(self, config: dict, performance: float, runtime: float = 2.0): if self.flag0: config[self.temp[0]] = self.temp[1] performance = performance[1] self.optimizer.stats.ta_runs += 1 self.optimizer.stats.ta_time_used += runtime hash_config = self.hashable_dict(config) self.memory[hash_config] = performance # convert dictionary to list in correct order config = Configuration(self.space, values=config) if self.run_obj == "runtime": performance = -runtime if runtime > self.cutoff_time: runtime = self.cutoff_time self.optimizer.runhistory.add( config=config, cost=performance, time=runtime, status=StatusType.TIMEOUT, ) else: self.optimizer.runhistory.add( config=config, cost=performance, time=runtime, status=StatusType.SUCCESS, ) else: self.optimizer.runhistory.add( config=config, cost=performance, time=runtime, status=StatusType.SUCCESS ) if self.flag: pass else: self.optimizer.incumbent = self.compare_configs( self.optimizer.incumbent, config )
def get_cs(): cs = ConfigurationSpace() root = CategoricalHyperparameter("root", choices=["l1", "ln"]) x1 = CategoricalHyperparameter("x1", choices=["l1", "ln"]) x2 = CategoricalHyperparameter("x2", choices=["l1", "ln"]) x3 = CategoricalHyperparameter("x3", choices=["l1", "ln"]) x4 = CategoricalHyperparameter("x4", choices=["l1", "ln"]) x5 = CategoricalHyperparameter("x5", choices=["l1", "ln"]) x6 = CategoricalHyperparameter("x6", choices=["l1", "ln"]) # r1 is the data associated in x1 r1_1 = UniformFloatHyperparameter("r1_1", lower=0.01, upper=0.99, log=False) r1_2 = UniformFloatHyperparameter("r1_2", lower=0.01, upper=0.99, log=False) r1_3 = UniformFloatHyperparameter("r1_3", lower=0.01, upper=0.99, log=False) r1_4 = UniformFloatHyperparameter("r1_4", lower=0.01, upper=0.99, log=False) r2_1 = UniformFloatHyperparameter("r2_1", lower=0.01, upper=0.99, log=False) r2_2 = UniformFloatHyperparameter("r2_2", lower=0.01, upper=0.99, log=False) r2_3 = UniformFloatHyperparameter("r2_3", lower=0.01, upper=0.99, log=False) r2_4 = UniformFloatHyperparameter("r2_4", lower=0.01, upper=0.99, log=False) r3_1 = UniformFloatHyperparameter("r3_1", lower=0.01, upper=0.99, log=False) r3_2 = UniformFloatHyperparameter("r3_2", lower=0.01, upper=0.99, log=False) r3_3 = Constant("r3_3", 0.5) r3_4 = Constant("r3_4", 0.5) r4_1 = UniformFloatHyperparameter("r4_1", lower=0.01, upper=0.99, log=False) r4_2 = UniformFloatHyperparameter("r4_2", lower=0.01, upper=0.99, log=False) r4_3 = Constant("r4_3", 0.5) r4_4 = Constant("r4_4", 0.5) r5_1 = UniformFloatHyperparameter("r5_1", lower=0.01, upper=0.99, log=False) r5_2 = UniformFloatHyperparameter("r5_2", lower=0.01, upper=0.99, log=False) r5_3 = Constant("r5_3", 0.5) r5_4 = Constant("r5_4", 0.5) r6_1 = UniformFloatHyperparameter("r6_1", lower=0.01, upper=0.99, log=False) r6_2 = UniformFloatHyperparameter("r6_2", lower=0.01, upper=0.99, log=False) r6_3 = UniformFloatHyperparameter("r6_3", lower=0.01, upper=0.99, log=False) r6_4 = Constant("r6_4", 0.5) r7_1 = UniformFloatHyperparameter("r7_1", lower=0.01, upper=0.99, log=False) r7_2 = UniformFloatHyperparameter("r7_2", lower=0.01, upper=0.99, log=False) r7_3 = Constant("r7_3", 0.5) r7_4 = Constant("r7_4", 0.5) r8_1 = UniformFloatHyperparameter("r8_1", lower=0.01, upper=0.99, log=False) r8_2 = UniformFloatHyperparameter("r8_2", lower=0.01, upper=0.99, log=False) r8_3 = Constant("r8_3", 0.5) r8_4 = Constant("r8_4", 0.5) r9_1 = UniformFloatHyperparameter("r9_1", lower=0.01, upper=0.99, log=False) r9_2 = UniformFloatHyperparameter("r9_2", lower=0.01, upper=0.99, log=False) r9_3 = Constant("r9_3", 0.5) r9_4 = Constant("r9_4", 0.5) r10_1 = UniformFloatHyperparameter("r10_1", lower=0.01, upper=0.99, log=False) r10_2 = UniformFloatHyperparameter("r10_2", lower=0.01, upper=0.99, log=False) r10_3 = Constant("r10_3", 0.5) r10_4 = Constant("r10_4", 0.5) r11_1 = UniformFloatHyperparameter("r11_1", lower=0.01, upper=0.99, log=False) r11_2 = UniformFloatHyperparameter("r11_2", lower=0.01, upper=0.99, log=False) r11_3 = Constant("r11_3", 0.5) r11_4 = Constant("r11_4", 0.5) r12_1 = UniformFloatHyperparameter("r12_1", lower=0.01, upper=0.99, log=False) r12_2 = UniformFloatHyperparameter("r12_2", lower=0.01, upper=0.99, log=False) r12_3 = Constant("r12_3", 0.5) r12_4 = Constant("r12_4", 0.5) r13_1 = UniformFloatHyperparameter("r13_1", lower=0.01, upper=0.99, log=False) r13_2 = UniformFloatHyperparameter("r13_2", lower=0.01, upper=0.99, log=False) r13_3 = Constant("r13_3", 0.5) r13_4 = Constant("r13_4", 0.5) r14_1 = UniformFloatHyperparameter("r14_1", lower=0.01, upper=0.99, log=False) r14_2 = UniformFloatHyperparameter("r14_2", lower=0.01, upper=0.99, log=False) r14_3 = Constant("r14_3", 0.5) r14_4 = Constant("r14_4", 0.5) cs.add_hyperparameters([ root, x1, x2, x3, x4, x5, x6, r1_1, r1_2, r1_3, r1_4, r2_1, r2_2, r2_3, r2_4, r3_1, r3_2, r3_3, r3_4, r4_1, r4_2, r4_3, r4_4, r5_1, r5_2, r5_3, r5_4, r6_1, r6_2, r6_3, r6_4, r7_1, r7_2, r7_3, r7_4, r8_1, r8_2, r8_3, r8_4, r9_1, r9_2, r9_3, r9_4, r10_1, r10_2, r10_3, r10_4, r11_1, r11_2, r11_3, r11_4, r12_1, r12_2, r12_3, r12_4, r13_1, r13_2, r13_3, r13_4, r14_1, r14_2, r14_3, r14_4, ]) # add condition cs.add_condition(InCondition(x1, root, ["l1"])) cs.add_condition(InCondition(x2, root, ["ln"])) cs.add_condition(InCondition(r1_1, root, ["l1"])) cs.add_condition(InCondition(r1_2, root, ["l1"])) cs.add_condition(InCondition(r1_3, root, ["l1"])) cs.add_condition(InCondition(r1_4, root, ["l1"])) cs.add_condition(InCondition(r2_1, root, ["ln"])) cs.add_condition(InCondition(r2_2, root, ["ln"])) cs.add_condition(InCondition(r2_3, root, ["ln"])) cs.add_condition(InCondition(r2_4, root, ["ln"])) cs.add_condition(InCondition(x3, x1, ["l1"])) cs.add_condition(InCondition(x4, x1, ["ln"])) cs.add_condition(InCondition(r3_1, x1, ["l1"])) cs.add_condition(InCondition(r3_2, x1, ["l1"])) cs.add_condition(InCondition(r3_3, x1, ["l1"])) cs.add_condition(InCondition(r3_4, x1, ["l1"])) cs.add_condition(InCondition(r4_1, x1, ["ln"])) cs.add_condition(InCondition(r4_2, x1, ["ln"])) cs.add_condition(InCondition(r4_3, x1, ["ln"])) cs.add_condition(InCondition(r4_4, x1, ["ln"])) cs.add_condition(InCondition(x5, x2, ["l1"])) cs.add_condition(InCondition(x6, x2, ["ln"])) cs.add_condition(InCondition(r5_1, x2, ["l1"])) cs.add_condition(InCondition(r5_2, x2, ["l1"])) cs.add_condition(InCondition(r5_3, x2, ["l1"])) cs.add_condition(InCondition(r5_4, x2, ["l1"])) cs.add_condition(InCondition(r6_1, x2, ["ln"])) cs.add_condition(InCondition(r6_2, x2, ["ln"])) cs.add_condition(InCondition(r6_3, x2, ["ln"])) cs.add_condition(InCondition(r6_4, x2, ["ln"])) cs.add_condition(InCondition(r7_1, x3, ["l1"])) cs.add_condition(InCondition(r7_2, x3, ["l1"])) cs.add_condition(InCondition(r7_3, x3, ["l1"])) cs.add_condition(InCondition(r7_4, x3, ["l1"])) cs.add_condition(InCondition(r8_1, x3, ["ln"])) cs.add_condition(InCondition(r8_2, x3, ["ln"])) cs.add_condition(InCondition(r8_3, x3, ["ln"])) cs.add_condition(InCondition(r8_4, x3, ["ln"])) cs.add_condition(InCondition(r9_1, x4, ["l1"])) cs.add_condition(InCondition(r9_2, x4, ["l1"])) cs.add_condition(InCondition(r9_3, x4, ["l1"])) cs.add_condition(InCondition(r9_4, x4, ["l1"])) cs.add_condition(InCondition(r10_1, x4, ["ln"])) cs.add_condition(InCondition(r10_2, x4, ["ln"])) cs.add_condition(InCondition(r10_3, x4, ["ln"])) cs.add_condition(InCondition(r10_4, x4, ["ln"])) cs.add_condition(InCondition(r11_1, x5, ["l1"])) cs.add_condition(InCondition(r11_2, x5, ["l1"])) cs.add_condition(InCondition(r11_3, x5, ["l1"])) cs.add_condition(InCondition(r11_4, x5, ["l1"])) cs.add_condition(InCondition(r12_1, x5, ["ln"])) cs.add_condition(InCondition(r12_2, x5, ["ln"])) cs.add_condition(InCondition(r12_3, x5, ["ln"])) cs.add_condition(InCondition(r12_4, x5, ["ln"])) cs.add_condition(InCondition(r13_1, x6, ["l1"])) cs.add_condition(InCondition(r13_2, x6, ["l1"])) cs.add_condition(InCondition(r13_3, x6, ["l1"])) cs.add_condition(InCondition(r13_4, x6, ["l1"])) cs.add_condition(InCondition(r14_1, x6, ["ln"])) cs.add_condition(InCondition(r14_2, x6, ["ln"])) cs.add_condition(InCondition(r14_3, x6, ["ln"])) cs.add_condition(InCondition(r14_4, x6, ["ln"])) return cs
class ClusteringArmThread: def __init__(self, name, metric, X): self.thread_name = name self.metric = metric self.X = X self.clu_cs = ConfigurationSpace() if (name == Constants.kmeans_algo): algorithm = CategoricalHyperparameter("algorithm", ["auto", "full", "elkan"]) tol = UniformFloatHyperparameter("tol", 1e-6, 1e-2) n_clusters = UniformIntegerHyperparameter("n_clusters", 2, 15) n_init = UniformIntegerHyperparameter("n_init", 2, 15) max_iter = UniformIntegerHyperparameter("max_iter", 50, 1500) verbose = Constant("verbose", 0) self.clu_cs.add_hyperparameters([n_clusters, n_init, max_iter, tol, verbose, algorithm]) elif (name == Constants.affinity_algo): damping = UniformFloatHyperparameter("damping", 0.5, 1.0) max_iter = UniformIntegerHyperparameter("max_iter", 100, 1000) convergence_iter = UniformIntegerHyperparameter("convergence_iter", 5, 20) self.clu_cs.add_hyperparameters([damping, max_iter, convergence_iter]) elif (name == Constants.mean_shift_algo): quantile = UniformFloatHyperparameter("quantile", 0.0, 1.0) bin_seeding = UniformIntegerHyperparameter("bin_seeding", 0, 1) min_bin_freq = UniformIntegerHyperparameter("min_bin_freq", 1, 100) cluster_all = UniformIntegerHyperparameter("cluster_all", 0, 1) self.clu_cs.add_hyperparameters([quantile, bin_seeding, min_bin_freq, cluster_all]) elif (name == Constants.ward_algo): linkage = CategoricalHyperparameter("linkage", ["ward", "complete", "average"]) affinity_all = CategoricalHyperparameter("affinity_a", ["euclidean", "l1", "l2", "manhattan", "cosine", "precomputed"]) affinity_ward = CategoricalHyperparameter("affinity_w", ["euclidean"]) n_clusters = UniformIntegerHyperparameter("n_clusters", 2, 15) self.clu_cs.add_hyperparameters([n_clusters, affinity_all, affinity_ward, linkage]) self.clu_cs.add_condition(InCondition(child=affinity_ward, parent=linkage, values=["ward"])) self.clu_cs.add_condition( InCondition(child=affinity_all, parent=linkage, values=["ward", "complete", "average"])) elif (name == Constants.dbscan_algo): algorithm = CategoricalHyperparameter("algorithm", ["auto", "ball_tree", "kd_tree", "brute"]) eps = UniformFloatHyperparameter("eps", 0.1, 0.9) min_samples = UniformIntegerHyperparameter("min_samples", 2, 10) leaf_size = UniformIntegerHyperparameter("leaf_size", 5, 100) self.clu_cs.add_hyperparameters([eps, min_samples, algorithm, leaf_size]) elif (name == Constants.gm_algo): cov_t = CategoricalHyperparameter("covariance_type", ["full", "tied", "diag", "spherical"]) tol = UniformFloatHyperparameter("tol", 1e-6, 0.1) reg_c = UniformFloatHyperparameter("reg_covar", 1e-10, 0.1) n_com = UniformIntegerHyperparameter("n_components", 2, 15) max_iter = UniformIntegerHyperparameter("max_iter", 10, 1000) self.clu_cs.add_hyperparameters([cov_t, tol, reg_c, n_com, max_iter]) elif (name == Constants.bgm_algo): cov_t = CategoricalHyperparameter("covariance_type", ["full", "tied", "diag", "spherical"]) tol = UniformFloatHyperparameter("tol", 1e-6, 0.1) reg_c = UniformFloatHyperparameter("reg_covar", 1e-10, 0.1) wcp = UniformFloatHyperparameter("weight_concentration_prior", 1e-10, 0.1) mpp = UniformFloatHyperparameter("mean_precision_prior", 1e-10, 0.1) n_com = UniformIntegerHyperparameter("n_components", 2, 15) max_iter = UniformIntegerHyperparameter("max_iter", 10, 1000) self.clu_cs.add_hyperparameters([wcp, mpp, cov_t, tol, reg_c, n_com, max_iter]) def clu_run(self, cfg): cl = None if (self.thread_name == Constants.kmeans_algo): cl = KMeans(**cfg) elif (self.thread_name == Constants.affinity_algo): cl = AffinityPropagation(**cfg) elif (self.thread_name == Constants.mean_shift_algo): bandwidth = estimate_bandwidth(self.X, quantile=cfg['quantile']) cl = MeanShift(bandwidth=bandwidth, bin_seeding=bool(cfg['bin_seeding']), min_bin_freq=cfg['min_bin_freq'], cluster_all=bool(cfg['cluster_all'])) elif (self.thread_name == Constants.ward_algo): linkage = cfg["linkage"] aff = "" if ("ward" in linkage): aff = cfg["affinity_w"] else: aff = cfg["affinity_a"] n_c = cfg["n_clusters"] cl = AgglomerativeClustering(n_clusters=n_c, linkage=linkage, affinity=aff) elif (self.thread_name == Constants.dbscan_algo): cl = DBSCAN(**cfg) elif (self.thread_name == Constants.gm_algo): cl = GaussianMixture(**cfg) elif (self.thread_name == Constants.bgm_algo): cl = BayesianGaussianMixture(**cfg) cl.fit(self.X) if (self.thread_name == Constants.gm_algo) or (self.thread_name == Constants.bgm_algo): labels = cl.predict(self.X) else: labels = cl.labels_ # labels_unique = np.unique(labels) # n_clusters = len(labels_unique) value = self.metric(self.X, labels) # print(self.thread_name + " n_clusters=" + str(n_clusters) + " metric=" + str(value)) return value
def main_loop(problem): logging.basicConfig(level=logging.INFO) # logging.DEBUG for debug output cs = ConfigurationSpace() n_estimators = UniformIntegerHyperparameter("n_estimators", 5, 50, default_value=10) #criterion = CategoricalHyperparameter("criterion", ["mse", "mae"], default_value="mse") min_samples_split = UniformIntegerHyperparameter("min_samples_split", 2, 20, default_value=2) min_samples_leaf = UniformIntegerHyperparameter("min_samples_leaf", 1, 20, default_value=1) min_weight_fraction_leaf = UniformFloatHyperparameter( "min_weight_fraction_leaf", 0.0, 0.5, default_value=0.0) max_leaf_nodes = UniformIntegerHyperparameter("max_leaf_nodes", 10, 1000, default_value=100) min_impurity_decrease = UniformFloatHyperparameter("min_impurity_decrease", 0.0, 0.5, default_value=0.0) warm_start = CategoricalHyperparameter("warm_start", ["true", "false"], default_value="false") cs.add_hyperparameters([ n_estimators, min_weight_fraction_leaf, min_samples_split, min_samples_leaf, max_leaf_nodes, warm_start, min_impurity_decrease ]) max_features = CategoricalHyperparameter( "max_features", ["auto", "log2", "sqrt", "int", "None", "float"], default_value="auto") # only rbf, poly, sigmoid max_features_int = UniformIntegerHyperparameter("max_features_int", 2, len(X[0]), default_value=5) max_features_float = UniformFloatHyperparameter("max_features_float", 0.0, 0.9, default_value=0.0) cs.add_hyperparameters( [max_features, max_features_int, max_features_float]) use_max_features_int = InCondition(child=max_features_int, parent=max_features, values=["int"]) use_max_features_float = InCondition(child=max_features_float, parent=max_features, values=["float"]) cs.add_conditions([use_max_features_int, use_max_features_float]) max_depth = CategoricalHyperparameter("max_depth", ["None", "value"], default_value="None") max_depth_value = UniformIntegerHyperparameter("max_depth_value", 2, 20, default_value=5) cs.add_hyperparameters([max_depth, max_depth_value]) cs.add_condition( InCondition(child=max_depth_value, parent=max_depth, values=["value"])) random_state = CategoricalHyperparameter("random_state", ["None", "value"], default_value="None") random_state_value = UniformIntegerHyperparameter("random_state_value", 1, 20, default_value=1) cs.add_hyperparameters([random_state, random_state_value]) cs.add_condition( InCondition(child=random_state_value, parent=random_state, values=["value"])) with open("/home/naamah/Documents/CatES/result_All/X1.p", "rb") as fp: X = pickle.load(fp) # Scenario object max_eval = 100000 scenario = Scenario({ "run_obj": "quality", # we optimize quality (alternatively runtime) "runcount-limit": max_eval, # maximum function evaluations "cs": cs, # configuration space "shared_model": True, "output_dir": "/home/naamah/Documents/CatES/result_All/smac/RF/run_{}_{}_{}".format( max_eval, datetime.datetime.fromtimestamp( time.time()).strftime('%Y-%m-%d_%H:%M:%S'), problem), "input_psmac_dirs": "/home/naamah/Documents/CatES/result_All/smac/psmac", "deterministic": "False" }) def_value = svm_from_cfg(cs.get_default_configuration()) print("Default Value: %.2f" % (def_value)) # Optimize, using a SMAC-object print( "Optimizing! Depending on your machine, this might take a few minutes." ) smac = SMAC(scenario=scenario, tae_runner=svm_from_cfg) incumbent = smac.optimize() inc_value = svm_from_cfg(incumbent) print("Optimized Value: %.2f" % (inc_value)) return (incumbent) # main_loop()
class SMACOptimizer(PhotonMasterOptimizer): def __init__(self, facade='SMAC4HPO', scenario_dict=None, intensifier_kwargs=None, rng=42, smac_helper=None): """ SMAC Wrapper for PHOTON. SMAC usage and implementation details: https://automl.github.io/SMAC3/master/quickstart.html Parameters ---------- * `facade` [str or smac.facade.class, default: 'SMAC4HPO']: Choice of SMAC backend strategy, [SMAC4BO, SMAC4HPO, SMAC4AC]. * `scenario_dict` [dict, default: None (warning scenario with wallclock_limit = 60*40)] Informations for scenario settings like run_limit or wallclock_limit. Different to main SMAC cs (configspace) is not required or used cause PHOTON translate own param_space to SMAC.configspace. * `rng`: [int, default: 42] random seed of SMAC.facade * `smac_helper` [dict] For testing this object give public access to SMAC.facade object. Currently help object for test cases. """ super(SMACOptimizer, self).__init__() if not __found__: msg = "Module smac not found or not installed as expected. " \ "Please install the smac_requirements.txt PHOTON provides." logger.error(msg) raise ModuleNotFoundError(msg) if not scenario_dict: self.scenario_dict = {"run_obj": "quality", "deterministic": "true", "wallclock_limit": 60 * 40} msg = "No scenario_dict for smac was given. Falling back to default values: {}.".format(self.scenario_dict) logger.warning(msg) else: self.scenario_dict = scenario_dict if facade in ["SMAC4BO", SMAC4BO, "SMAC4AC", SMAC4AC, "SMAC4HPO", SMAC4HPO, "BOHB4HPO", BOHB4HPO]: if type(facade) == str: self.facade = eval(facade) else: self.facade = facade else: msg = "SMAC.facade {} not known. Please use one of ['SMAC4BO', 'SMAC4AC', 'SMAC4HPO']." logger.error(msg.format(str(facade))) raise ValueError(msg.format(str(facade))) self.rng = rng if not intensifier_kwargs: self.intensifier_kwargs = {} else: self.intensifier_kwargs = intensifier_kwargs self.cspace = ConfigurationSpace() # Hyperparameter space for SMAC self.switch_optiones = {} self.hyperparameters = [] if smac_helper: self.smac_helper = smac_helper self.debug = True else: self.debug = False self.maximize_metric = False @staticmethod def _convert_photon_to_smac_param(hyperparam: object, name: str): """ Helper function: Convert PHOTON hyperparameter to SMAC hyperparameter. Parameters ---------- * `hyperparam` [object]: One of photonai.optimization.hyperparameters. * `name` [str] Name of hyperparameter. """ if not hyperparam: return None if isinstance(hyperparam, PhotonCategorical): return CategoricalHyperparameter(name, hyperparam.values) elif isinstance(hyperparam, list): return CategoricalHyperparameter(name, hyperparam) elif isinstance(hyperparam, FloatRange): if hyperparam.range_type == 'linspace': return UniformFloatHyperparameter(name, hyperparam.start, hyperparam.stop) elif hyperparam.range_type == 'logspace': raise NotImplementedError("Logspace in your float hyperparameter is not implemented in SMAC.") else: return UniformFloatHyperparameter(name, hyperparam.start, hyperparam.stop) elif isinstance(hyperparam, IntegerRange): return UniformIntegerHyperparameter(name, hyperparam.start, hyperparam.stop) def build_smac_space(self, pipeline_elements): """ Build entire SMAC hyperparameter space. Parameters ---------- * `pipeline_elements` [list]: List of all pipeline_elements to create hyperparameter_space. """ for pipe_element in pipeline_elements: # build conditions for switch elements if pipe_element.__class__.__name__ == 'Switch': algorithm_options = {} for algo in pipe_element.elements: algo_params = [] # hyper params corresponding to "algo" for name, value in algo.hyperparameters.items(): smac_param = self._convert_photon_to_smac_param(value, ( pipe_element.name + "__" + name)) # or element.name__algo.name__name algo_params.append(smac_param) algorithm_options[(pipe_element.name + "__" + algo.name)] = algo_params algos = CategoricalHyperparameter(pipe_element.name + "__algos", choices=algorithm_options.keys()) self.switch_optiones[pipe_element.name + "__algos"] = algorithm_options.keys() self.cspace.add_hyperparameter(algos) for algo, params in algorithm_options.items(): for param in params: cond = InCondition(child=param, parent=algos, values=[algo]) self.cspace.add_hyperparameter(param) self.cspace.add_condition(cond) continue if hasattr(pipe_element, 'hyperparameters'): for name, value in pipe_element.hyperparameters.items(): self.hyperparameters.append(name) # if we only have one value we do not need to optimize if isinstance(value, list) and len(value) < 2: self.constant_dictionary[name] = value[0] continue if isinstance(value, PhotonCategorical) and len(value.values) < 2: self.constant_dictionary[name] = value.values[0] continue smac_param = self._convert_photon_to_smac_param(value, name) if smac_param is not None: self.cspace.add_hyperparameter(smac_param) def prepare(self, pipeline_elements: list, maximize_metric: bool, objective_function): """ Initializes SMAC Optimizer. Parameters ---------- * `pipeline_elements` [list]: List of all pipeline_elements to create hyperparameter_space. * `maximize_metric` [bool]: Boolean for distinguish between score and error. * `objective_function` [callable]: The cost or objective function. """ self.space = ConfigurationSpace() # build space self.build_smac_space(pipeline_elements) self.maximize_metric = maximize_metric self.scenario_dict["cs"] = self.cspace self.scenario_dict["limit_resources"] = False self.scenario = Scenario(self.scenario_dict) def smac_objective_function(current_config): current_config = {k: current_config[k] for k in current_config if (current_config[k] and 'algos' not in k)} return objective_function(current_config) self.smac = self.facade(scenario=self.scenario, intensifier_kwargs=self.intensifier_kwargs, rng=self.rng, tae_runner=smac_objective_function) if self.debug: self.smac_helper['data'] = self.smac def optimize(self): """ Start optimization process. """ self.smac.optimize()
def main_loop(problem): #logger = logging.getLogger("SVMExample") logging.basicConfig(level=logging.INFO) # logging.DEBUG for debug output cs = ConfigurationSpace() kernel = CategoricalHyperparameter("kernel", ["rbf", "sigmoid"], default_value="rbf") cs.add_hyperparameter(kernel) C = UniformFloatHyperparameter("C", 1, 1000.0, default_value=900.0) shrinking = CategoricalHyperparameter("shrinking", ["true", "false"], default_value="false") cs.add_hyperparameters([C, shrinking]) #degree = UniformIntegerHyperparameter("degree", 1, 3 ,default_value=3) # Only used by kernel poly coef0 = UniformFloatHyperparameter("coef0", 0.0, 1.0, default_value=0.0) # poly, sigmoid #cs.add_hyperparameters([degree, coef0]) cs.add_hyperparameter(coef0) #use_degree = InCondition(child=degree, parent=kernel, values=["poly"]) use_coef0 = InCondition(child=coef0, parent=kernel, values=["sigmoid"]) #cs.add_conditions([use_degree, use_coef0]) cs.add_condition(use_coef0) gamma = CategoricalHyperparameter( "gamma", ["auto", "value"], default_value="auto") # only rbf, poly, sigmoid gamma_value = UniformFloatHyperparameter("gamma_value", 0.001, 8, default_value=1) cs.add_hyperparameters([gamma, gamma_value]) cs.add_condition( InCondition(child=gamma_value, parent=gamma, values=["value"])) cs.add_condition( InCondition(child=gamma, parent=kernel, values=["rbf", "sigmoid"])) epsilon = UniformFloatHyperparameter("epsilon", 0.001, 5.0, default_value=0.1) cs.add_hyperparameter(epsilon) # Scenario object max_eval = 100000 scenario = Scenario({ "run_obj": "quality", # we optimize quality (alternatively runtime) "runcount-limit": max_eval, # maximum function evaluations "cs": cs, # configuration space "shared_model": True, "output_dir": "/home/naamah/Documents/CatES/result_All/smac/svm/run_{}_{}_{}".format( max_eval, datetime.datetime.fromtimestamp( time.time()).strftime('%Y-%m-%d_%H:%M:%S'), problem), # "input_psmac_dirs": "/home/naamah/Documents/CatES/result_All/smac/svm/{}/run_{}_{}/*".format(problem,max_eval, datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d_%H:%M:%S_%f')), "deterministic": "False" #"instance_file":"/home/naamah/Documents/CatES/result_All", #"test_instance_file":"/home/naamah/Documents/CatES/result_All" }) # Example call of the function # It returns: Status, Cost, Runtime, Additional Infos def_value = svm_from_cfg(cs.get_default_configuration()) print("Default Value: %.2f" % (def_value)) # Optimize, using a SMAC-object print( "Optimizing! Depending on your machine, this might take a few minutes." ) smac = SMAC(scenario=scenario, tae_runner=svm_from_cfg) incumbent = smac.optimize() inc_value = svm_from_cfg(incumbent) print("Optimized Value: %.2f" % (inc_value)) return (incumbent)
def generate_data(smac_class, n_runs=1, output_dir: Union[str, Path] = ".", dataset=None, runcount_limit=50): output_dir = Path(output_dir) if dataset is None: dataset = datasets.load_iris() def svm_from_cfg(cfg): """ Creates a SVM based on a configuration and evaluates it on the iris-dataset using cross-validation. Parameters: ----------- cfg: Configuration (ConfigSpace.ConfigurationSpace.Configuration) Configuration containing the parameters. Configurations are indexable! Returns: -------- A crossvalidated mean score for the svm on the loaded data2-set. """ # For deactivated parameters, the configuration stores None-values. # This is not accepted by the SVM, so we remove them. cfg = {k: cfg[k] for k in cfg if cfg[k]} # We translate boolean values: cfg["shrinking"] = True if cfg["shrinking"] == "true" else False # And for gamma, we set it to a fixed value or to "auto" (if used) if "gamma" in cfg: cfg["gamma"] = cfg["gamma_value"] if cfg[ "gamma"] == "value" else "auto" cfg.pop("gamma_value", None) # Remove "gamma_value" clf = svm.SVC(**cfg, random_state=None) scores = cross_val_score(clf, dataset.data, dataset.target, cv=5) return 1 - np.mean(scores) # Minimize! # logger = logging.getLogger("SVMExample") logging.basicConfig(level=logging.INFO) # logging.DEBUG for debug output # Build Configuration Space which defines all parameters and their ranges cs = ConfigurationSpace() # We define a few possible types of SVM-kernels and add them as "kernel" to our cs kernel = CategoricalHyperparameter("kernel", ["linear", "rbf", "poly", "sigmoid"], default_value="poly") cs.add_hyperparameter(kernel) # There are some hyperparameters shared by all kernels C = UniformFloatHyperparameter("C", 0.001, 1000.0, default_value=1.0) shrinking = CategoricalHyperparameter("shrinking", ["true", "false"], default_value="true") cs.add_hyperparameters([C, shrinking]) # Others are kernel-specific, so we can add conditions to limit the searchspace degree = UniformIntegerHyperparameter( "degree", 1, 5, default_value=3) # Only used by kernel poly coef0 = UniformFloatHyperparameter("coef0", 0.0, 10.0, default_value=0.0) # poly, sigmoid cs.add_hyperparameters([degree, coef0]) use_degree = InCondition(child=degree, parent=kernel, values=["poly"]) use_coef0 = InCondition(child=coef0, parent=kernel, values=["poly", "sigmoid"]) cs.add_conditions([use_degree, use_coef0]) # This also works for parameters that are a mix of categorical and values from a range of numbers # For example, gamma can be either "auto" or a fixed float gamma = CategoricalHyperparameter( "gamma", ["auto", "value"], default_value="auto") # only rbf, poly, sigmoid gamma_value = UniformFloatHyperparameter("gamma_value", 0.0001, 8, default_value=1) cs.add_hyperparameters([gamma, gamma_value]) # We only activate gamma_value if gamma is set to "value" cs.add_condition( InCondition(child=gamma_value, parent=gamma, values=["value"])) # And again we can restrict the use of gamma in general to the choice of the kernel cs.add_condition( InCondition(child=gamma, parent=kernel, values=["rbf", "poly", "sigmoid"])) # Scenario object for i in range(n_runs): scenario = Scenario({ "run_obj": "quality", # we optimize quality (alternatively runtime) "runcount-limit": runcount_limit, # max. number of function evaluations; for this example set to a low number "cs": cs, # configuration space "deterministic": "true", "limit_resources": "false", "output_dir": str((output_dir / smac_class.__name__ / f"{i:02d}").absolute()) }) # Example call of the function # It returns: Status, Cost, Runtime, Additional Infos # def_value = svm_from_cfg(cs.get_default_configuration()) # print(f"Default Value: {def_value:.2f}") # # Optimize, using a SMAC-object smac = smac_class(scenario=scenario, rng=None, tae_runner=svm_from_cfg) incumbent = smac.optimize() # inc_value = svm_from_cfg(incumbent) # # print(f"Optimized Value: {inc_value:.2f}") # # # We can also validate our results (though this makes a lot more sense with instances) smac.validate( config_mode='inc', # We can choose which configurations to evaluate # instance_mode='train+test', # Defines what instances to validate repetitions= 100, # Ignored, unless you set "deterministic" to "false" in line 95 n_jobs=1) # How many cores to use in parallel for optimization
logger = logging.getLogger("Optimizer") # Enable to show Debug outputs logging.basicConfig(level=logging.DEBUG) # build Configuration Space which defines all parameters and their ranges n_params = 16 use_conditionals = True # using conditionals should help a lot in this example cs = ConfigurationSpace() previous_param = None for n in range(n_params): p = CategoricalHyperparameter("%d" % (n), [0, 1], default=0) cs.add_hyperparameter(p) if n > 0 and use_conditionals: cond = InCondition(child=p, parent=previous_param, values=[1]) cs.add_condition(cond) previous_param = p # SMAC scenario object scenario = Scenario({ "run_obj": "quality", # we optimize quality (alternative runtime) "runcount-limit": n_params * 2, # at most 200 function evaluations "cs": cs, # configuration space "deterministic": "true" }) # register function to be optimize taf = ExecuteTAFuncDict(leading_ones) # example call of the function